diff --git a/docs/source/user_guide.rst b/docs/source/user_guide.rst index a8d3c7f8d..394cdff4f 100644 --- a/docs/source/user_guide.rst +++ b/docs/source/user_guide.rst @@ -36,6 +36,7 @@ User Guide user_guide/format_table user_guide/system_tables user_guide/write + user_guide/realtime user_guide/commit user_guide/compaction user_guide/read diff --git a/docs/source/user_guide/realtime.rst b/docs/source/user_guide/realtime.rst new file mode 100644 index 000000000..5033d547a --- /dev/null +++ b/docs/source/user_guide/realtime.rst @@ -0,0 +1,617 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +Real-time Write and Read +======================== + +Real-time mode makes writes visible to queries in the same process before a +snapshot commit. A shared ``RealtimeContext`` owns the uncommitted stores; +queries combine an immutable view of those stores with committed table files. +Readers without that context see only committed files. + +Real-time storage is pluggable through the ``RealtimeStore`` interface. The +default implementation uses Apache Arrow to buffer rows and serve reads. +Applications can provide their own ``RealtimeStoreFactory`` through +``RealtimeContext::Create(factory)`` to customize storage and retrieval while +keeping Paimon's real-time write, read, and commit lifecycle. + +Durability still comes from snapshot commits. Upstream must retain replayable +input and provide monotonically increasing offsets for each partition-bucket. +Real-time operations currently support only the main branch. Use a separate +context for each table, and coordinate a single active writer for each +partition-bucket. + +Modes and Capabilities +---------------------- + +The table schema and options select the mode; there is no separate +``realtime.mode`` option. + +.. list-table:: + :header-rows: 1 + :widths: 18 25 27 30 + + * - Mode + - Selection + - Uncommitted state + - Query semantics + * - Append + - No primary key; no real-time deduplicate key + - Arrow batches in write order + - Concatenate committed files and the visible memory tail + * - PK MOR + - Primary key; ``deletion-vectors.enabled=false`` + - Batches sorted by primary key and sequence number + - Merge disk runs, sealed batches, and building batches by primary key + * - PK DV + - Primary key; ``deletion-vectors.enabled=true`` + - The same sorted PK store as PK MOR + - Apply committed DVs, then merge disk and memory by primary key + * - Append deduplicate + - Append table with ``realtime.deduplicate-key-fields`` set and + ``deletion-vectors.enabled=true`` + - Mark superseded rows in an in-memory DV during writes; persist DV files + at commit + - Full predicate pushdown; apply the in-memory DV to disk and memory reads + without MOR + +All four modes support read-before-prepare, sealed-but-uncommitted reads, +partition-bucket isolation, offset-aware commit and recovery, immutable read +views, projection, and optional sealed-segment spill with the default store. +The shared read pipeline also handles nested projection, selected MAP keys, +and supported VARIANT access over physical store fields. Predicate pruning +and exact residual filtering follow the mode-specific rules below. + +Append accepts ``INSERT`` only. PK and append deduplicate also accept +``UPDATE_AFTER``, ``DELETE``, and ``UPDATE_BEFORE``. PK supports composite +primary keys, but its real-time merge engine is currently limited to +``merge-engine=deduplicate``. Append deduplicate currently accepts exactly one +non-null, non-nested key field and is experimental: its persisted index format +may change. + +PK DV and append deduplicate are different designs. PK DV retains primary-key +merge semantics; append deduplicate uses a persisted stable row offset and +does not turn the table into a primary-key table. + +Configuration +------------- + +Common Options +~~~~~~~~~~~~~~ + +.. list-table:: + :header-rows: 1 + :widths: 33 15 52 + + * - Option + - Default + - Meaning + * - ``realtime.enabled`` + - ``false`` + - Enables real-time operations. The builders must also receive a shared + ``RealtimeContext`` to select process-local writing and union reading. + * - ``realtime.spill-enabled`` + - ``false`` + - The default store synchronously spills frozen Arrow data to temporary + IPC files during explicit ``Seal``. Prepare does not trigger spill. + * - ``realtime.read-view-ttl`` + - ``5 min`` + - Positive lifetime of a scan's read-view ticket before reader creation. + * - ``realtime.store.stats-mode`` + - ``none`` + - ``none`` or ``full``. Full mode collects per-batch min/max and null counts + for conservative predicate pruning, including retained spill metadata. + * - ``realtime.deduplicate-key-fields`` + - Unset + - Selects append deduplicate mode. Although expressed as a comma-separated + list, currently exactly one field is accepted. + * - ``bucket`` + - Must be set + - Real-time mode requires fixed buckets: ``bucket > 0``. + * - ``read.batch-size`` + - ``1024`` + - Maximum raw batch size returned by the default store's query readers; + not the number of visible rows after bitmap filtering. + * - ``spill-compression`` + - ``zstd`` + - Compression used by temporary spill files. + * - ``spill-compression.zstd-level`` + - ``1`` + - Compression level when using Zstandard. + +Set ``WriteContextBuilder::WithTempDirectory()`` when enabling spill. The +default factory rejects spill without a non-empty temporary directory or a +file system. Spill is explicit at seal boundaries; the normal write-buffer +memory manager does not automatically bound real-time store memory. + +Setting ``realtime.enabled=true`` alone does not attach an in-process store. +This permits a separate ordinary writer to compact committed files without +joining the real-time context. It does not permit ordinary row writes into an +append deduplicate table. + +Mode-specific Setup +~~~~~~~~~~~~~~~~~~~ + +For append mode, create a table without primary keys and use: + +.. code-block:: text + + bucket = 4 + realtime.enabled = true + +For PK MOR, define the table's primary keys and use: + +.. code-block:: text + + bucket = 4 + realtime.enabled = true + merge-engine = deduplicate + deletion-vectors.enabled = false + +For PK DV, define primary keys and enable DVs instead: + +.. code-block:: text + + bucket = 4 + realtime.enabled = true + merge-engine = deduplicate + deletion-vectors.enabled = true + +Both PK modes use the same real-time writer and store. PK DV uses the normal +PK deletion-vector and compaction infrastructure; it does not require the +append deduplicate ``.offset`` sidecar or an offset bitmap file index. + +For append deduplicate, create a table with no primary keys. Its schema must +contain ``_REALTIME_OFFSET BIGINT NOT NULL`` and a non-null key, for example +``id BIGINT NOT NULL``. Deduplication is scoped to a partition: the same key may +exist independently in different partitions. The bucket key must use only +deduplicate key fields; applications are responsible for assigning rows to the +correct bucket. For example, use ``bucket-key=id``: + +.. code-block:: text + + bucket = 4 + bucket-key = id + realtime.enabled = true + realtime.deduplicate-key-fields = id + deletion-vectors.enabled = true + deletion-vectors.bitmap64 = false + file-index.read.enabled = true + file-index.bitmap.columns = _REALTIME_OFFSET + +The two indexes serve different purposes: + +* The ``.offset`` sidecar looks up existing offsets by deduplicate key during + writes, so superseded rows can be marked in the in-memory DV. +* ``file-index.bitmap.columns=_REALTIME_OFFSET`` builds an offset index used + during queries to quickly map in-memory DV offsets to exact file row + positions and filter those rows. + +The offset column must have an index supporting exact BIGINT ``IN`` and +``NOT_IN`` row-position matching. Among Paimon's default plugins, supported +choices include ``bitmap``, ``bsi``, and ``range-bitmap``. Indexes may be embedded +in file metadata or stored in an external file-index file. Approximate indexes +such as Bloom filters are insufficient. + +Index capabilities are checked when creating a table and loading its schema, +not for each file during reads. Applications must keep the index configuration +consistent with existing files and ensure that those files have compatible +exact indexes. + +The bitmap64 prohibition concerns persisted file DVs; logical deleted offsets +are still represented by ``RoaringBitmap64`` in memory. + +The ``.offset`` sidecar lookup uses ``cache-page-size``, +``lookup.cache-max-memory-size``, and +``lookup.cache.high-priority-pool-ratio``. These configure its lookup cache, +not a total memory limit for the real-time store. + +Input Schema and Offsets +------------------------ + +Offsets are application-assigned, non-negative ``int64`` values scoped to one +logical partition-bucket. They must be strictly increasing within each input +batch and across subsequent batches. Gaps are allowed; reuse or backward +movement is rejected. ``INT64_MAX`` cannot be an input offset because the +exclusive end must also fit in ``int64``. + +.. list-table:: + :header-rows: 1 + :widths: 20 45 35 + + * - Mode + - ``FileStoreWrite::Write`` input layout + - Persisted offset + * - Append + - ``[_REALTIME_OFFSET, table fields...]`` + - Internal prefix removed before writing data files + * - PK MOR / PK DV + - ``[_REALTIME_OFFSET, table fields...]`` + - Internal prefix removed before writing data files + * - Append deduplicate + - Table schema in its declared field order, including ``_REALTIME_OFFSET`` + - Normal table column retained in data files and compaction output + +Do not add the internal offset prefix to the formal table schema in ordinary +append or PK mode. In append deduplicate mode, do not add a second prefix: the +offset is already a formal field and may occupy any declared position. + +Construct Arrow batches in the required layout and set their partition and +bucket with ``RecordBatchBuilder::SetPartition()`` and ``SetBucket()``. Routing +and offset assignment belong to the application. Deduplication never crosses +partition-bucket boundaries. + +For PK writes, the framework adds ``_SEQUENCE_NUMBER`` and ``_VALUE_KIND`` and +sorts each stored batch by primary key, then sequence number. Input offsets +are validated before sorting. The sequence number, not the numeric offset, +resolves PK versions; application-defined ``sequence.field`` is not supported +by real-time PK mode. + +An ``OffsetRange`` is half-open: ``[begin, end)``. It is an event-progress +envelope, not a row count. For example, input offsets ``10, 12, 15`` cover +``[10, 16)``. Deletes consume offsets even when they produce no data-file row, +so a delete-only prepare can advance progress with an empty data increment. + +API Lifecycle +------------- + +Use the same ``RealtimeContext`` for writing, planning, and reading. Enable +streaming mode on the writer for progress-aware checkpoints; real-time union +queries themselves use batch scans. + +The snippets below run inside a ``Status`` or ``Result``-returning function +using the ``paimon`` namespace. They assume an existing native table, +``table_path``, a stable ``commit_user``, the mode-specific ``options`` map, +and an input ``batch`` built as described above. + +.. code-block:: cpp + + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr realtime, + RealtimeContext::Create()); + + WriteContextBuilder write_builder(table_path, commit_user); + write_builder.SetOptions(options) + .WithStreamingMode(true) + .WithRealtimeContext(realtime); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, + write_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr writer, + FileStoreWrite::Create(std::move(write_context))); + PAIMON_RETURN_NOT_OK(writer->Write(std::move(batch))); + +To query before preparing or committing: + +.. code-block:: cpp + + ScanContextBuilder scan_builder(table_path); + scan_builder.SetOptions(options).WithRealtimeContext(realtime); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, + scan_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, + TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, scan->CreatePlan()); + + ReadContextBuilder read_builder(table_path); + read_builder.SetOptions(options).WithRealtimeContext(realtime); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, + read_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, + TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, + read->CreateReader(plan->Splits())); + // Consume NextBatch() or NextBatchWithBitmap(), then Close() the reader. + +Apply the same query predicate to scan and read. If exact residual filtering +must happen inside the library, call +``ReadContextBuilder::EnablePredicateFilter(true)``. File and batch statistics +are conservative pruning, not a substitute for evaluating the user predicate. + +Checkpoint with progress-aware APIs: + +.. code-block:: cpp + + const int64_t checkpoint_id = 1; + PAIMON_ASSIGN_OR_RAISE(std::vector progress, + writer->PrepareCommitWithProgress(checkpoint_id)); + + CommitContextBuilder commit_builder(table_path, commit_user); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, + commit_builder.SetOptions(options).Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, + FileStoreCommit::Create(std::move(commit_context))); + PAIMON_ASSIGN_OR_RAISE( + int64_t snapshot_id, + commit->CommitWithProgress(progress, checkpoint_id, /*watermark=*/std::nullopt)); + PAIMON_RETURN_NOT_OK(writer->RefreshCommittedSnapshot(snapshot_id)); + PAIMON_RETURN_NOT_OK(writer->Close()); + +The operations have different responsibilities: + +1. ``Write`` publishes rows to the process-local building store. +2. Optional ``Seal`` freezes a segment, opens a new building segment, and may + spill. It does not create a committed snapshot or prepare data-file messages. +3. ``PrepareCommitWithProgress`` freezes the current building segment without + spilling and writes it together with earlier sealed segments to table data + files, returning their offset ranges together with commit messages. It does + not publish a snapshot or reclaim the real-time data. +4. ``CommitWithProgress`` atomically publishes files, indexes, and the updated + partition-bucket offset map. The snapshot's ``realtime.offsets`` property + references a metadata file containing the durable progress. +5. ``RefreshCommittedSnapshot`` loads the committed progress and reclaims + fully covered sealed segments. Deduplicate refresh also replaces the active + file-key lookup atomically with memory/DV advancement. + +Keep every prepared-but-uncommitted ``RealtimeCommitProgress`` entry. If a later +checkpoint subsumes earlier prepares, pass the complete outstanding prefix to +commit, not just the latest prepare result. Offset gaps make omitted prepares +indistinguishable from valid sparse input, and skipping one can cause data loss. + +``CommitMessage`` serialization does not contain the progress range or the +deduplicate ``deleted_offsets`` bitmap. Preserve the complete progress when +coordinating checkpoints. Snapshot conflicts are retried internally against +the latest snapshot; commit identifiers must follow the streaming commit +protocol. With a version-managed catalog, configure the writer and committer +with the same ``WithCatalog()`` catalog and identifier. + +Read Views and Consistency +-------------------------- + +Planning first pins immutable memory views and then plans committed disk data. +For each active partition-bucket it captures two exclusive bounds: + +.. code-block:: text + + disk: progress covered by the selected snapshot + memory: [committed_end_offset, memory_end_offset) + +Filtering memory by these bounds prevents duplicate reads after a commit and +excludes writes after the captured memory view. Existing views remain valid +across later writes, seals, spill publication, and refresh-driven reclamation. + +Ordinary append scans keep earlier disk splits independently schedulable and +attach the memory view to a tail split. PK and append deduplicate group a +partition-bucket's disk splits with its memory view so that merge or pending +deletion semantics cover the whole bucket. + +A real-time split carries an opaque read-view ticket. Resolve it with the same +context in the same process. Successful reader creation consumes the ticket; +failed creation retains it until its TTL expires. Once a reader is created, +the reader pins its view independently of ticket expiry. Replan if a ticket +expires, or if a previously consumed split needs to be read again. + +An explicit snapshot id is accepted only if its progress is not behind the +context's reclaimed progress for relevant active stores. It does not restore +an old in-memory state and is not unrestricted historical time travel. For a +disk-only historical query, do not attach the real-time context. + +Implementation by Mode +---------------------- + +Append +~~~~~~ + +``RealtimeAppendOnlyWriter`` validates offsets and writes to an +``APPEND_ONLY`` store. The default ``ArrowRealtimeStore`` retains building +batches and immutable sealed segments in write order. Query readers project +the requested fields, optionally prune stored batches using statistics, and +apply the visible offset range before removing the hidden offset column. + +Preparing reads sealed segments through the ordinary ``AppendOnlyWriter``, +removing the internal offset prefix before writing table files. Queries do not +need key lookup or version merge; disk and memory readers are concatenated. + +Primary Key MOR +~~~~~~~~~~~~~~~ + +``RealtimePrimaryKeyWriter`` materializes PK transport batches with sequence +number, row kind, and offset. Each sorted batch is an independent merge input; +multiple stored batches do not form a single globally sorted stream. + +Query construction merges sorted disk sections and the memory readers, applies +deduplicate merge semantics, and drops final delete records. It collects all +disk splits belonging to the active partition-bucket, rather than merging +memory against only one disk split. Preparing similarly merges the sealed +streams before handing them to ``MergeTreeWriter`` to write normal PK files. + +Only primary-key predicates are safe to push into unresolved versions. A +predicate on a value column must be evaluated after version merge: filtering +out the newest value first could incorrectly expose an older matching value. + +Primary Key DV +~~~~~~~~~~~~~~ + +PK DV uses the same real-time PK write and sequence-number logic. During a +union read, committed file DVs remove already-invalid physical rows, but L0 +runs and memory may still contain duplicate keys and delete records. The final +reader therefore still performs a primary-key merge across disk and memory. + +For real-time scans, L0 is retained instead of being skipped as in the normal +DV batch scan. Predicate handling distinguishes storage levels: + +* High-level files (``level > 0``) can use the full predicate for statistics and + file-index pruning under committed DV semantics. +* L0 and memory receive only primary-key predicates before merge. +* The full residual predicate is applied after merge when explicitly enabled + in the read context. + +Enabling DVs does not make uncommitted PK data append-only and does not make +arbitrary value-predicate pushdown safe for L0 or memory. + +Append Deduplicate / Offset DV +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This mode uses two independent lookup directions: + +.. code-block:: text + + key -> logical offset(s) memory key indexes / .offset sidecars + logical offset -> physical row positions exact _REALTIME_OFFSET file index + +For each write, ``RealtimeDeduplicateWriter`` looks up existing keys across +committed files, sealed segments, and the building segment. Matching old offsets +are marked deleted, and within-batch duplicates are resolved in row order. +Add rows are appended to the store; delete events update only the key/DV state, +without persisting a delete row. + +Key indexes and DV bitmaps use copy-on-write snapshots, so later writes do not +change the state held by existing lookups, read views, or seal checkpoints. + +Memory reads use the read view's DV to exclude rows whose offsets are marked +deleted. + +Disk reads apply both committed file DVs and pending offset deletions. Pending +deletions become per-file ``_REALTIME_OFFSET NOT IN (...)`` predicates, trimmed +using reliable min/max statistics or left untrimmed when statistics are missing +or ambiguous. The exact file index enforces these predicates independently of +optional residual filtering. The offset is omitted from the final result unless +the user selected it. + +Prepare retains each seal's logical deletion checkpoint and writes surviving +add rows with their original offsets. Each output file gets a ``.offset`` +sidecar mapping key fingerprints to offsets, separate from the exact offset +file index. + +Each commit attempt resolves deleted offsets to row positions in the latest +snapshot's files and the prepared new files, then merges them into the committed +file DVs. Repeating this resolution on retries handles file changes caused by +conflicting commits or compaction. + +Spill and Memory Accounting +--------------------------- + +The default store freezes the current building data and opens a new building +segment under a short lock. Explicit ``Seal`` writes the frozen segment to an +Arrow IPC spill file outside the write lock when spill is enabled. Prepare +freezes without spilling and writes directly to table data files, reading any +earlier spilled segments as needed. There is no background spill worker: the +seal/prepare thread waits for its I/O, while another thread can write to the +new building segment. + +Deduplicate uses two-phase seal to freeze data, key index, offset range, and DV +checkpoint at the same boundary. The frozen segment remains queryable and its +keys remain discoverable while spill is in progress. Only after writing and +closing the file successfully is its stored representation replaced. A spill +failure keeps the in-memory segment and cleans up the incomplete file, but is +still terminal for the owning writer/context. + +Old views can retain the memory representation after spill or reclamation; +new views use the published representation. Spill files are deleted when the +store and all remaining handles/readers have released them. They are temporary +storage, not a recoverable log after process restart. Deduplicate key indexes +and DVs remain in memory; full-mode statistics also remain resident after spill. + +``RealtimeContext::GetMetrics()`` and real-time writer metrics expose: + +* ``realtimeBuildingMemoryBytes``, ``realtimeSealedMemoryBytes``, + ``realtimeTotalMemoryBytes``; +* ``realtimeBuildingRowCount``, ``realtimeSealedRowCount``, + ``realtimeTotalRowCount``. + +These are current store accounting, not process RSS or a visible-row count. +Rows include stored versions hidden by merge/DV filtering. Memory counts include +Arrow buffers and retained statistics; deduplicate also counts retained key-map +buffers, row-offset bitmaps, and distinct deletion checkpoints. + +Recovery, Schema Changes, and Compaction +---------------------------------------- + +Recovery and Failure Handling +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Read durable progress with ``RealtimeSnapshotOffsets::ReadAllOffsets()`` or +``ReadOffset()`` from a committed snapshot. Replay starting at the returned +offset. If ``ReadOffset()`` returns ``-1`` for a partition-bucket, use the +upstream's initial position. + +A new context restores committed progress, but uncommitted data requires +upstream replay. Reusing a healthy context preserves its uncommitted tail; +retain any outstanding prepared progress when replacing the writer. + +After a write/flush, seal, prepare, or terminal progress-commit failure, discard +the writer and context and recover from the latest committed progress. Closing +with unprepared data also invalidates the context. Prepare alone is not durable: +retain prepared progress until commit, even after the writer closes. + +Schema Changes +~~~~~~~~~~~~~~ + +Reusing a live store requires the same physical schema and mode. After a schema +change, recreate the context and writer and replay uncommitted input. +``data-evolution.enabled`` is not supported in real-time mode. + +Deduplicate reads and lookups support historical file schemas, which must remain +available together with their key options. Key renames and field reordering are +supported when key field ID, type, and non-nullability stay compatible. +``_REALTIME_OFFSET`` must retain its name, field ID, and non-null int64 type. + +Compaction and Destructive Operations +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The real-time writer does not compact files automatically or support explicit +``Compact``. Use a separate ordinary compaction writer without +``WithRealtimeContext()``, then refresh to the resulting snapshot. + +Append deduplicate compaction preserves surviving offsets, rebuilds indexes, +and remaps the latest deletions to the output files. External rewrites must not +drop the required offsets or indexes. + +Overwrite, truncate, partition drop, and rollback may reset progress and are not +automatically coordinated with active real-time writers. Stop affected writers, +recreate their contexts, and reset the upstream recovery position before resuming. + +Extension Points and Current Limits +----------------------------------- + +``RealtimeContext::Create(factory)`` accepts a custom ``RealtimeStoreFactory``. +Custom stores manage buffering, read views, and spill while following the +``RealtimeStore`` contracts. Paimon manages table files, indexes, merge semantics, +and commits. Append deduplicate also keeps key indexes and DVs in the framework; +custom delegates need ``RealtimeStoreTwoPhaseSeal`` to enable its spill. + +Current restrictions include: + +* Native tables on the main branch with fixed positive bucket counts and full + write schemas only. +* Process-local batch union scans only. Streaming scans, limit pushdown, + global-index result splits, vector/full-text search, and count-only real-time + readers are unsupported. +* PK mode supports the deduplicate merge engine with default delete handling, + ascending sequence order, and ``changelog-producer=none``. Custom + ``sequence.field``, floating-point keys, PK global indexes, lookup without DVs, + and read-optimized union scans are unsupported. +* Append deduplicate requires a single non-null scalar key, a persisted offset + column, exact offset file-index reads, and non-bitmap64 DVs. Existing ordinary + append files are not migrated automatically. +* Spill covers Arrow data, not key indexes or DVs, and provides neither a hard + memory limit nor crash recovery. + +Code Map +-------- + +* ``include/paimon/realtime/``: public context, store, progress, and recovery APIs. +* ``src/paimon/core/realtime/``: mode-specific writers, default Arrow store, + deduplicate wrapper/indexes, schema layout, and memory-read pipeline. +* ``src/paimon/core/table/source/realtime_table_scan.cpp``: memory pinning, + snapshot-progress validation, and partition-bucket split grouping. +* ``src/paimon/core/table/source/append_only_table_read.cpp``: append union reads + and per-file pending offset-DV predicates. +* ``src/paimon/core/operation/merge_file_split_read.cpp``: PK disk/memory merge + and level-aware predicate pushdown. +* ``src/paimon/core/operation/commit/realtime_*``: durable progress, logical-DV + materialization, commit retry, and compaction DV remapping. +* ``test/inte/realtime_write_inte_test.cpp``: lifecycle and mode-specific + integration coverage; store/reader unit tests live next to their implementation. diff --git a/include/paimon/file_index/file_indexer.h b/include/paimon/file_index/file_indexer.h index 2f2fdd9a2..0d0c84f9e 100644 --- a/include/paimon/file_index/file_indexer.h +++ b/include/paimon/file_index/file_indexer.h @@ -23,11 +23,13 @@ #include #include +#include "paimon/defs.h" #include "paimon/file_index/file_index_reader.h" #include "paimon/file_index/file_index_result.h" #include "paimon/file_index/file_index_writer.h" #include "paimon/fs/file_system.h" #include "paimon/memory/memory_pool.h" +#include "paimon/predicate/function.h" #include "paimon/visibility.h" struct ArrowSchema; @@ -38,6 +40,13 @@ class PAIMON_EXPORT FileIndexer { public: virtual ~FileIndexer() = default; + /// Whether this indexer returns exact file-local matching row positions for the field type. + /// Supported evaluations return BitmapIndexResult (or Skip), never Remain or false positives. + /// The schema and values must satisfy the index's writer constraints. + virtual bool SupportsExactPredicate(Function::Type, FieldType) const { + return false; + } + /// Extra columns required while building this index, in addition to the indexed column. virtual Result>> GetExtraFieldNames() const { return std::optional>(std::nullopt); diff --git a/include/paimon/realtime/realtime_store.h b/include/paimon/realtime/realtime_store.h index f31c5c97d..db46f6bd9 100644 --- a/include/paimon/realtime/realtime_store.h +++ b/include/paimon/realtime/realtime_store.h @@ -95,6 +95,7 @@ struct PAIMON_EXPORT RealtimeWriteBatch { /// Row counts are physical stored rows. For primary-key stores they include old versions and /// delete records, rather than the rows visible after merge-on-read. A segment removed by /// `AdvanceCommittedOffset` is no longer included, even if an older read view still pins it. +/// Deduplicate stores also include retained in-memory key-index buffers and offset bitmaps. struct PAIMON_EXPORT RealtimeStoreDataUsage { uint64_t building_memory_bytes = 0; uint64_t sealed_memory_bytes = 0; @@ -211,6 +212,28 @@ class PAIMON_EXPORT RealtimeStore { virtual uint64_t GetMemoryUsage() const = 0; }; +/// Optional capability for `RealtimeStore` implementations to separate segment freezing from spill +/// I/O. +class PAIMON_EXPORT RealtimeStoreTwoPhaseSeal { + public: + virtual ~RealtimeStoreTwoPhaseSeal() = default; + + /// Freezes the building segment without spill I/O. + virtual Result>> + FreezeBuildingSegment() = 0; + + /// Spills a segment returned by FreezeBuildingSegment and returns its commit handle. + virtual Result> SpillFrozenSegment( + const std::shared_ptr& segment) = 0; + + /// Advances the committed offset without destroying retired storage. The returned view owns + /// exactly the segments removed in this operation, including concurrent spill replacements. + /// The caller releases it after unlocking its write mutex, so cleanup never runs under that + /// mutex. This operation must not perform I/O. + virtual Result> RetireCommittedSegments( + int64_t committed_end_offset) = 0; +}; + /// Factory for application-provided `RealtimeStore` implementations. class PAIMON_EXPORT RealtimeStoreFactory { public: diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index ec0c6ea8c..7c9799776 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -423,6 +423,7 @@ set(PAIMON_CORE_SRCS core/realtime/data_file_key_offset_index_writer.cpp core/realtime/key_offset_index_utils.cpp core/realtime/realtime_key_offset_index.cpp + core/realtime/realtime_file_schema.cpp core/realtime/realtime_offset_file_index_lookup.cpp core/realtime/realtime_deduplicate_state.cpp core/realtime/realtime_deduplicate_writer.cpp @@ -907,6 +908,7 @@ if(PAIMON_BUILD_TESTS) core/realtime/key_offset_index_utils_test.cpp core/realtime/primary_key_realtime_store_test.cpp core/realtime/realtime_deduplicate_state_test.cpp + core/realtime/realtime_file_schema_test.cpp core/realtime/realtime_key_offset_index_test.cpp core/realtime/realtime_offset_batch_reader_test.cpp core/realtime/realtime_offset_file_index_lookup_test.cpp diff --git a/src/paimon/common/file_index/bitmap/bitmap_file_index.cpp b/src/paimon/common/file_index/bitmap/bitmap_file_index.cpp index f883224b9..2316122e0 100644 --- a/src/paimon/common/file_index/bitmap/bitmap_file_index.cpp +++ b/src/paimon/common/file_index/bitmap/bitmap_file_index.cpp @@ -46,6 +46,37 @@ class MemoryPool; BitmapFileIndex::BitmapFileIndex(const std::map& options) : options_(options) {} +bool BitmapFileIndex::SupportsExactPredicate(Function::Type function, FieldType field_type) const { + switch (field_type) { + case FieldType::BOOLEAN: + case FieldType::TINYINT: + case FieldType::SMALLINT: + case FieldType::INT: + case FieldType::BIGINT: + case FieldType::FLOAT: + case FieldType::DOUBLE: + case FieldType::STRING: + case FieldType::DATE: + break; + case FieldType::TIMESTAMP: + // Nanosecond timestamps are truncated to microseconds. + return function == Function::Type::IS_NULL || function == Function::Type::IS_NOT_NULL; + default: + return false; + } + switch (function) { + case Function::Type::EQUAL: + case Function::Type::NOT_EQUAL: + case Function::Type::IN: + case Function::Type::NOT_IN: + case Function::Type::IS_NULL: + case Function::Type::IS_NOT_NULL: + return true; + default: + return false; + } +} + Result BitmapFileIndex::ConvertLiteral( const Literal& literal, const std::shared_ptr& arrow_type) { if (literal.GetType() != FieldType::TIMESTAMP) { diff --git a/src/paimon/common/file_index/bitmap/bitmap_file_index.h b/src/paimon/common/file_index/bitmap/bitmap_file_index.h index 9f8272f52..c7e2204b3 100644 --- a/src/paimon/common/file_index/bitmap/bitmap_file_index.h +++ b/src/paimon/common/file_index/bitmap/bitmap_file_index.h @@ -45,6 +45,8 @@ class PAIMON_EXPORT BitmapFileIndex : public FileIndexer { explicit BitmapFileIndex(const std::map& options); ~BitmapFileIndex() override = default; + bool SupportsExactPredicate(Function::Type function, FieldType field_type) const override; + Result> CreateReader( ::ArrowSchema* arrow_schema, int64_t start, int64_t length, const std::shared_ptr& input_stream, diff --git a/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index.cpp b/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index.cpp index 37266e494..508a4b200 100644 --- a/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index.cpp +++ b/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index.cpp @@ -65,6 +65,39 @@ class MemoryPool; BitSliceIndexBitmapFileIndex::BitSliceIndexBitmapFileIndex( const std::map& options) {} +bool BitSliceIndexBitmapFileIndex::SupportsExactPredicate(Function::Type function, + FieldType field_type) const { + switch (field_type) { + case FieldType::TINYINT: + case FieldType::SMALLINT: + case FieldType::INT: + case FieldType::BIGINT: + case FieldType::DATE: + case FieldType::DECIMAL: + break; + case FieldType::TIMESTAMP: + // Nanosecond timestamps are truncated to microseconds. + return function == Function::Type::IS_NULL || function == Function::Type::IS_NOT_NULL; + default: + return false; + } + switch (function) { + case Function::Type::EQUAL: + case Function::Type::NOT_EQUAL: + case Function::Type::IN: + case Function::Type::NOT_IN: + case Function::Type::IS_NULL: + case Function::Type::IS_NOT_NULL: + case Function::Type::GREATER_THAN: + case Function::Type::GREATER_OR_EQUAL: + case Function::Type::LESS_THAN: + case Function::Type::LESS_OR_EQUAL: + return true; + default: + return false; + } +} + Result> BitSliceIndexBitmapFileIndex::CreateWriter( ::ArrowSchema* c_arrow_schema, const std::shared_ptr& pool) const { PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr arrow_schema, diff --git a/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index.h b/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index.h index 69f1b65a2..9ae60dcf4 100644 --- a/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index.h +++ b/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index.h @@ -48,6 +48,8 @@ class BitSliceIndexBitmapFileIndex : public FileIndexer { explicit BitSliceIndexBitmapFileIndex(const std::map& options); ~BitSliceIndexBitmapFileIndex() override = default; + bool SupportsExactPredicate(Function::Type function, FieldType field_type) const override; + Result> CreateReader( ::ArrowSchema* arrow_schema, int64_t start, int64_t length, const std::shared_ptr& input_stream, diff --git a/src/paimon/common/file_index/rangebitmap/range_bitmap_file_index.cpp b/src/paimon/common/file_index/rangebitmap/range_bitmap_file_index.cpp index 862271bad..b9c7c240b 100644 --- a/src/paimon/common/file_index/rangebitmap/range_bitmap_file_index.cpp +++ b/src/paimon/common/file_index/rangebitmap/range_bitmap_file_index.cpp @@ -40,6 +40,41 @@ namespace paimon { RangeBitmapFileIndex::RangeBitmapFileIndex(const std::map& options) : options_(options) {} +bool RangeBitmapFileIndex::SupportsExactPredicate(Function::Type function, + FieldType field_type) const { + switch (field_type) { + case FieldType::BOOLEAN: + case FieldType::TINYINT: + case FieldType::SMALLINT: + case FieldType::INT: + case FieldType::BIGINT: + case FieldType::FLOAT: + case FieldType::DOUBLE: + case FieldType::STRING: + case FieldType::DATE: + case FieldType::TIMESTAMP: + case FieldType::DECIMAL: + break; + default: + return false; + } + switch (function) { + case Function::Type::EQUAL: + case Function::Type::NOT_EQUAL: + case Function::Type::IN: + case Function::Type::NOT_IN: + case Function::Type::IS_NULL: + case Function::Type::IS_NOT_NULL: + case Function::Type::GREATER_THAN: + case Function::Type::GREATER_OR_EQUAL: + case Function::Type::LESS_THAN: + case Function::Type::LESS_OR_EQUAL: + return true; + default: + return false; + } +} + Result> RangeBitmapFileIndex::CreateReader( ::ArrowSchema* arrow_schema, const int64_t start, const int64_t length, const std::shared_ptr& input_stream, diff --git a/src/paimon/common/file_index/rangebitmap/range_bitmap_file_index.h b/src/paimon/common/file_index/rangebitmap/range_bitmap_file_index.h index 36b2bd675..62b04f3cb 100644 --- a/src/paimon/common/file_index/rangebitmap/range_bitmap_file_index.h +++ b/src/paimon/common/file_index/rangebitmap/range_bitmap_file_index.h @@ -44,6 +44,8 @@ class PAIMON_EXPORT RangeBitmapFileIndex final : public FileIndexer { ~RangeBitmapFileIndex() override = default; + bool SupportsExactPredicate(Function::Type function, FieldType field_type) const override; + Result> CreateReader( ::ArrowSchema* arrow_schema, int64_t start, int64_t length, const std::shared_ptr& input_stream, diff --git a/src/paimon/common/reader/prefetch_file_batch_reader_impl_test.cpp b/src/paimon/common/reader/prefetch_file_batch_reader_impl_test.cpp index 7d0879d2a..54ed531ec 100644 --- a/src/paimon/common/reader/prefetch_file_batch_reader_impl_test.cpp +++ b/src/paimon/common/reader/prefetch_file_batch_reader_impl_test.cpp @@ -19,10 +19,7 @@ #include "paimon/common/reader/prefetch_file_batch_reader_impl.h" #include -#include -#include #include -#include #include #include "arrow/compute/api.h" @@ -44,6 +41,7 @@ #include "paimon/testing/mock/mock_file_batch_reader.h" #include "paimon/testing/mock/mock_file_system.h" #include "paimon/testing/mock/mock_format_reader_builder.h" +#include "paimon/testing/utils/blocking_file_system.h" #include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/testharness.h" @@ -133,37 +131,6 @@ class FailingFileSystem : public MockFileSystem { } }; -/// Blocks every Open() until `opens_needed` of them are in flight at once, so a caller that -/// opens its streams one at a time never reaches the barrier and fails instead of passing -/// silently. The wait is bounded so that failure is reported rather than hanging the test. -class BarrierFileSystem : public MockFileSystem { - public: - explicit BarrierFileSystem(uint32_t opens_needed) : opens_needed_(opens_needed) {} - - Result> Open(const std::string& path) const override { - const uint32_t arrived = ++inflight_opens_; - std::unique_lock lock(mutex_); - if (!condition_.wait_for(lock, kBarrierTimeout, - [this] { return inflight_opens_ >= opens_needed_; })) { - return Status::IOError(fmt::format( - "only {} of {} opens were in flight at once, so they are not concurrent", arrived, - opens_needed_)); - } - condition_.notify_all(); - return std::make_unique(); - } - - private: - // Generous so a loaded machine does not turn this into a flaky test; a regression makes the - // opens serial, which no wait ever satisfies. - static constexpr std::chrono::seconds kBarrierTimeout{60}; - - const uint32_t opens_needed_; - mutable std::atomic inflight_opens_{0}; - mutable std::mutex mutex_; - mutable std::condition_variable condition_; -}; - class IoReadingMockFileBatchReader : public MockFileBatchReader { public: IoReadingMockFileBatchReader(const std::shared_ptr& data, @@ -1421,7 +1388,10 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestCreateOpensAllStreamsConcurrently) { const int32_t prefetch_max_parallel_num = 3; // One stream for the cache plus one per reader. const uint32_t opens_needed = prefetch_max_parallel_num + 1; - auto barrier_fs = std::make_shared(opens_needed); + auto barrier_fs = std::make_shared(std::make_shared()); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr barrier, + barrier_fs->Barrier(BlockingFileSystem::Operation::OPEN, /*path=*/"", opens_needed)); MockFormatReaderBuilder reader_builder(data_array, data_type_, batch_size); // Unlike executor_ above, this one has a thread per open: with fewer threads than opens the // barrier could not be reached however concurrent the dispatch is. @@ -1436,6 +1406,7 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestCreateOpensAllStreamsConcurrently) { /*initialize_read_ranges=*/true, /*read_ahead_cache_enabled=*/true, CacheConfig(), /*enable_io_metrics=*/false, WarmupLevel::DECODED, pool_, GetArrowPool(pool_))); ASSERT_NE(reader, nullptr); + ASSERT_TRUE(barrier->WaitForArrival()); } } // namespace paimon::test diff --git a/src/paimon/core/operation/abstract_split_read.cpp b/src/paimon/core/operation/abstract_split_read.cpp index c5298e2c6..b087e20b8 100644 --- a/src/paimon/core/operation/abstract_split_read.cpp +++ b/src/paimon/core/operation/abstract_split_read.cpp @@ -102,11 +102,27 @@ Result>> AbstractSplitRead::CreateR DeletionVector::Factory dv_factory, const std::optional>& row_ranges, const std::shared_ptr& data_file_path_factory, const std::map& extra_format_options) const { - PAIMON_ASSIGN_OR_RAISE( - std::vector readers_with_meta, - CreateRawFileReadersWithMeta(partition, data_files, read_schema, predicate, dv_factory, - row_ranges, data_file_path_factory, extra_format_options, - /*attach_blob_context=*/false)); + PredicateProvider predicate_provider = + [predicate](const std::shared_ptr&) -> Result> { + return predicate; + }; + return CreateRawFileReadersWithPredicateProvider( + partition, data_files, read_schema, std::move(predicate_provider), dv_factory, row_ranges, + data_file_path_factory, extra_format_options); +} + +Result>> +AbstractSplitRead::CreateRawFileReadersWithPredicateProvider( + const BinaryRow& partition, const std::vector>& data_files, + const std::shared_ptr& read_schema, PredicateProvider predicate_provider, + DeletionVector::Factory dv_factory, const std::optional>& row_ranges, + const std::shared_ptr& data_file_path_factory, + const std::map& extra_format_options) const { + PAIMON_ASSIGN_OR_RAISE(std::vector readers_with_meta, + CreateRawFileReadersWithMetaImpl( + partition, data_files, read_schema, std::move(predicate_provider), + dv_factory, row_ranges, data_file_path_factory, extra_format_options, + /*attach_blob_context=*/false)); std::vector> raw_file_readers; raw_file_readers.reserve(readers_with_meta.size()); for (auto& reader_with_meta : readers_with_meta) { @@ -123,12 +139,40 @@ AbstractSplitRead::CreateRawFileReadersWithMeta( const std::shared_ptr& data_file_path_factory, const std::map& extra_format_options, bool attach_blob_context) const { + PredicateProvider predicate_provider = + [predicate](const std::shared_ptr&) -> Result> { + return predicate; + }; + return CreateRawFileReadersWithMetaImpl( + partition, data_files, read_schema, std::move(predicate_provider), dv_factory, row_ranges, + data_file_path_factory, extra_format_options, attach_blob_context); +} + +Result> +AbstractSplitRead::CreateRawFileReadersWithMetaImpl( + const BinaryRow& partition, const std::vector>& data_files, + const std::shared_ptr& read_schema, PredicateProvider predicate_provider, + DeletionVector::Factory dv_factory, const std::optional>& row_ranges, + const std::shared_ptr& data_file_path_factory, + const std::map& extra_format_options, + bool attach_blob_context) const { if (data_files.empty()) { return std::vector(); } - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr field_mapping_builder, - FieldMappingBuilder::Create(read_schema, context_->GetPartitionKeys(), predicate)); + if (!predicate_provider) { + return Status::Invalid("file predicate provider is null"); + } + + auto create_reader = + [&](const std::shared_ptr& file) -> Result { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr predicate, predicate_provider(file)); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr field_mapping_builder, + FieldMappingBuilder::Create(read_schema, context_->GetPartitionKeys(), predicate)); + return CreateRawFileReader(partition, file, field_mapping_builder.get(), dv_factory, + row_ranges, data_file_path_factory, extra_format_options, + attach_blob_context); + }; std::vector raw_file_readers; raw_file_readers.reserve(data_files.size()); @@ -136,11 +180,7 @@ AbstractSplitRead::CreateRawFileReadersWithMeta( static_cast(std::min(kReaderBuildMaxParallelNum, data_files.size())); if (parallel_num <= 1) { for (const auto& file : data_files) { - PAIMON_ASSIGN_OR_RAISE( - RawFileReaderWithMeta file_reader, - CreateRawFileReader(partition, file, field_mapping_builder.get(), dv_factory, - row_ranges, data_file_path_factory, extra_format_options, - attach_blob_context)); + PAIMON_ASSIGN_OR_RAISE(RawFileReaderWithMeta file_reader, create_reader(file)); if (file_reader.reader) { raw_file_readers.push_back(std::move(file_reader)); } @@ -156,13 +196,7 @@ AbstractSplitRead::CreateRawFileReadersWithMeta( futures.reserve(data_files.size()); for (const auto& file : data_files) { futures.push_back( - Via(build_executor.get(), - [this, file, &partition, &field_mapping_builder, &dv_factory, &row_ranges, - &data_file_path_factory, &extra_format_options, attach_blob_context]() { - return CreateRawFileReader(partition, file, field_mapping_builder.get(), - dv_factory, row_ranges, data_file_path_factory, - extra_format_options, attach_blob_context); - })); + Via(build_executor.get(), [file, &create_reader]() { return create_reader(file); })); } // Readers keep the file order of the split, and CollectAll preserves the submit order. Status first_error; diff --git a/src/paimon/core/operation/abstract_split_read.h b/src/paimon/core/operation/abstract_split_read.h index a71a60594..d5857d716 100644 --- a/src/paimon/core/operation/abstract_split_read.h +++ b/src/paimon/core/operation/abstract_split_read.h @@ -18,6 +18,7 @@ #pragma once +#include #include #include #include @@ -65,6 +66,11 @@ class TableSchema; class AbstractSplitRead : public SplitRead { public: + /// Supplies the complete predicate for a file. The provider must be nonempty, but may return + /// a null predicate when no predicate applies to an individual file. + using PredicateProvider = + std::function>(const std::shared_ptr&)>; + ~AbstractSplitRead() override = default; /// `extra_format_options` are merged over the table options when building the format @@ -91,6 +97,15 @@ class AbstractSplitRead : public SplitRead { const std::shared_ptr& memory_pool, const std::shared_ptr& executor); + /// Creates raw readers using each file's predicate for pushdown. The provider is called once + /// per file and may be called concurrently. + Result>> CreateRawFileReadersWithPredicateProvider( + const BinaryRow& partition, const std::vector>& data_files, + const std::shared_ptr& read_schema, PredicateProvider predicate_provider, + DeletionVector::Factory dv_factory, const std::optional>& row_ranges, + const std::shared_ptr& data_file_path_factory, + const std::map& extra_format_options) const; + /// Creates raw readers while preserving the metadata associated with every reader which was /// not eliminated by a file index or deletion vector. Set `attach_blob_context` only for /// data-evolution blob fallback readers. @@ -131,6 +146,14 @@ class AbstractSplitRead : public SplitRead { const std::optional>& write_cols); private: + Result> CreateRawFileReadersWithMetaImpl( + const BinaryRow& partition, const std::vector>& data_files, + const std::shared_ptr& read_schema, PredicateProvider predicate_provider, + DeletionVector::Factory dv_factory, const std::optional>& row_ranges, + const std::shared_ptr& data_file_path_factory, + const std::map& extra_format_options, + bool attach_blob_context) const; + /// What a data file is opened with, gathered from the context and the table options once so /// that the reader builder and the file reader cannot be built from different answers. DataFileReadOptions BuildDataFileReadOptions() const; diff --git a/src/paimon/core/operation/append_only_file_store_write.cpp b/src/paimon/core/operation/append_only_file_store_write.cpp index 0d6b699ef..666260267 100644 --- a/src/paimon/core/operation/append_only_file_store_write.cpp +++ b/src/paimon/core/operation/append_only_file_store_write.cpp @@ -324,7 +324,7 @@ Result> AppendOnlyFileStoreWrite::CreateWriter( return RealtimeDeduplicateWriter::Create( partition_map, bucket, realtime_context_, writer, realtime_schema_layout_, table_schema_->Id(), deduplicate_key_fields, restore_data_files, data_file_path_factory, - options_, io_manager_ ? io_manager_->GetTempDir() : "", pool_); + options_, io_manager_ ? io_manager_->GetTempDir() : "", schema_manager_, pool_); } return RealtimeAppendOnlyWriter::Create(partition_map, bucket, realtime_context_, writer, realtime_schema_layout_, options_, diff --git a/src/paimon/core/operation/commit/realtime_deduplicate_compaction_changes_provider.cpp b/src/paimon/core/operation/commit/realtime_deduplicate_compaction_changes_provider.cpp index 038761dbd..da115ca2b 100644 --- a/src/paimon/core/operation/commit/realtime_deduplicate_compaction_changes_provider.cpp +++ b/src/paimon/core/operation/commit/realtime_deduplicate_compaction_changes_provider.cpp @@ -79,10 +79,10 @@ Status CollectDeletedOffsets(const ManifestEntry& source, const std::shared_ptr< [](const std::string&) -> Result> { return std::shared_ptr(); }; - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr reader, - read->CreateReader(source.Partition(), source.Bucket(), {source.File()}, no_deletions, - ranges, offset_schema, /*predicate=*/nullptr)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, + read->CreateReader(source.Partition(), source.Bucket(), {source.File()}, + no_deletions, ranges, offset_schema, + /*predicate=*/nullptr)); ScopeGuard reader_guard([&]() { reader->Close(); }); const std::shared_ptr arrow_pool = GetArrowPool(pool); int64_t emitted_rows = 0; diff --git a/src/paimon/core/operation/commit/realtime_deduplicate_dv_materializer.cpp b/src/paimon/core/operation/commit/realtime_deduplicate_dv_materializer.cpp index 8d2e61523..a64e9354a 100644 --- a/src/paimon/core/operation/commit/realtime_deduplicate_dv_materializer.cpp +++ b/src/paimon/core/operation/commit/realtime_deduplicate_dv_materializer.cpp @@ -26,6 +26,7 @@ #include "paimon/core/deletionvectors/deletion_vectors_index_file.h" #include "paimon/core/operation/commit/commit_scanner.h" #include "paimon/core/realtime/realtime_offset_file_index_lookup.h" +#include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" #include "paimon/core/utils/file_store_path_factory.h" #include "paimon/fs/file_system.h" @@ -40,6 +41,8 @@ RealtimeDeduplicateDvMaterializer::RealtimeDeduplicateDvMaterializer( path_factory_(std::move(path_factory)), scanner_(std::move(scanner)), options_(options), + schema_manager_(std::make_shared( + options.GetFileSystem(), path_factory_->RootPath(), options.GetBranch())), pool_(std::move(pool)) {} Result @@ -107,7 +110,8 @@ RealtimeDeduplicateDvMaterializer::LookupFilePositions(const BinaryRow& partitio std::shared_ptr lookup, RealtimeOffsetFileIndexLookup::Create( schema, table_schema_->Id(), schema->GetFieldByName(key_fields.front()), - update.lookup_files, paths, options_.GetFileSystem(), pool_, options_)); + update.lookup_files, paths, options_.GetFileSystem(), options_, schema_manager_, + pool_)); PAIMON_ASSIGN_OR_RAISE(positions, lookup->LookupFilePositions(update.deleted_offsets)); } diff --git a/src/paimon/core/operation/commit/realtime_deduplicate_dv_materializer.h b/src/paimon/core/operation/commit/realtime_deduplicate_dv_materializer.h index bbd1d90af..23d07ac40 100644 --- a/src/paimon/core/operation/commit/realtime_deduplicate_dv_materializer.h +++ b/src/paimon/core/operation/commit/realtime_deduplicate_dv_materializer.h @@ -40,6 +40,7 @@ class DeletionVector; class DeletionVectorsIndexFile; class FileStorePathFactory; class MemoryPool; +class SchemaManager; class TableSchema; /// Shared, attempt-local DV processing for realtime deduplicate writes and compaction. @@ -95,6 +96,7 @@ class RealtimeDeduplicateDvMaterializer { std::shared_ptr path_factory_; std::shared_ptr scanner_; CoreOptions options_; + std::shared_ptr schema_manager_; std::shared_ptr pool_; }; diff --git a/src/paimon/core/operation/file_store_commit_impl.cpp b/src/paimon/core/operation/file_store_commit_impl.cpp index 8239c8784..ee8adfee8 100644 --- a/src/paimon/core/operation/file_store_commit_impl.cpp +++ b/src/paimon/core/operation/file_store_commit_impl.cpp @@ -74,6 +74,7 @@ #include "paimon/core/operation/metrics/commit_metrics.h" #include "paimon/core/operation/metrics/commit_stats.h" #include "paimon/core/partition/partition_statistics.h" +#include "paimon/core/realtime/realtime_utils.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" #include "paimon/core/table/bucket_mode.h" @@ -984,6 +985,7 @@ Result FileStoreCommitImpl::CommitWithProgress( if (!options_.RealtimeEnabled()) { return Status::Invalid("CommitWithProgress requires realtime.enabled=true"); } + PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateBranch(options_.GetBranch())); if (realtime_commits.empty()) { return Status::Invalid("real-time commits must not be empty"); } diff --git a/src/paimon/core/operation/raw_file_split_read.cpp b/src/paimon/core/operation/raw_file_split_read.cpp index 58a19fc6b..d81bc4850 100644 --- a/src/paimon/core/operation/raw_file_split_read.cpp +++ b/src/paimon/core/operation/raw_file_split_read.cpp @@ -148,6 +148,21 @@ Result> RawFileSplitRead::CreateReader( DeletionVector::Factory dv_factory, const std::optional>& local_row_ranges, const std::shared_ptr& read_schema, const std::shared_ptr& predicate) { + PredicateProvider predicate_provider = + [predicate](const std::shared_ptr&) -> Result> { + return predicate; + }; + return CreateReaderWithPredicateProvider(partition, bucket, data_files, std::move(dv_factory), + local_row_ranges, read_schema, predicate, + std::move(predicate_provider)); +} + +Result> RawFileSplitRead::CreateReaderWithPredicateProvider( + const BinaryRow& partition, int32_t bucket, + const std::vector>& data_files, + DeletionVector::Factory dv_factory, const std::optional>& local_row_ranges, + const std::shared_ptr& read_schema, + const std::shared_ptr& residual_predicate, PredicateProvider predicate_provider) { PAIMON_RETURN_NOT_OK(ValidateFileLocalRowRanges(data_files, local_row_ranges)); if (!read_schema) { return Status::Invalid("raw file read schema is null"); @@ -157,16 +172,17 @@ Result> RawFileSplitRead::CreateReader( PAIMON_ASSIGN_OR_RAISE( std::vector> raw_file_readers, - CreateRawFileReaders(partition, data_files, read_schema, predicate, dv_factory, - local_row_ranges, data_file_path_factory, - /*extra_format_options=*/{})); + CreateRawFileReadersWithPredicateProvider( + partition, data_files, read_schema, std::move(predicate_provider), dv_factory, + local_row_ranges, data_file_path_factory, /*extra_format_options=*/{})); auto raw_readers = ObjectUtils::MoveVector>(std::move(raw_file_readers)); auto concat_batch_reader = std::make_unique(std::move(raw_readers), arrow_pool_); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, - ApplyPredicateFilterIfNeeded(std::move(concat_batch_reader), predicate)); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr batch_reader, + ApplyPredicateFilterIfNeeded(std::move(concat_batch_reader), residual_predicate)); return std::make_unique(std::move(batch_reader), arrow_pool_); } diff --git a/src/paimon/core/operation/raw_file_split_read.h b/src/paimon/core/operation/raw_file_split_read.h index 6e9007688..14df8cd66 100644 --- a/src/paimon/core/operation/raw_file_split_read.h +++ b/src/paimon/core/operation/raw_file_split_read.h @@ -85,6 +85,7 @@ class RawFileSplitRead : public AbstractSplitRead { const std::vector>& files, DeletionVector::Factory dv_factory, const std::optional>& local_row_ranges); + /// Uses the supplied predicate for pushdown and optional residual filtering. Result> CreateReader( const BinaryRow& partition, int32_t bucket, const std::vector>& files, DeletionVector::Factory dv_factory, @@ -92,6 +93,15 @@ class RawFileSplitRead : public AbstractSplitRead { const std::shared_ptr& read_schema, const std::shared_ptr& predicate); + /// Uses file-specific predicates for pushdown and the supplied residual predicate for + /// optional filtering after concatenation. + Result> CreateReaderWithPredicateProvider( + const BinaryRow& partition, int32_t bucket, + const std::vector>& files, DeletionVector::Factory dv_factory, + const std::optional>& local_row_ranges, + const std::shared_ptr& read_schema, + const std::shared_ptr& residual_predicate, PredicateProvider predicate_provider); + Result Match(const std::shared_ptr& split, bool force_keep_delete) const override; Result> ApplyIndexAndDvReaderIfNeeded( std::unique_ptr&& file_reader, const std::shared_ptr& file, diff --git a/src/paimon/core/operation/raw_file_split_read_test.cpp b/src/paimon/core/operation/raw_file_split_read_test.cpp index 77f92109c..04b5462b8 100644 --- a/src/paimon/core/operation/raw_file_split_read_test.cpp +++ b/src/paimon/core/operation/raw_file_split_read_test.cpp @@ -285,6 +285,89 @@ TEST_F(RawFileSplitReadTest, TestCreateReader) { CheckReadResult(read_schema, expected_array); } +TEST_F(RawFileSplitReadTest, TestCreateReaderUsesExplicitPredicate) { + const std::string path = + GetDataDir() + "/orc/multi_partition_append_table.db/multi_partition_append_table"; + ReadContextBuilder builder(path); + builder.EnablePredicateFilter(true) + .SetPredicate(PredicateBuilder::Equal(0, "f0", FieldType::STRING, + Literal(FieldType::STRING, "Bob", 3))) + .AddOption(Options::FILE_INDEX_READ_ENABLED, "false"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, builder.Finish()); + SchemaManager schemas(std::make_shared(), path); + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, schemas.ReadSchema(0)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr context, + InternalReadContext::Create(read_context, table_schema, table_schema->Options())); + const CoreOptions& options = context->GetCoreOptions(); + const auto schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); + ASSERT_OK_AND_ASSIGN(std::vector external_paths, options.CreateExternalPaths()); + ASSERT_OK_AND_ASSIGN(std::optional global_index_path, + options.CreateGlobalIndexExternalPath()); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr paths, + FileStorePathFactory::Create( + path, schema, table_schema->PartitionKeys(), options.GetPartitionDefaultName(), + options.GetFileFormat()->Identifier(), options.DataFilePrefix(), + options.LegacyPartitionNameEnabled(), external_paths, global_index_path, + options.IndexFileInDataFileDir(), pool_)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr executor, CreateDefaultExecutor(2)); + RawFileSplitRead read(paths, context, pool_, executor); + auto split = std::dynamic_pointer_cast(PrepareDataSplits().front()); + ASSERT_NE(nullptr, split); + const auto first = split->DataFiles().front(); + const auto second = std::make_shared(*first); + // The supplied predicate, not the context's predicate, must be used for both files. + const std::shared_ptr predicate = + PredicateBuilder::Equal(0, "f0", FieldType::STRING, Literal(FieldType::STRING, "Tony", 4)); + RoaringBitmap32 deleted; + deleted.Add(1); + std::shared_ptr dv = std::make_shared(deleted); + DeletionVector::Factory dv_factory = + [dv](const std::string&) -> Result> { return dv; }; + RawFileSplitRead::PredicateProvider no_pushdown = + [](const std::shared_ptr&) -> Result> { + return std::shared_ptr(); + }; + ASSERT_NOK_WITH_MSG( + read.CreateReaderWithPredicateProvider(split->Partition(), split->Bucket(), {first, second}, + dv_factory, std::nullopt, context->GetReadSchema(), + predicate, /*predicate_provider=*/{}), + "file predicate provider is null"); + ASSERT_NOK_WITH_MSG( + read.CreateReaderWithPredicateProvider(split->Partition(), split->Bucket(), {first, second}, + dv_factory, std::vector{Range(0, 0)}, + context->GetReadSchema(), predicate, no_pushdown), + "file-local row ranges require exactly one data file"); + arrow::FieldVector output_fields = {arrow::field("_VALUE_KIND", arrow::int8())}; + output_fields.insert(output_fields.end(), schema->fields().begin(), schema->fields().end()); + auto expected = arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(output_fields), R"([ + [0, "Tony", 10, 0, 14.1], + [0, "Tony", 10, 0, 14.1] + ])") + .ValueOrDie(); + for (bool use_provider : {false, true}) { + SCOPED_TRACE(use_provider); + std::unique_ptr reader; + if (use_provider) { + // A provider without pushdown must not suppress the explicit residual predicate. + ASSERT_OK_AND_ASSIGN( + reader, read.CreateReaderWithPredicateProvider( + split->Partition(), split->Bucket(), {first, second}, dv_factory, + std::nullopt, context->GetReadSchema(), predicate, no_pushdown)); + } else { + ASSERT_OK_AND_ASSIGN( + reader, + read.CreateReader(split->Partition(), split->Bucket(), {first, second}, dv_factory, + std::nullopt, context->GetReadSchema(), predicate)); + } + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, + ReadResultCollector::CollectResult(std::move(reader))); + ASSERT_TRUE(result->Equals(std::make_shared(expected))) + << result->ToString(); + } +} + // recall all columns with reverse sequence in data TEST_F(RawFileSplitReadTest, TestCreateReaderWithReserveSequence) { std::vector read_fields = {DataField(3, arrow::field("f3", arrow::float64())), diff --git a/src/paimon/core/realtime/arrow_deduplicate_realtime_store.cpp b/src/paimon/core/realtime/arrow_deduplicate_realtime_store.cpp index a19368157..ae2d34095 100644 --- a/src/paimon/core/realtime/arrow_deduplicate_realtime_store.cpp +++ b/src/paimon/core/realtime/arrow_deduplicate_realtime_store.cpp @@ -19,6 +19,7 @@ #include "paimon/core/realtime/arrow_deduplicate_realtime_store.h" #include +#include #include #include "arrow/api.h" @@ -26,6 +27,7 @@ #include "arrow/c/helpers.h" #include "arrow/compute/api.h" #include "paimon/common/metrics/metrics_impl.h" +#include "paimon/common/reader/reader_utils.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/common/utils/arrow/status_utils.h" @@ -65,13 +67,9 @@ Result ExportBatch(const std::shared_ptr> FilterDeletedOffsets( - const std::shared_ptr& source, - const std::shared_ptr& deleted_offsets, - const std::shared_ptr& pool) { - if (!deleted_offsets || deleted_offsets->IsEmpty() || source->length() == 0) { - return source; - } +Result FilterDeletedOffsets( + const std::shared_ptr& source, const RoaringBitmap32& input_bitmap, + const std::shared_ptr& deleted_offsets) { std::shared_ptr offset_array = source->GetFieldByName(SpecialFields::RealtimeOffset().Name()); if (!offset_array || offset_array->type_id() != arrow::Type::INT64 || @@ -81,21 +79,27 @@ Result> FilterDeletedOffsets( } std::shared_ptr offsets = checked_pointer_cast(offset_array); - arrow::BooleanBuilder selection(pool.get()); - PAIMON_RETURN_NOT_OK_FROM_ARROW(selection.Reserve(source->length())); - // TODO(xinyu.lxy): Merge-scan the ordered offsets with a RoaringBitmap64 iterator instead of - // calling Contains for every row. - for (int64_t row = 0; row < source->length(); ++row) { - selection.UnsafeAppend(!deleted_offsets->Contains(offsets->Value(row))); - } - std::shared_ptr selection_array; - PAIMON_RETURN_NOT_OK_FROM_ARROW(selection.Finish(&selection_array)); - arrow::compute::ExecContext exec_context(pool.get()); - PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( - arrow::Datum filtered, - arrow::compute::Filter(source, selection_array, arrow::compute::FilterOptions::Defaults(), - &exec_context)); - return checked_pointer_cast(filtered.make_array()); + for (auto row = input_bitmap.Begin(); row != input_bitmap.End(); ++row) { + if (static_cast(*row) >= offsets->length()) { + return Status::Invalid("selected row id is out of bounds for deduplicate batch"); + } + } + if (!deleted_offsets || deleted_offsets->IsEmpty() || input_bitmap.IsEmpty()) { + return input_bitmap; + } + RoaringBitmap32 selection; + auto deleted = deleted_offsets->EqualOrLarger(offsets->Value(*input_bitmap.Begin())); + const auto deleted_end = deleted_offsets->End(); + for (auto row = input_bitmap.Begin(); row != input_bitmap.End(); ++row) { + const int64_t offset = offsets->Value(*row); + if (deleted != deleted_end && *deleted < offset) { + deleted.EqualOrLarger(offset); + } + if (deleted == deleted_end || *deleted != offset) { + selection.Add(*row); + } + } + return selection; } } // namespace @@ -158,6 +162,17 @@ class ArrowDeduplicateRealtimeStore::ReadView final : public RealtimeReadView { std::shared_ptr offset_deletions_; }; +class ArrowDeduplicateRealtimeStore::RetiredReadView final : public RealtimeReadView { + public: + std::optional GetOffsetRange() const override { + return std::nullopt; + } + + std::shared_ptr delegate_view; + std::vector states; + std::shared_ptr committed_lookup; +}; + class ArrowDeduplicateRealtimeStore::OffsetFilteringBatchReader final : public BatchReader { public: OffsetFilteringBatchReader(std::unique_ptr inner, @@ -169,15 +184,19 @@ class ArrowDeduplicateRealtimeStore::OffsetFilteringBatchReader final : public B offset_deletions_(std::move(offset_deletions)), arrow_pool_(std::move(arrow_pool)) {} - // TODO(xinyu.lxy): Override NextBatchWithBitmap to compose the inner selection with offset - // deletions and avoid materializing an intermediate filtered array. Result NextBatch() override { + PAIMON_ASSIGN_OR_RAISE(ReadBatchWithBitmap batch, NextBatchWithBitmap()); + return ReaderUtils::ApplyBitmapToReadBatch(std::move(batch), arrow_pool_); + } + + Result NextBatchWithBitmap() override { while (true) { - PAIMON_ASSIGN_OR_RAISE(ReadBatch batch, inner_->NextBatch()); + PAIMON_ASSIGN_OR_RAISE(ReadBatchWithBitmap batch, inner_->NextBatchWithBitmap()); if (BatchReader::IsEofBatch(batch)) { return batch; } - auto& [c_array, c_schema] = batch; + auto& [read_batch, input_bitmap] = batch; + auto& [c_array, c_schema] = read_batch; PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr imported, arrow::ImportArray(c_array.get(), c_schema.get())); if (!imported || imported->type_id() != arrow::Type::STRUCT) { @@ -185,14 +204,15 @@ class ArrowDeduplicateRealtimeStore::OffsetFilteringBatchReader final : public B } std::shared_ptr source = checked_pointer_cast(imported); - PAIMON_ASSIGN_OR_RAISE(source, - FilterDeletedOffsets(source, offset_deletions_, arrow_pool_)); - if (source->length() == 0) { + PAIMON_ASSIGN_OR_RAISE(RoaringBitmap32 selection, + FilterDeletedOffsets(source, input_bitmap, offset_deletions_)); + if (selection.IsEmpty()) { continue; } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr projected, ProjectByName(source, output_schema_)); - return ExportBatch(projected, arrow_pool_); + PAIMON_ASSIGN_OR_RAISE(ReadBatch output, ExportBatch(projected, arrow_pool_)); + return ReadBatchWithBitmap(std::move(output), std::move(selection)); } } @@ -374,30 +394,52 @@ Status ArrowDeduplicateRealtimeStore::WriteAndLookup( Result>> ArrowDeduplicateRealtimeStore::SealForCommit() { - std::lock_guard lock(mutex_); - if (!building_offset_range_) { - return std::optional>(); - } - // Delegate rotation and the key/deletion state rotation must be atomic. Deduplicate mode - // currently rejects real-time spill, so the delegate only performs its short in-memory - // rotation here. - // TODO(xinyu.lxy): Introduce two-phase store sealing before allowing deduplicate spill, so - // delegate spill I/O can complete without holding this mutex. - PAIMON_ASSIGN_OR_RAISE(std::optional> delegate_segment, - delegate_->SealForCommit()); + return SealCurrentSegment(/*spill=*/true); +} + +Result>> +ArrowDeduplicateRealtimeStore::FreezeBuildingSegment() { + return SealCurrentSegment(/*spill=*/false); +} + +Result>> +ArrowDeduplicateRealtimeStore::SealCurrentSegment(bool spill) { + std::lock_guard seal_lock(seal_mutex_); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr next_building_index, MutableKeyOffsetIndex::Create(key_field_, memory_pool_)); - std::shared_ptr checkpoint = pending_offset_deletions_; - RoaringBitmap64 live_rows = - RoaringBitmap64::AndNot(building_row_offsets_, *pending_offset_deletions_); - auto handle = std::make_shared(delegate_segment.value_or(nullptr), - building_offset_range_.value(), - live_rows.Cardinality(), checkpoint); - sealed_states_.push_back( - SealedState{building_offset_range_.value(), building_key_index_->Snapshot(), checkpoint}); - building_key_index_ = std::move(next_building_index); - building_row_offsets_ = RoaringBitmap64(); - building_offset_range_.reset(); + auto* two_phase = dynamic_cast(delegate_.get()); + std::optional> delegate_segment; + std::optional offset_range; + std::shared_ptr checkpoint; + RoaringBitmap64 row_offsets; + { + std::lock_guard lock(mutex_); + if (!building_offset_range_) { + return std::optional>(); + } + // The fallback preserves compatibility with custom, non-spilling delegates. Context + // creation rejects deduplicate spill when this optional capability is absent. + PAIMON_ASSIGN_OR_RAISE(delegate_segment, two_phase ? two_phase->FreezeBuildingSegment() + : delegate_->SealForCommit()); + offset_range = building_offset_range_; + checkpoint = pending_offset_deletions_; + sealed_states_.push_back( + SealedState{offset_range.value(), building_key_index_->Snapshot(), checkpoint}); + building_key_index_ = std::move(next_building_index); + row_offsets = std::move(building_row_offsets_); + building_row_offsets_ = RoaringBitmap64(); + building_offset_range_.reset(); + } + // The frozen keys remain discoverable while another thread writes the new building segment. + // Both the row-count calculation and all spill I/O use only this immutable checkpoint. + RoaringBitmap64 live_rows = RoaringBitmap64::AndNot(row_offsets, *checkpoint); + if (spill && two_phase && delegate_segment) { + PAIMON_ASSIGN_OR_RAISE(delegate_segment.value(), + two_phase->SpillFrozenSegment(delegate_segment.value())); + } + auto handle = + std::make_shared(delegate_segment.value_or(nullptr), offset_range.value(), + live_rows.Cardinality(), checkpoint); return std::optional>(std::move(handle)); } @@ -492,7 +534,8 @@ Status ArrowDeduplicateRealtimeStore::AdvanceCommittedOffset(int64_t) { "deduplicate committed advancement requires an atomic key-offset lookup replacement"); } -Status ArrowDeduplicateRealtimeStore::AdvanceCommittedOffsetAndLookup( +Result> +ArrowDeduplicateRealtimeStore::RetireCommittedOffsetAndLookup( int64_t committed_end_offset, const std::shared_ptr& committed_lookup) { if (committed_end_offset < 0) { return Status::Invalid("committed real-time offset must not be negative"); @@ -500,24 +543,36 @@ Status ArrowDeduplicateRealtimeStore::AdvanceCommittedOffsetAndLookup( if (!committed_lookup) { return Status::Invalid("committed key-offset lookup must not be null"); } - std::lock_guard lock(mutex_); - PAIMON_RETURN_NOT_OK(delegate_->AdvanceCommittedOffset(committed_end_offset)); - std::shared_ptr applied_checkpoint; - size_t reclaimed_count = 0; - while (reclaimed_count < sealed_states_.size() && - sealed_states_[reclaimed_count].offset_range.end <= committed_end_offset) { - applied_checkpoint = sealed_states_[reclaimed_count].offset_deletions; - ++reclaimed_count; - } - if (reclaimed_count > 0) { - sealed_states_.erase(sealed_states_.begin(), sealed_states_.begin() + reclaimed_count); - } - if (applied_checkpoint) { - PAIMON_RETURN_NOT_OK(EnsureMutableDeletionBitmap()); - *pending_offset_deletions_ -= *applied_checkpoint; + auto retired = std::make_shared(); + { + std::lock_guard lock(mutex_); + auto* two_phase = dynamic_cast(delegate_.get()); + if (two_phase) { + PAIMON_ASSIGN_OR_RAISE(retired->delegate_view, + two_phase->RetireCommittedSegments(committed_end_offset)); + } else { + PAIMON_ASSIGN_OR_RAISE(retired->delegate_view, delegate_->AcquireReadView()); + PAIMON_RETURN_NOT_OK(delegate_->AdvanceCommittedOffset(committed_end_offset)); + } + std::shared_ptr applied_checkpoint; + size_t reclaimed_count = 0; + while (reclaimed_count < sealed_states_.size() && + sealed_states_[reclaimed_count].offset_range.end <= committed_end_offset) { + applied_checkpoint = sealed_states_[reclaimed_count].offset_deletions; + retired->states.push_back(std::move(sealed_states_[reclaimed_count])); + ++reclaimed_count; + } + if (reclaimed_count > 0) { + sealed_states_.erase(sealed_states_.begin(), sealed_states_.begin() + reclaimed_count); + } + if (applied_checkpoint) { + PAIMON_RETURN_NOT_OK(EnsureMutableDeletionBitmap()); + *pending_offset_deletions_ -= *applied_checkpoint; + } + retired->committed_lookup = std::move(committed_key_lookup_); + committed_key_lookup_ = committed_lookup; } - committed_key_lookup_ = committed_lookup; - return Status::OK(); + return std::shared_ptr(std::move(retired)); } Status ArrowDeduplicateRealtimeStore::AttachCommittedKeyOffsetLookup( @@ -532,11 +587,32 @@ Status ArrowDeduplicateRealtimeStore::AttachCommittedKeyOffsetLookup( } RealtimeStoreDataUsage ArrowDeduplicateRealtimeStore::GetDataUsage() const { - return delegate_->GetDataUsage(); + std::lock_guard lock(mutex_); + RealtimeStoreDataUsage usage = delegate_->GetDataUsage(); + usage.building_memory_bytes += building_key_index_->GetMemoryUsage(); + if (!building_row_offsets_.IsEmpty()) { + usage.building_memory_bytes += building_row_offsets_.GetSizeInBytes(); + } + // A checkpoint can be shared by several sealed states and the pending building DV. Attribute + // shared storage to sealed memory first, and count only the distinct pending DV as building. + std::set counted; + for (const SealedState& sealed : sealed_states_) { + usage.sealed_memory_bytes += sealed.key_lookup->GetMemoryUsage(); + if (counted.insert(sealed.offset_deletions.get()).second && + !sealed.offset_deletions->IsEmpty()) { + usage.sealed_memory_bytes += sealed.offset_deletions->GetSizeInBytes(); + } + } + if (counted.insert(pending_offset_deletions_.get()).second && + !pending_offset_deletions_->IsEmpty()) { + usage.building_memory_bytes += pending_offset_deletions_->GetSizeInBytes(); + } + return usage; } uint64_t ArrowDeduplicateRealtimeStore::GetMemoryUsage() const { - return delegate_->GetMemoryUsage(); + const RealtimeStoreDataUsage usage = GetDataUsage(); + return usage.building_memory_bytes + usage.sealed_memory_bytes; } } // namespace paimon diff --git a/src/paimon/core/realtime/arrow_deduplicate_realtime_store.h b/src/paimon/core/realtime/arrow_deduplicate_realtime_store.h index 191b20aba..b070502f7 100644 --- a/src/paimon/core/realtime/arrow_deduplicate_realtime_store.h +++ b/src/paimon/core/realtime/arrow_deduplicate_realtime_store.h @@ -72,6 +72,9 @@ class ArrowDeduplicateRealtimeStore final : public RealtimeStore { Result>> SealForCommit() override; + /// Freezes data, key indexes, and the DV checkpoint for prepare. + Result>> FreezeBuildingSegment(); + Result>> CreateCommitReaders( const std::shared_ptr& segment) override; @@ -85,7 +88,8 @@ class ArrowDeduplicateRealtimeStore final : public RealtimeStore { /// Calling the base notification would create a query/write cutover with mismatched state. Status AdvanceCommittedOffset(int64_t committed_end_offset) override; - Status AdvanceCommittedOffsetAndLookup( + /// Advances committed state and returns retired resources for release by the caller. + Result> RetireCommittedOffsetAndLookup( int64_t committed_end_offset, const std::shared_ptr& committed_lookup); @@ -103,12 +107,13 @@ class ArrowDeduplicateRealtimeStore final : public RealtimeStore { private: struct SealedState { OffsetRange offset_range; - std::shared_ptr key_lookup; + std::shared_ptr key_lookup; std::shared_ptr offset_deletions; }; class SegmentHandle; class ReadView; + class RetiredReadView; class OffsetFilteringBatchReader; ArrowDeduplicateRealtimeStore(const std::shared_ptr& write_schema, @@ -122,6 +127,7 @@ class ArrowDeduplicateRealtimeStore final : public RealtimeStore { Result> ExtractKeys( const std::shared_ptr& data) const; Status EnsureMutableDeletionBitmap(); + Result>> SealCurrentSegment(bool spill); std::shared_ptr write_schema_; int32_t key_field_position_; @@ -130,6 +136,8 @@ class ArrowDeduplicateRealtimeStore final : public RealtimeStore { std::shared_ptr memory_pool_; std::shared_ptr arrow_pool_; + // Serializes seal attempts only; writes never acquire this mutex during spill I/O. + std::mutex seal_mutex_; mutable std::mutex mutex_; std::shared_ptr building_key_index_; std::shared_ptr committed_key_lookup_; diff --git a/src/paimon/core/realtime/arrow_deduplicate_realtime_store_test.cpp b/src/paimon/core/realtime/arrow_deduplicate_realtime_store_test.cpp index e671b6959..f064bce35 100644 --- a/src/paimon/core/realtime/arrow_deduplicate_realtime_store_test.cpp +++ b/src/paimon/core/realtime/arrow_deduplicate_realtime_store_test.cpp @@ -19,9 +19,13 @@ #include "paimon/core/realtime/arrow_deduplicate_realtime_store.h" +#include +#include +#include #include #include #include +#include #include #include "arrow/api.h" @@ -31,9 +35,13 @@ #include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/checked_cast.h" +#include "paimon/common/utils/scope_guard.h" #include "paimon/core/realtime/arrow_realtime_store.h" +#include "paimon/fs/local/local_file_system.h" #include "paimon/macros.h" #include "paimon/memory/memory_pool.h" +#include "paimon/testing/utils/blocking_file_system.h" +#include "paimon/testing/utils/test_helper.h" #include "paimon/testing/utils/testharness.h" namespace paimon::test { @@ -46,6 +54,83 @@ class EmptyKeyOffsetLookup final : public KeyOffsetLookup { } }; +class BitmapSelectingRealtimeStore final : public RealtimeStore { + public: + BitmapSelectingRealtimeStore(std::shared_ptr delegate, RoaringBitmap32 bitmap) + : delegate_(std::move(delegate)), bitmap_(std::move(bitmap)) {} + + Status Write(RealtimeWriteBatch&& batch) override { + return delegate_->Write(std::move(batch)); + } + Result>> SealForCommit() override { + return delegate_->SealForCommit(); + } + Result> AcquireReadView() override { + return delegate_->AcquireReadView(); + } + Result>> CreateCommitReaders( + const std::shared_ptr& segment) override { + PAIMON_ASSIGN_OR_RAISE(std::vector> readers, + delegate_->CreateCommitReaders(segment)); + return Wrap(std::move(readers)); + } + Result>> CreateQueryReaders( + const std::shared_ptr& view, + const RealtimeQueryContext& context) override { + PAIMON_ASSIGN_OR_RAISE(std::vector> readers, + delegate_->CreateQueryReaders(view, context)); + return Wrap(std::move(readers)); + } + Status AdvanceCommittedOffset(int64_t offset) override { + return delegate_->AdvanceCommittedOffset(offset); + } + RealtimeStoreDataUsage GetDataUsage() const override { + return delegate_->GetDataUsage(); + } + uint64_t GetMemoryUsage() const override { + return delegate_->GetMemoryUsage(); + } + + private: + class Reader final : public BatchReader { + public: + Reader(std::unique_ptr inner, RoaringBitmap32 bitmap) + : inner_(std::move(inner)), bitmap_(std::move(bitmap)) {} + + Result NextBatch() override { + return Status::Invalid("bitmap input must not be materialized"); + } + Result NextBatchWithBitmap() override { + PAIMON_ASSIGN_OR_RAISE(ReadBatchWithBitmap batch, inner_->NextBatchWithBitmap()); + if (!IsEofBatch(batch)) { + batch.second = bitmap_; + } + return batch; + } + std::shared_ptr GetReaderMetrics() const override { + return inner_->GetReaderMetrics(); + } + void Close() override { + inner_->Close(); + } + + private: + std::unique_ptr inner_; + RoaringBitmap32 bitmap_; + }; + + std::vector> Wrap( + std::vector>&& readers) const { + for (std::unique_ptr& reader : readers) { + reader = std::make_unique(std::move(reader), bitmap_); + } + return std::move(readers); + } + + std::shared_ptr delegate_; + RoaringBitmap32 bitmap_; +}; + class ArrowDeduplicateRealtimeStoreTest : public testing::Test { public: void SetUp() override { @@ -87,6 +172,33 @@ class ArrowDeduplicateRealtimeStoreTest : public testing::Test { ASSERT_TRUE(segment.has_value()); } + void ResetStoreWithBitmap(const RoaringBitmap32& bitmap) { + delegate_ = std::make_shared( + schema_, RealtimeStoreMode::APPEND_ONLY, StatisticsMode::NONE, + /*temp_directory=*/"", /*spill_file_system=*/nullptr, + /*spill_compression=*/"zstd", /*spill_compression_level=*/1, pool_, arrow_pool_); + auto selecting = std::make_shared(delegate_, bitmap); + ASSERT_OK_AND_ASSIGN(store_, ArrowDeduplicateRealtimeStore::Create( + schema_, {"id"}, selecting, pool_, arrow_pool_)); + ASSERT_OK(store_->AttachCommittedKeyOffsetLookup(empty_lookup_)); + } + + Result>> QueryReaders( + const std::shared_ptr& view, const std::shared_ptr& schema, + int32_t batch_size) const { + auto c_schema = std::make_unique(); + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema, c_schema.get())); + RealtimeQueryContext context{c_schema.get(), /*predicate=*/nullptr, batch_size}; + return store_->CreateQueryReaders(view, context); + } + + Result> ImportBatch(BatchReader::ReadBatch* batch) const { + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::shared_ptr array, + arrow::ImportArray(batch->first.get(), batch->second.get())); + return checked_pointer_cast(array); + } + Result> ReadOffsets(const std::shared_ptr& view) const { auto c_schema = std::make_unique(); PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema_, c_schema.get())); @@ -112,6 +224,7 @@ class ArrowDeduplicateRealtimeStoreTest : public testing::Test { result.push_back(offsets->Value(row)); } } + reader->Close(); } return result; } @@ -139,11 +252,11 @@ TEST_F(ArrowDeduplicateRealtimeStoreTest, TestReclaimsOnlyCompleteSegmentPrefix) ArrowDeduplicateRealtimeStore::OffsetDeletionsOf(initial_view)); ASSERT_EQ("{2}", initial_deletions->ToString()); - ASSERT_OK(store_->AdvanceCommittedOffsetAndLookup(/*committed_end_offset=*/5, empty_lookup_)); + ASSERT_OK(store_->RetireCommittedOffsetAndLookup(/*committed_end_offset=*/5, empty_lookup_)); usage = store_->GetDataUsage(); ASSERT_EQ(2, usage.sealed_row_count); - ASSERT_OK(store_->AdvanceCommittedOffsetAndLookup(/*committed_end_offset=*/10, empty_lookup_)); + ASSERT_OK(store_->RetireCommittedOffsetAndLookup(/*committed_end_offset=*/10, empty_lookup_)); usage = store_->GetDataUsage(); ASSERT_EQ(1, usage.sealed_row_count); ASSERT_OK_AND_ASSIGN(std::shared_ptr retained_view, @@ -153,14 +266,14 @@ TEST_F(ArrowDeduplicateRealtimeStoreTest, TestReclaimsOnlyCompleteSegmentPrefix) ArrowDeduplicateRealtimeStore::OffsetDeletionsOf(retained_view)); ASSERT_EQ("{2}", retained_deletions->ToString()); - ASSERT_OK(store_->AdvanceCommittedOffsetAndLookup(/*committed_end_offset=*/15, empty_lookup_)); + ASSERT_OK(store_->RetireCommittedOffsetAndLookup(/*committed_end_offset=*/15, empty_lookup_)); usage = store_->GetDataUsage(); ASSERT_EQ(1, usage.sealed_row_count); ASSERT_OK_AND_ASSIGN(std::shared_ptr crossing_view, store_->AcquireReadView()); ASSERT_EQ(std::optional(OffsetRange(10, 20)), crossing_view->GetOffsetRange()); - ASSERT_OK(store_->AdvanceCommittedOffsetAndLookup(/*committed_end_offset=*/20, empty_lookup_)); + ASSERT_OK(store_->RetireCommittedOffsetAndLookup(/*committed_end_offset=*/20, empty_lookup_)); usage = store_->GetDataUsage(); ASSERT_EQ(0, usage.sealed_row_count); ASSERT_OK_AND_ASSIGN(std::shared_ptr reclaimed_view, @@ -224,6 +337,477 @@ TEST_F(ArrowDeduplicateRealtimeStoreTest, TestDeleteProducesNoLiveRows) { ASSERT_TRUE(BatchReader::IsEofBatch(batch)); } +TEST_F(ArrowDeduplicateRealtimeStoreTest, TestBitmapFilteringComposesInputSelection) { + struct TestCase { + std::vector deleted_offsets; + std::vector expected_rows; + std::vector expected_offsets; + }; + const std::vector cases = { + {{}, {0, 2, 4}, {10, 12, 14}}, + {{12}, {0, 4}, {10, 14}}, + {{10, 12, 14}, {}, {}}, + {{99}, {0, 2, 4}, {10, 12, 14}}, + }; + const RoaringBitmap32 input = RoaringBitmap32::From({0, 2, 4}); + for (const TestCase& test_case : cases) { + SCOPED_TRACE(RoaringBitmap64::From(test_case.deleted_offsets).ToString()); + ASSERT_NO_FATAL_FAILURE(ResetStoreWithBitmap(input)); + ASSERT_OK(store_->WriteAndLookup( + MakeBatch(R"([[10, 1, "a"], [11, 2, "b"], [12, 3, "c"], + [13, 4, "d"], [14, 5, "e"]])"), + std::vector(5, RecordBatch::RowKind::INSERT), OffsetRange(10, 15), + RoaringBitmap64())); + RoaringBitmap64 deletions = RoaringBitmap64::From(test_case.deleted_offsets); + ASSERT_OK(store_->WriteAndLookup(MakeBatch(R"([[100, 99, "delete"]])"), + {RecordBatch::RowKind::DELETE}, OffsetRange(100, 101), + deletions)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr view, store_->AcquireReadView()); + auto projection = arrow::schema({schema_->field(2), schema_->field(0)}); + ASSERT_OK_AND_ASSIGN(std::vector> readers, + QueryReaders(view, projection, 5)); + ASSERT_EQ(1, readers.size()); + const RoaringBitmap32 expected = RoaringBitmap32::From(test_case.expected_rows); + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatchWithBitmap batch, + readers[0]->NextBatchWithBitmap()); + if (expected.IsEmpty()) { + ASSERT_TRUE(BatchReader::IsEofBatch(batch)); + } else { + ASSERT_FALSE(BatchReader::IsEofBatch(batch)); + ASSERT_EQ(expected, batch.second); + ASSERT_OK_AND_ASSIGN(std::shared_ptr raw, + ImportBatch(&batch.first)); + ASSERT_EQ(5, raw->length()); + ASSERT_EQ(2, raw->num_fields()); + ASSERT_EQ("value", raw->struct_type()->field(0)->name()); + const auto offsets = checked_pointer_cast(raw->field(1)); + ASSERT_EQ(12, offsets->Value(2)); + ASSERT_OK_AND_ASSIGN(batch, readers[0]->NextBatchWithBitmap()); + ASSERT_TRUE(BatchReader::IsEofBatch(batch)); + } + readers[0]->Close(); + ASSERT_OK_AND_ASSIGN(std::vector visible, ReadOffsets(view)); + ASSERT_EQ(test_case.expected_offsets, visible); + + ASSERT_OK_AND_ASSIGN(std::optional> segment, + store_->SealForCommit()); + ASSERT_TRUE(segment.has_value()); + ASSERT_OK_AND_ASSIGN(readers, store_->CreateCommitReaders(segment.value())); + ASSERT_EQ(1, readers.size()); + ASSERT_OK_AND_ASSIGN(batch, readers[0]->NextBatchWithBitmap()); + if (expected.IsEmpty()) { + ASSERT_TRUE(BatchReader::IsEofBatch(batch)); + } else { + ASSERT_EQ(expected, batch.second); + ASSERT_OK_AND_ASSIGN(std::shared_ptr raw, + ImportBatch(&batch.first)); + ASSERT_EQ(5, raw->length()); + } + readers[0]->Close(); + } +} + +TEST_F(ArrowDeduplicateRealtimeStoreTest, TestBitmapFilteringSkipsDeletedBatches) { + ASSERT_OK( + store_->WriteAndLookup(MakeBatch(R"([[10, 1, "a"], [11, 2, "b"], [12, 3, "c"], + [13, 4, "d"], [14, 5, "e"], [15, 6, "f"]])"), + std::vector(6, RecordBatch::RowKind::INSERT), + OffsetRange(10, 16), RoaringBitmap64())); + ASSERT_OK(store_->WriteAndLookup(MakeBatch(R"([[16, 99, "delete"]])"), + {RecordBatch::RowKind::DELETE}, OffsetRange(16, 17), + RoaringBitmap64::From({10, 11, 14, 15}))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr view, store_->AcquireReadView()); + ASSERT_OK_AND_ASSIGN(std::vector> readers, + QueryReaders(view, schema_, 2)); + ASSERT_EQ(1, readers.size()); + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatchWithBitmap batch, readers[0]->NextBatchWithBitmap()); + ASSERT_FALSE(BatchReader::IsEofBatch(batch)); + ASSERT_EQ(2, batch.second.Cardinality()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr raw, ImportBatch(&batch.first)); + ASSERT_EQ(2, raw->length()); + const auto offsets = checked_pointer_cast(raw->field(0)); + ASSERT_EQ(12, offsets->Value(0)); + ASSERT_EQ(13, offsets->Value(1)); + ASSERT_OK_AND_ASSIGN(batch, readers[0]->NextBatchWithBitmap()); + ASSERT_TRUE(BatchReader::IsEofBatch(batch)); + readers[0]->Close(); +} + +TEST_F(ArrowDeduplicateRealtimeStoreTest, TestBitmapFilteringSkipsEmptyInput) { + ASSERT_NO_FATAL_FAILURE(ResetStoreWithBitmap(RoaringBitmap32())); + ASSERT_OK(store_->WriteAndLookup(MakeBatch(R"([[10, 1, "a"]])"), {RecordBatch::RowKind::INSERT}, + OffsetRange(10, 11), RoaringBitmap64())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr view, store_->AcquireReadView()); + ASSERT_OK_AND_ASSIGN(std::vector> readers, + QueryReaders(view, schema_, 1)); + ASSERT_EQ(1, readers.size()); + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatchWithBitmap batch, readers[0]->NextBatchWithBitmap()); + ASSERT_TRUE(BatchReader::IsEofBatch(batch)); + readers[0]->Close(); +} + +TEST_F(ArrowDeduplicateRealtimeStoreTest, TestSpilledBitmapFilteringPreservesRawRows) { + auto directory = UniqueTestDirectory::Create(); + ASSERT_NE(nullptr, directory); + auto fs = std::make_shared(); + delegate_ = std::make_shared(schema_, RealtimeStoreMode::APPEND_ONLY, + StatisticsMode::NONE, directory->Str(), fs, + "zstd", 1, pool_, arrow_pool_); + ASSERT_OK_AND_ASSIGN(store_, ArrowDeduplicateRealtimeStore::Create(schema_, {"id"}, delegate_, + pool_, arrow_pool_)); + ASSERT_OK(store_->AttachCommittedKeyOffsetLookup(empty_lookup_)); + ASSERT_OK(store_->WriteAndLookup( + MakeBatch(R"([[10, 1, "a"], [11, 2, "b"], [12, 3, "c"], [13, 4, "d"]])"), + std::vector(4, RecordBatch::RowKind::INSERT), OffsetRange(10, 14), + RoaringBitmap64())); + ASSERT_OK_AND_ASSIGN(std::optional> segment, + store_->SealForCommit()); + ASSERT_TRUE(segment.has_value()); + ASSERT_EQ(0, delegate_->GetDataUsage().sealed_memory_bytes); + ASSERT_OK( + store_->WriteAndLookup(MakeBatch(R"([[14, 2, "delete"], [15, 4, "delete"]])"), + std::vector(2, RecordBatch::RowKind::DELETE), + OffsetRange(14, 16), RoaringBitmap64::From({11, 13}))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr view, store_->AcquireReadView()); + ASSERT_OK_AND_ASSIGN(std::vector> readers, + QueryReaders(view, schema_, 2)); + ASSERT_EQ(1, readers.size()); + for (int64_t expected_offset : {10, 12}) { + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatchWithBitmap batch, + readers[0]->NextBatchWithBitmap()); + ASSERT_FALSE(BatchReader::IsEofBatch(batch)); + ASSERT_EQ(1, batch.second.Cardinality()); + ASSERT_TRUE(batch.second.Contains(0)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr raw, ImportBatch(&batch.first)); + ASSERT_EQ(2, raw->length()); + const auto offsets = checked_pointer_cast(raw->field(0)); + ASSERT_EQ(expected_offset, offsets->Value(0)); + } + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatchWithBitmap eof, readers[0]->NextBatchWithBitmap()); + ASSERT_TRUE(BatchReader::IsEofBatch(eof)); + readers[0]->Close(); + ASSERT_OK_AND_ASSIGN(std::vector visible, ReadOffsets(view)); + ASSERT_EQ((std::vector{10, 12}), visible); +} + +TEST_F(ArrowDeduplicateRealtimeStoreTest, TestOffsetMergeScanMatchesContains) { + const char* ordered = R"([[10, 0, "value"], [11, 1, "value"], [12, 2, "value"], + [13, 3, "value"], [14, 4, "value"]])"; + const char* gaps = R"([[0, 0, "value"], [10, 1, "value"], [1099511627776, 2, "value"], + [1099511627777, 3, "value"], [9223372036854775804, 4, "value"]])"; + const int64_t large = 1099511627776; + const int64_t high = 9223372036854775804; + struct TestCase { + const char* input_json; + std::vector input_rows; + std::vector deleted_offsets; + std::vector expected_rows; + std::vector expected_offsets; + }; + const std::vector cases = { + {ordered, {0, 1, 2, 3, 4}, {}, {0, 1, 2, 3, 4}, {10, 11, 12, 13, 14}}, + {ordered, {0, 2, 4}, {}, {0, 2, 4}, {10, 12, 14}}, + {ordered, {0, 1, 2, 3, 4}, {1}, {0, 1, 2, 3, 4}, {10, 11, 12, 13, 14}}, + {ordered, {0, 2, 4}, {1}, {0, 2, 4}, {10, 12, 14}}, + {ordered, {0, 1, 2, 3, 4}, {10, 12, large, high}, {1, 3, 4}, {11, 13, 14}}, + {ordered, {0, 2, 4}, {10, 12, large, high}, {4}, {14}}, + {ordered, {0, 1, 2, 3, 4}, {0, 10, 11, 12, 13, 14, large, 1099511627777, high}, {}, {}}, + {ordered, {0, 2, 4}, {0, 10, 11, 12, 13, 14, large, 1099511627777, high}, {}, {}}, + + {gaps, {0, 1, 2, 3, 4}, {}, {0, 1, 2, 3, 4}, {0, 10, large, 1099511627777, high}}, + {gaps, {0, 2, 4}, {}, {0, 2, 4}, {0, large, high}}, + {gaps, {0, 1, 2, 3, 4}, {1}, {0, 1, 2, 3, 4}, {0, 10, large, 1099511627777, high}}, + {gaps, {0, 2, 4}, {1}, {0, 2, 4}, {0, large, high}}, + {gaps, {0, 1, 2, 3, 4}, {10, 12, large, high}, {0, 3}, {0, 1099511627777}}, + {gaps, {0, 2, 4}, {10, 12, large, high}, {0}, {0}}, + {gaps, {0, 1, 2, 3, 4}, {0, 10, 11, 12, 13, 14, large, 1099511627777, high}, {}, {}}, + {gaps, {0, 2, 4}, {0, 10, 11, 12, 13, 14, large, 1099511627777, high}, {}, {}}, + }; + for (const TestCase& test_case : cases) { + SCOPED_TRACE(test_case.input_json); + SCOPED_TRACE(RoaringBitmap32::From(test_case.input_rows).ToString()); + SCOPED_TRACE(RoaringBitmap64::From(test_case.deleted_offsets).ToString()); + ASSERT_NO_FATAL_FAILURE(ResetStoreWithBitmap(RoaringBitmap32::From(test_case.input_rows))); + const std::shared_ptr data = MakeBatch(test_case.input_json); + ASSERT_OK(store_->WriteAndLookup( + data, std::vector(5, RecordBatch::RowKind::INSERT), + OffsetRange(0, 9223372036854775805), RoaringBitmap64::From(test_case.deleted_offsets))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr view, store_->AcquireReadView()); + ASSERT_OK_AND_ASSIGN(std::vector> readers, + QueryReaders(view, schema_, 5)); + ASSERT_EQ(1, readers.size()); + const RoaringBitmap32 expected = RoaringBitmap32::From(test_case.expected_rows); + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatchWithBitmap batch, + readers[0]->NextBatchWithBitmap()); + if (expected.IsEmpty()) { + ASSERT_TRUE(BatchReader::IsEofBatch(batch)); + } else { + ASSERT_FALSE(BatchReader::IsEofBatch(batch)); + ASSERT_EQ(expected, batch.second); + ASSERT_OK_AND_ASSIGN(std::shared_ptr raw, + ImportBatch(&batch.first)); + ASSERT_TRUE(raw->Equals(data)); + } + readers[0]->Close(); + ASSERT_OK_AND_ASSIGN(std::vector visible, ReadOffsets(view)); + ASSERT_EQ(test_case.expected_offsets, visible); + } +} + +TEST_F(ArrowDeduplicateRealtimeStoreTest, TestFreezePreservesCheckpointWithoutSpilling) { + auto directory = UniqueTestDirectory::Create(); + ASSERT_NE(nullptr, directory); + delegate_ = std::make_shared( + schema_, RealtimeStoreMode::APPEND_ONLY, StatisticsMode::NONE, directory->Str(), + directory->GetFileSystem(), "zstd", 1, pool_, arrow_pool_); + ASSERT_OK_AND_ASSIGN(store_, ArrowDeduplicateRealtimeStore::Create(schema_, {"id"}, delegate_, + pool_, arrow_pool_)); + ASSERT_OK(store_->AttachCommittedKeyOffsetLookup(empty_lookup_)); + ASSERT_OK(store_->WriteAndLookup( + MakeBatch(R"([[1, 1, "old"], [2, 1, "new"], [3, 2, "two"]])"), + {RecordBatch::RowKind::INSERT, RecordBatch::RowKind::INSERT, RecordBatch::RowKind::INSERT}, + OffsetRange(1, 4), RoaringBitmap64())); + ASSERT_OK_AND_ASSIGN(std::optional> first, + store_->FreezeBuildingSegment()); + ASSERT_TRUE(first.has_value()); + ASSERT_EQ(OffsetRange(1, 4), first.value()->GetOffsetRange()); + ASSERT_EQ(2, first.value()->GetRowCount()); + ASSERT_EQ(0, TestHelper::CountChannelFiles(directory->GetFileSystem(), directory->Str())); + ASSERT_EQ(0, delegate_->GetDataUsage().building_memory_bytes); + ASSERT_GT(delegate_->GetDataUsage().sealed_memory_bytes, 0); + ASSERT_OK_AND_ASSIGN(std::shared_ptr checkpoint, + ArrowDeduplicateRealtimeStore::OffsetDeletionsOf(first.value())); + ASSERT_EQ("{1}", checkpoint->ToString()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr pinned, store_->AcquireReadView()); + { + ASSERT_OK_AND_ASSIGN(std::shared_ptr lookup, + store_->AcquireKeyOffsetLookup()); + ASSERT_OK_AND_ASSIGN(RoaringBitmap64 previous, + lookup->LookupOffsets(MakeKeys(R"([[1], [2]])"))); + ASSERT_EQ("{2,3}", previous.ToString()); + ASSERT_OK( + store_->WriteAndLookup(MakeBatch(R"([[4, 1, "latest"], [5, 2, "deleted"]])"), + {RecordBatch::RowKind::INSERT, RecordBatch::RowKind::DELETE}, + OffsetRange(4, 6), previous)); + } + ASSERT_EQ("{1}", checkpoint->ToString()); + ASSERT_OK_AND_ASSIGN(std::vector visible, ReadOffsets(pinned)); + ASSERT_EQ((std::vector{2, 3}), visible); + ASSERT_OK_AND_ASSIGN(std::shared_ptr current, store_->AcquireReadView()); + ASSERT_OK_AND_ASSIGN(visible, ReadOffsets(current)); + ASSERT_EQ((std::vector{4}), visible); + ASSERT_OK_AND_ASSIGN(std::vector> readers, + store_->CreateCommitReaders(first.value())); + ASSERT_EQ(1, readers.size()); + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, readers[0]->NextBatch()); + ASSERT_FALSE(BatchReader::IsEofBatch(batch)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr committed, ImportBatch(&batch)); + ASSERT_TRUE(committed->Equals(MakeBatch(R"([[2, 1, "new"], [3, 2, "two"]])"))); + readers[0]->Close(); + + ASSERT_OK_AND_ASSIGN(std::optional> second, + store_->SealForCommit()); + ASSERT_TRUE(second.has_value()); + ASSERT_EQ(OffsetRange(4, 6), second.value()->GetOffsetRange()); + ASSERT_EQ(1, second.value()->GetRowCount()); + ASSERT_EQ(1, TestHelper::CountChannelFiles(directory->GetFileSystem(), directory->Str())); + ASSERT_OK_AND_ASSIGN(checkpoint, + ArrowDeduplicateRealtimeStore::OffsetDeletionsOf(second.value())); + ASSERT_EQ("{1,2,3}", checkpoint->ToString()); + + ASSERT_OK(store_->WriteAndLookup(MakeBatch(R"([[6, 1, "deleted"]])"), + {RecordBatch::RowKind::DELETE}, OffsetRange(6, 7), + RoaringBitmap64::From({4}))); + ASSERT_OK_AND_ASSIGN(std::optional> deletion, + store_->FreezeBuildingSegment()); + ASSERT_TRUE(deletion.has_value()); + ASSERT_EQ(OffsetRange(6, 7), deletion.value()->GetOffsetRange()); + ASSERT_EQ(0, deletion.value()->GetRowCount()); + ASSERT_OK_AND_ASSIGN(checkpoint, + ArrowDeduplicateRealtimeStore::OffsetDeletionsOf(deletion.value())); + ASSERT_EQ("{1,2,3,4}", checkpoint->ToString()); + ASSERT_OK_AND_ASSIGN(readers, store_->CreateCommitReaders(deletion.value())); + ASSERT_TRUE(readers.empty()); + ASSERT_EQ(1, TestHelper::CountChannelFiles(directory->GetFileSystem(), directory->Str())); +} + +TEST_F(ArrowDeduplicateRealtimeStoreTest, TestSpillDoesNotBlockWriteAndPreservesCheckpoint) { + for (bool fail_close : {false, true}) { + SCOPED_TRACE(fail_close); + auto directory = UniqueTestDirectory::Create(); + ASSERT_NE(nullptr, directory); + auto fs = std::make_shared(directory->GetFileSystem()); + auto create_gate = fs->BlockNext(BlockingFileSystem::Operation::CREATE, /*path=*/""); + if (fail_close) { + auto close_gate = fs->BlockNext(BlockingFileSystem::Operation::CLOSE, /*path=*/""); + close_gate->Release(Status::IOError("injected spill close failure")); + } + delegate_ = std::make_shared(schema_, RealtimeStoreMode::APPEND_ONLY, + StatisticsMode::NONE, directory->Str(), fs, + "zstd", 1, pool_, arrow_pool_); + ASSERT_OK_AND_ASSIGN(store_, ArrowDeduplicateRealtimeStore::Create( + schema_, {"id"}, delegate_, pool_, arrow_pool_)); + ASSERT_OK(store_->AttachCommittedKeyOffsetLookup(empty_lookup_)); + ASSERT_OK(store_->WriteAndLookup(MakeBatch(R"([[1, 1, "old"]])"), + {RecordBatch::RowKind::INSERT}, OffsetRange(1, 2), + RoaringBitmap64())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr pinned, store_->AcquireReadView()); + auto sealing = std::async(std::launch::async, [&]() { return store_->SealForCommit(); }); + std::future writing; + // Release the I/O gate before joining either task, including on a failed assertion. + ScopeGuard finish([&]() { + create_gate->Release(); + if (writing.valid()) { + writing.wait(); + } + if (sealing.valid()) { + sealing.wait(); + } + }); + ASSERT_TRUE(create_gate->WaitForArrival()); + writing = std::async(std::launch::async, [&]() -> Status { + RoaringBitmap64 previous; + { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr lookup, + store_->AcquireKeyOffsetLookup()); + PAIMON_ASSIGN_OR_RAISE(previous, lookup->LookupOffsets(MakeKeys(R"([[1]])"))); + } + if (previous.ToString() != "{1}") { + return Status::Invalid("spilling segment disappeared from the key lookup"); + } + return store_->WriteAndLookup(MakeBatch(R"([[2, 1, "new"]])"), + {RecordBatch::RowKind::INSERT}, OffsetRange(2, 3), + previous); + }); + ASSERT_EQ(std::future_status::ready, writing.wait_for(std::chrono::seconds(5))); + ASSERT_OK(writing.get()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr during_spill, + store_->AcquireReadView()); + ASSERT_OK_AND_ASSIGN(std::vector visible, ReadOffsets(during_spill)); + ASSERT_EQ((std::vector{2}), visible); + ASSERT_OK_AND_ASSIGN(visible, ReadOffsets(pinned)); + ASSERT_EQ((std::vector{1}), visible); + + create_gate->Release(); + auto result = sealing.get(); + if (fail_close) { + ASSERT_NOK_WITH_MSG(result, "injected spill close failure"); + ASSERT_OK_AND_ASSIGN(bool exists, fs->Exists(create_gate->MatchedPath())); + ASSERT_FALSE(exists); + ASSERT_GT(delegate_->GetDataUsage().sealed_memory_bytes, 0); + } else { + ASSERT_OK_AND_ASSIGN(std::optional> segment, + std::move(result)); + ASSERT_TRUE(segment.has_value()); + ASSERT_EQ(1, segment.value()->GetRowCount()); + ASSERT_EQ(OffsetRange(1, 2), segment.value()->GetOffsetRange()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr checkpoint, + ArrowDeduplicateRealtimeStore::OffsetDeletionsOf(segment.value())); + ASSERT_TRUE(checkpoint->IsEmpty()); + ASSERT_EQ(0, delegate_->GetDataUsage().sealed_memory_bytes); + ASSERT_GT(store_->GetDataUsage().sealed_memory_bytes, 0); + ASSERT_OK_AND_ASSIGN(std::vector> readers, + store_->CreateCommitReaders(segment.value())); + ASSERT_EQ(1, readers.size()); + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, readers[0]->NextBatch()); + ASSERT_FALSE(BatchReader::IsEofBatch(batch)); + auto imported = arrow::ImportArray(batch.first.get(), batch.second.get()); + ASSERT_TRUE(imported.ok()) << imported.status().ToString(); + const std::shared_ptr committed = std::move(imported).ValueOrDie(); + ASSERT_EQ(1, committed->length()); + const auto committed_rows = checked_pointer_cast(committed); + const auto offsets = checked_pointer_cast( + committed_rows->GetFieldByName(SpecialFields::RealtimeOffset().Name())); + ASSERT_EQ(1, offsets->Value(0)); + readers[0]->Close(); + } + ASSERT_OK_AND_ASSIGN(std::shared_ptr after_spill, + store_->AcquireReadView()); + ASSERT_OK_AND_ASSIGN(visible, ReadOffsets(after_spill)); + ASSERT_EQ((std::vector{2}), visible); + ASSERT_OK_AND_ASSIGN(visible, ReadOffsets(during_spill)); + ASSERT_EQ((std::vector{2}), visible); + } +} + +TEST_F(ArrowDeduplicateRealtimeStoreTest, TestMemoryUsageIncludesIndexesAndDistinctBitmaps) { + const uint64_t empty_index_bytes = store_->GetMemoryUsage(); + ASSERT_GT(empty_index_bytes, 0); + ASSERT_OK(store_->WriteAndLookup(MakeBatch(R"([[1, 1, "old"], [2, 1, "new"]])"), + {RecordBatch::RowKind::INSERT, RecordBatch::RowKind::INSERT}, + OffsetRange(1, 3), RoaringBitmap64())); + RoaringBitmap64 rows; + rows.Add(1); + rows.Add(2); + RoaringBitmap64 deleted; + deleted.Add(1); + RealtimeStoreDataUsage usage = store_->GetDataUsage(); + ASSERT_EQ(delegate_->GetDataUsage().building_memory_bytes + empty_index_bytes + + rows.GetSizeInBytes() + deleted.GetSizeInBytes(), + usage.building_memory_bytes); + ASSERT_EQ(0, usage.sealed_memory_bytes); + ASSERT_OK_AND_ASSIGN(std::shared_ptr pinned_view, store_->AcquireReadView()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr pinned_lookup, + store_->AcquireKeyOffsetLookup()); + ASSERT_OK_AND_ASSIGN(std::optional> first, + store_->SealForCommit()); + ASSERT_TRUE(first.has_value()); + usage = store_->GetDataUsage(); + // Pending DV and the sealed checkpoint share one bitmap: charge it to sealed memory only. + ASSERT_EQ(empty_index_bytes, usage.building_memory_bytes); + ASSERT_EQ(delegate_->GetDataUsage().sealed_memory_bytes + empty_index_bytes + + deleted.GetSizeInBytes(), + usage.sealed_memory_bytes); + + RoaringBitmap64 previous; + previous.Add(2); + ASSERT_OK(store_->WriteAndLookup(MakeBatch(R"([[3, 1, "latest"]])"), + {RecordBatch::RowKind::INSERT}, OffsetRange(3, 4), previous)); + rows = RoaringBitmap64(); + rows.Add(3); + deleted.Add(2); + usage = store_->GetDataUsage(); + ASSERT_EQ(delegate_->GetDataUsage().building_memory_bytes + empty_index_bytes + + rows.GetSizeInBytes() + deleted.GetSizeInBytes(), + usage.building_memory_bytes); + ASSERT_EQ(usage.building_memory_bytes + usage.sealed_memory_bytes, store_->GetMemoryUsage()); + ASSERT_OK_AND_ASSIGN(std::optional> second, + store_->SealForCommit()); + ASSERT_TRUE(second.has_value()); + ASSERT_OK(store_->RetireCommittedOffsetAndLookup(4, empty_lookup_)); + usage = store_->GetDataUsage(); + ASSERT_EQ(empty_index_bytes, usage.building_memory_bytes); + ASSERT_EQ(0, usage.sealed_memory_bytes); + ASSERT_EQ(empty_index_bytes, store_->GetMemoryUsage()); + // Reclaimed data and index snapshots can remain pinned without entering the store metrics. + ASSERT_OK_AND_ASSIGN(std::vector visible, ReadOffsets(pinned_view)); + ASSERT_EQ((std::vector{2}), visible); + ASSERT_OK_AND_ASSIGN(RoaringBitmap64 pinned_offsets, + pinned_lookup->LookupOffsets(MakeKeys(R"([[1]])"))); + ASSERT_EQ("{2}", pinned_offsets.ToString()); +} + +TEST_F(ArrowDeduplicateRealtimeStoreTest, TestDeleteOnlySegmentHasCheckpointWithoutData) { + RoaringBitmap64 previous; + previous.Add(99); + ASSERT_OK(store_->WriteAndLookup(MakeBatch(R"([[100, 1, "delete"]])"), + {RecordBatch::RowKind::DELETE}, OffsetRange(100, 101), + previous)); + ASSERT_OK_AND_ASSIGN(std::optional> segment, + store_->SealForCommit()); + ASSERT_TRUE(segment.has_value()); + ASSERT_EQ(0, segment.value()->GetRowCount()); + ASSERT_EQ(OffsetRange(100, 101), segment.value()->GetOffsetRange()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr checkpoint, + ArrowDeduplicateRealtimeStore::OffsetDeletionsOf(segment.value())); + ASSERT_EQ("{99}", checkpoint->ToString()); + ASSERT_OK_AND_ASSIGN(std::vector> readers, + store_->CreateCommitReaders(segment.value())); + ASSERT_TRUE(readers.empty()); + ASSERT_EQ(0, store_->GetDataUsage().sealed_row_count); + ASSERT_GT(store_->GetDataUsage().sealed_memory_bytes, previous.GetSizeInBytes()); +} + TEST_F(ArrowDeduplicateRealtimeStoreTest, TestValidationAndSpecializedApis) { ASSERT_NOK_WITH_MSG( ArrowDeduplicateRealtimeStore::Create(schema_, {}, delegate_, pool_, arrow_pool_), @@ -236,9 +820,9 @@ TEST_F(ArrowDeduplicateRealtimeStoreTest, TestValidationAndSpecializedApis) { "Write requires key-offset lookup"); ASSERT_NOK_WITH_MSG(store_->AdvanceCommittedOffset(1), "requires an atomic key-offset lookup replacement"); - ASSERT_NOK_WITH_MSG(store_->AdvanceCommittedOffsetAndLookup(-1, empty_lookup_), + ASSERT_NOK_WITH_MSG(store_->RetireCommittedOffsetAndLookup(-1, empty_lookup_), "must not be negative"); - ASSERT_NOK_WITH_MSG(store_->AdvanceCommittedOffsetAndLookup(1, nullptr), "must not be null"); + ASSERT_NOK_WITH_MSG(store_->RetireCommittedOffsetAndLookup(1, nullptr), "must not be null"); ASSERT_NOK_WITH_MSG( store_->AttachCommittedKeyOffsetLookup(std::make_shared()), "already attached"); diff --git a/src/paimon/core/realtime/arrow_realtime_store.cpp b/src/paimon/core/realtime/arrow_realtime_store.cpp index 45be410ab..7f9212741 100644 --- a/src/paimon/core/realtime/arrow_realtime_store.cpp +++ b/src/paimon/core/realtime/arrow_realtime_store.cpp @@ -286,6 +286,7 @@ class ArrowRealtimeStore::SharedSpillFileReader { } Result> ReadRecordBatch(int32_t batch_index) { + // TODO(xinyu.lxy): Use independent IPC readers to read the shared spill file in parallel. std::lock_guard lock(mutex_); return file_reader_->ReadRecordBatch(batch_index); } @@ -689,6 +690,18 @@ Status ArrowRealtimeStore::Write(RealtimeWriteBatch&& write_batch) { } Result>> ArrowRealtimeStore::SealForCommit() { + PAIMON_ASSIGN_OR_RAISE(std::optional> frozen, + FreezeBuildingSegment()); + if (!frozen) { + return frozen; + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr segment, + SpillFrozenSegment(frozen.value())); + return std::optional>(std::move(segment)); +} + +Result>> +ArrowRealtimeStore::FreezeBuildingSegment() { std::shared_ptr memory_segment; { std::lock_guard lock(mutex_); @@ -705,21 +718,30 @@ Result>> ArrowRealtimeStore building_memory_usage_ = 0; building_row_count_ = 0; } + return std::optional>(std::move(memory_segment)); +} +Result> ArrowRealtimeStore::SpillFrozenSegment( + const std::shared_ptr& segment) { + std::shared_ptr memory_segment = + std::dynamic_pointer_cast(segment); + if (!memory_segment) { + return Status::Invalid("real-time spill requires a frozen memory segment"); + } if (!io_manager_) { - return std::optional>(memory_segment); + return segment; } PAIMON_ASSIGN_OR_RAISE(FileIOChannel::ID channel_id, io_manager_->CreateChannel("realtime")); - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr writer, - ArrowIpcFileWriter::Create(spill_file_system_, channel_id.GetPath(), write_schema_, - spill_compression_, spill_compression_level_, - /*use_threads=*/false, arrow_pool_)); + std::unique_ptr writer; auto cleanup_guard = ScopeGuard([&]() { writer.reset(); [[maybe_unused]] Status status = spill_file_system_->Delete(channel_id.GetPath()); }); + PAIMON_ASSIGN_OR_RAISE( + writer, ArrowIpcFileWriter::Create(spill_file_system_, channel_id.GetPath(), write_schema_, + spill_compression_, spill_compression_level_, + /*use_threads=*/false, arrow_pool_)); std::vector spilled_batches; spilled_batches.reserve(memory_segment->GetBatches().size()); for (const StoredBatch& batch : memory_segment->GetBatches()) { @@ -754,7 +776,7 @@ Result>> ArrowRealtimeStore sealed_memory_usage_ += spilled_segment->GetMemoryUsage(); } } - return std::optional>(std::move(spilled_segment)); + return std::shared_ptr(std::move(spilled_segment)); } Result>> ArrowRealtimeStore::CreateCommitReaders( @@ -902,6 +924,13 @@ Result>> ArrowRealtimeStore::CreateQuer } Status ArrowRealtimeStore::AdvanceCommittedOffset(int64_t committed_end_offset) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr retired, + RetireCommittedSegments(committed_end_offset)); + return Status::OK(); +} + +Result> ArrowRealtimeStore::RetireCommittedSegments( + int64_t committed_end_offset) { // Drop the store's references only after releasing the write mutex. Segment destruction may // release large Arrow buffers or delete a spill file. std::vector> reclaimed_segments; @@ -927,7 +956,7 @@ Status ArrowRealtimeStore::AdvanceCommittedOffset(int64_t committed_end_offset) sealed_memory_usage_ -= reclaimed_memory_usage; sealed_row_count_ -= reclaimed_row_count; } - return Status::OK(); + return std::shared_ptr(new ReadView(std::move(reclaimed_segments))); } RealtimeStoreDataUsage ArrowRealtimeStore::GetDataUsage() const { diff --git a/src/paimon/core/realtime/arrow_realtime_store.h b/src/paimon/core/realtime/arrow_realtime_store.h index e92a71993..81d8510b0 100644 --- a/src/paimon/core/realtime/arrow_realtime_store.h +++ b/src/paimon/core/realtime/arrow_realtime_store.h @@ -42,7 +42,7 @@ class MemoryPool; class PredicateFilter; /// Internal Arrow-backed implementation of the default `RealtimeStore`. -class ArrowRealtimeStore final : public RealtimeStore { +class ArrowRealtimeStore final : public RealtimeStore, public RealtimeStoreTwoPhaseSeal { public: ArrowRealtimeStore(const std::shared_ptr& write_schema, RealtimeStoreMode mode, StatisticsMode statistics_mode, const std::string& temp_directory, @@ -55,6 +55,11 @@ class ArrowRealtimeStore final : public RealtimeStore { Result>> SealForCommit() override; + Result>> FreezeBuildingSegment() override; + + Result> SpillFrozenSegment( + const std::shared_ptr& segment) override; + Result>> CreateCommitReaders( const std::shared_ptr& segment) override; @@ -66,6 +71,9 @@ class ArrowRealtimeStore final : public RealtimeStore { Status AdvanceCommittedOffset(int64_t committed_end_offset) override; + Result> RetireCommittedSegments( + int64_t committed_end_offset) override; + RealtimeStoreDataUsage GetDataUsage() const override; uint64_t GetMemoryUsage() const override; diff --git a/src/paimon/core/realtime/arrow_realtime_store_test.cpp b/src/paimon/core/realtime/arrow_realtime_store_test.cpp index 60ea9f98e..27dd14b9d 100644 --- a/src/paimon/core/realtime/arrow_realtime_store_test.cpp +++ b/src/paimon/core/realtime/arrow_realtime_store_test.cpp @@ -39,6 +39,7 @@ #include "paimon/realtime/arrow_realtime_store_factory.h" #include "paimon/record_batch.h" #include "paimon/testing/utils/read_result_collector.h" +#include "paimon/testing/utils/test_helper.h" #include "paimon/testing/utils/testharness.h" namespace paimon::test { @@ -372,6 +373,58 @@ TEST_F(ArrowRealtimeStoreTest, TestSealSpillsAndKeepsPinnedMemoryViewReadable) { } } +TEST_F(ArrowRealtimeStoreTest, TestTwoPhaseSealDoesNotResurrectReclaimedSegment) { + auto directory = UniqueTestDirectory::Create(); + ASSERT_NE(nullptr, directory); + auto store = std::make_shared( + schema_, RealtimeStoreMode::APPEND_ONLY, StatisticsMode::NONE, directory->Str(), + directory->GetFileSystem(), "zstd", 1, pool_, arrow_pool_); + ASSERT_NOK_WITH_MSG(store->SpillFrozenSegment(std::make_shared()), + "requires a frozen memory segment"); + ASSERT_OK_AND_ASSIGN(std::optional> empty, + store->FreezeBuildingSegment()); + ASSERT_FALSE(empty.has_value()); + ASSERT_OK(store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 1, "a"]])"), OffsetRange(0, 1)})); + ASSERT_OK_AND_ASSIGN(std::optional> frozen, + store->FreezeBuildingSegment()); + ASSERT_TRUE(frozen.has_value()); + ASSERT_GT(store->GetDataUsage().sealed_memory_bytes, 0); + ASSERT_OK(store->Write(RealtimeWriteBatch{MakeBatch(R"([[1, 2, "b"]])"), OffsetRange(1, 2)})); + const uint64_t building_bytes = store->GetDataUsage().building_memory_bytes; + ASSERT_GT(building_bytes, 0); + ASSERT_OK(store->AdvanceCommittedOffset(1)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr spilled, + store->SpillFrozenSegment(frozen.value())); + ASSERT_EQ(OffsetRange(0, 1), spilled->GetOffsetRange()); + ASSERT_EQ(1, spilled->GetRowCount()); + const RealtimeStoreDataUsage usage = store->GetDataUsage(); + ASSERT_EQ(building_bytes, usage.building_memory_bytes); + ASSERT_EQ(0, usage.sealed_memory_bytes); + ASSERT_EQ(1, usage.building_row_count); + ASSERT_EQ(0, usage.sealed_row_count); + ASSERT_OK_AND_ASSIGN(std::shared_ptr view, store->AcquireReadView()); + ASSERT_EQ(std::optional(OffsetRange(1, 2)), view->GetOffsetRange()); +} + +TEST_F(ArrowRealtimeStoreTest, TestRetiredSpillStorageIsReleasedByCaller) { + auto directory = UniqueTestDirectory::Create(); + ASSERT_NE(nullptr, directory); + auto store = std::make_shared( + schema_, RealtimeStoreMode::APPEND_ONLY, StatisticsMode::NONE, directory->Str(), + directory->GetFileSystem(), "zstd", 1, pool_, arrow_pool_); + ASSERT_OK(store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 1, "a"]])"), OffsetRange(0, 1)})); + ASSERT_OK_AND_ASSIGN(std::optional> segment, + store->SealForCommit()); + ASSERT_TRUE(segment.has_value()); + segment.reset(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr retired, + store->RetireCommittedSegments(1)); + ASSERT_EQ(0, store->GetDataUsage().sealed_row_count); + ASSERT_EQ(1, TestHelper::CountChannelFiles(directory->GetFileSystem(), directory->Str())); + retired.reset(); + ASSERT_EQ(0, TestHelper::CountChannelFiles(directory->GetFileSystem(), directory->Str())); +} + TEST_F(ArrowRealtimeStoreTest, TestFactoryDoesNotSpillWithoutOption) { std::unique_ptr temp_directory = UniqueTestDirectory::Create(); ASSERT_NE(nullptr, temp_directory); diff --git a/src/paimon/core/realtime/realtime_append_only_writer.cpp b/src/paimon/core/realtime/realtime_append_only_writer.cpp index a57646853..f910ad5fa 100644 --- a/src/paimon/core/realtime/realtime_append_only_writer.cpp +++ b/src/paimon/core/realtime/realtime_append_only_writer.cpp @@ -118,9 +118,11 @@ Status RealtimeAppendOnlyWriter::Write(std::unique_ptr&& batch) { return Status::OK(); } -Status RealtimeAppendOnlyWriter::SealCurrentSegment() { +Status RealtimeAppendOnlyWriter::SealCurrentSegment(bool spill) { + auto* two_phase = dynamic_cast(realtime_store_.get()); PAIMON_ASSIGN_OR_RAISE(std::optional> segment, - realtime_store_->SealForCommit()); + !spill && two_phase ? two_phase->FreezeBuildingSegment() + : realtime_store_->SealForCommit()); std::lock_guard lock(realtime_store_mutex_); if (!segment) { return Status::OK(); @@ -135,12 +137,12 @@ Status RealtimeAppendOnlyWriter::SealCurrentSegment() { Status RealtimeAppendOnlyWriter::Seal() { std::lock_guard lock(prepare_mutex_); - return SealCurrentSegment(); + return SealCurrentSegment(/*spill=*/true); } Result RealtimeAppendOnlyWriter::PrepareCommit(bool wait_compaction) { std::lock_guard lock(prepare_mutex_); - PAIMON_RETURN_NOT_OK(SealCurrentSegment()); + PAIMON_RETURN_NOT_OK(SealCurrentSegment(/*spill=*/false)); std::vector> segments; { std::lock_guard realtime_store_lock(realtime_store_mutex_); @@ -212,8 +214,7 @@ Status RealtimeAppendOnlyWriter::Compact(bool) { } uint64_t RealtimeAppendOnlyWriter::GetMemoryUsage() const { - // The first implementation does not spill sealed or building segments through - // WriterMemoryManager. + // Store usage is reported through RealtimeContext, not WriterMemoryManager. return 0; } diff --git a/src/paimon/core/realtime/realtime_append_only_writer.h b/src/paimon/core/realtime/realtime_append_only_writer.h index c007e3377..721b0fade 100644 --- a/src/paimon/core/realtime/realtime_append_only_writer.h +++ b/src/paimon/core/realtime/realtime_append_only_writer.h @@ -80,7 +80,7 @@ class RealtimeAppendOnlyWriter : public BatchWriter { int64_t next_offset, const std::shared_ptr& memory_pool); Status FlushSegment(const std::shared_ptr& segment); - Status SealCurrentSegment(); + Status SealCurrentSegment(bool spill); std::shared_ptr arrow_pool_; std::shared_ptr realtime_store_; diff --git a/src/paimon/core/realtime/realtime_context_impl.cpp b/src/paimon/core/realtime/realtime_context_impl.cpp index fc09f3047..558578e5e 100644 --- a/src/paimon/core/realtime/realtime_context_impl.cpp +++ b/src/paimon/core/realtime/realtime_context_impl.cpp @@ -44,6 +44,7 @@ #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/scope_guard.h" #include "paimon/common/utils/uuid.h" +#include "paimon/core/core_options.h" #include "paimon/core/realtime/arrow_deduplicate_realtime_store.h" #include "paimon/core/realtime/realtime_deduplicate_state.h" #include "paimon/macros.h" @@ -216,8 +217,11 @@ Result RealtimeContextImpl::GetOrCreateRealtimeStore( std::string temp_directory = request.temp_directory; std::vector deduplicate_key_fields = request.deduplicate_key_fields; std::shared_ptr memory_pool = request.memory_pool; + bool deduplicate_spill = false; std::shared_ptr deduplicate_state; if (mode == RealtimeStoreMode::DEDUPLICATE) { + PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(request.options)); + deduplicate_spill = options.RealtimeSpillEnabled(); deduplicate_state = std::make_shared(); // DEDUPLICATE semantics belong to the framework wrapper. The pluggable factory only // supplies the append storage delegate and cannot bypass key-index or offset-DV handling. @@ -228,6 +232,9 @@ Result RealtimeContextImpl::GetOrCreateRealtimeStore( if (!delegate) { return Status::Invalid("real-time store factory returned a null store"); } + if (deduplicate_spill && !dynamic_cast(delegate.get())) { + return Status::Invalid("deduplicate spill requires a delegate with two-phase sealing"); + } std::shared_ptr store = delegate; if (mode == RealtimeStoreMode::DEDUPLICATE) { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr deduplicate_store, @@ -369,104 +376,147 @@ Status RealtimeContextImpl::AdvanceCommittedProgress( if (snapshot_id < 0) { return Status::Invalid("real-time refresh snapshot id must not be negative"); } - std::lock_guard progress_lock(progress_mutex_); - if (last_refreshed_snapshot_id_ && snapshot_id < last_refreshed_snapshot_id_.value()) { - return Status::Invalid("real-time committed snapshot cannot move backwards"); - } - if (!last_refreshed_snapshot_id_ || snapshot_id > last_refreshed_snapshot_id_.value()) { - for (const auto& [partition_bucket, committed_end_offset] : committed_offsets) { - if (partition_bucket.bucket < 0 || committed_end_offset < 0) { - return Status::Invalid("invalid partition-bucket committed offset"); - } + std::lock_guard refresh_lock(refresh_mutex_); + RealtimeOffsetMap next_offsets; + RealtimeDataFileMap next_data_files; + { + std::lock_guard progress_lock(progress_mutex_); + if (last_refreshed_snapshot_id_ && snapshot_id < last_refreshed_snapshot_id_.value()) { + return Status::Invalid("real-time committed snapshot cannot move backwards"); } - // Only stores created by this context can contain state which cannot be restored in - // place. Offsets for other partition-buckets are reference state for lazy store creation - // and may be removed or rolled back without rebuilding the context. - std::lock_guard registry_lock(mutex_); - for (const auto& store_entry : stores_) { - const RealtimePartitionBucket& partition_bucket = store_entry.first; - auto previous_iter = committed_offsets_.find(partition_bucket); - if (previous_iter == committed_offsets_.end()) { - continue; - } - - auto current_iter = committed_offsets.find(partition_bucket); - if (current_iter == committed_offsets.end()) { - return Status::Invalid( - "real-time committed progress removed an active partition-bucket; recreate " - "RealtimeContext"); - } - if (current_iter->second < previous_iter->second) { - return Status::Invalid( - "real-time committed offset moved backwards for an active partition-bucket; " - "recreate RealtimeContext"); - } + const bool retry = last_refreshed_snapshot_id_ == snapshot_id; + next_offsets = retry ? committed_offsets_ : committed_offsets; + next_data_files = retry ? committed_data_files_ : committed_data_files; + } + for (const auto& [partition_bucket, committed_end_offset] : next_offsets) { + if (partition_bucket.bucket < 0 || committed_end_offset < 0) { + return Status::Invalid("invalid partition-bucket committed offset"); } - committed_offsets_ = committed_offsets; - committed_data_files_ = committed_data_files; - last_refreshed_snapshot_id_ = snapshot_id; } - struct Notification { - RealtimePartitionBucket partition_bucket; std::shared_ptr store; std::shared_ptr deduplicate_state; int64_t committed_end_offset; std::vector> active_data_files; + std::optional prepared; }; - std::vector notifications; - { - std::lock_guard registry_lock(mutex_); - for (const auto& [partition_bucket, committed_end_offset] : committed_offsets_) { - auto store_iter = stores_.find(partition_bucket); - if (store_iter == stores_.end()) { - continue; + std::map notifications; + std::vector> retired_views; + while (true) { + PAIMON_RETURN_NOT_OK(CheckUsable()); + std::vector pending; + std::unique_lock progress_lock(progress_mutex_); + { + std::lock_guard registry_lock(mutex_); + for (const auto& store_entry : stores_) { + const RealtimePartitionBucket& partition_bucket = store_entry.first; + auto previous = committed_offsets_.find(partition_bucket); + if (previous == committed_offsets_.end()) { + continue; + } + auto next = next_offsets.find(partition_bucket); + if (next == next_offsets.end()) { + return Status::Invalid( + "real-time committed progress removed an active partition-bucket; recreate " + "RealtimeContext"); + } + if (next->second < previous->second) { + return Status::Invalid( + "real-time committed offset moved backwards for an active " + "partition-bucket; " + "recreate RealtimeContext"); + } } - const std::shared_ptr& deduplicate_state = - store_iter->second.deduplicate_state; - if (deduplicate_state) { - auto installed_iter = installed_snapshot_ids_.find(partition_bucket); - if (installed_iter != installed_snapshot_ids_.end() && - installed_iter->second >= last_refreshed_snapshot_id_.value()) { + for (const auto& [partition_bucket, committed_end_offset] : next_offsets) { + auto store_iter = stores_.find(partition_bucket); + if (store_iter == stores_.end()) { continue; } - } else { - auto reclaimed_iter = reclaimed_offsets_.find(partition_bucket); - if (reclaimed_iter != reclaimed_offsets_.end() && - reclaimed_iter->second >= committed_end_offset) { + const StoreEntry& entry = store_iter->second; + if (entry.deduplicate_state) { + // Compaction can change lookup files without advancing the committed offset. + auto installed = installed_snapshot_ids_.find(partition_bucket); + if (installed != installed_snapshot_ids_.end() && + installed->second >= snapshot_id) { + continue; + } + } else { + auto reclaimed = reclaimed_offsets_.find(partition_bucket); + if (reclaimed != reclaimed_offsets_.end() && + reclaimed->second >= committed_end_offset) { + continue; + } + } + if (notifications.find(partition_bucket) != notifications.end()) { continue; } + auto files = next_data_files.find(partition_bucket); + auto iter = + notifications + .emplace( + partition_bucket, + Notification{entry.store, entry.deduplicate_state, committed_end_offset, + files == next_data_files.end() + ? std::vector>() + : files->second, + std::nullopt}) + .first; + if (entry.deduplicate_state) { + pending.push_back(&iter->second); + } } - auto files_iter = committed_data_files_.find(partition_bucket); - std::vector> active_data_files = - files_iter == committed_data_files_.end() - ? std::vector>() - : files_iter->second; - notifications.push_back(Notification{partition_bucket, store_iter->second.store, - deduplicate_state, committed_end_offset, - std::move(active_data_files)}); } - } - // Reclaim independent stores on a best-effort basis, then report the first error. - Status first_error = Status::OK(); - for (const Notification& notification : notifications) { - Status status = - notification.deduplicate_state - ? notification.deduplicate_state->InstallCommittedSnapshot( - notification.committed_end_offset, notification.active_data_files, - notification.store) - : notification.store->AdvanceCommittedOffset(notification.committed_end_offset); - if (status.ok()) { - reclaimed_offsets_[notification.partition_bucket] = notification.committed_end_offset; + if (!pending.empty()) { + // Open sidecars outside the progress lock to avoid blocking scans on I/O. + progress_lock.unlock(); + for (Notification* notification : pending) { + PAIMON_ASSIGN_OR_RAISE(RealtimeDeduplicateState::PreparedSnapshot prepared, + notification->deduplicate_state->PrepareCommittedSnapshot( + notification->active_data_files)); + notification->prepared = std::move(prepared); + } + // Recheck stores registered while sidecars were opened. + continue; + } + + committed_offsets_ = std::move(next_offsets); + committed_data_files_ = std::move(next_data_files); + last_refreshed_snapshot_id_ = snapshot_id; + Status first_error = Status::OK(); + for (const auto& [partition_bucket, notification] : notifications) { + Status status; if (notification.deduplicate_state) { - installed_snapshot_ids_[notification.partition_bucket] = - last_refreshed_snapshot_id_.value(); + Result> retired = + notification.deduplicate_state->InstallPreparedSnapshot( + notification.committed_end_offset, notification.prepared.value(), + notification.store); + status = retired.status(); + if (status.ok()) { + retired_views.push_back(std::move(retired).value()); + } + } else if (auto* two_phase = + dynamic_cast(notification.store.get())) { + Result> retired = + two_phase->RetireCommittedSegments(notification.committed_end_offset); + status = retired.status(); + if (status.ok()) { + retired_views.push_back(std::move(retired).value()); + } + } else { + status = + notification.store->AdvanceCommittedOffset(notification.committed_end_offset); + } + if (status.ok()) { + reclaimed_offsets_[partition_bucket] = notification.committed_end_offset; + if (notification.deduplicate_state) { + installed_snapshot_ids_[partition_bucket] = snapshot_id; + } + } else if (first_error.ok()) { + first_error = std::move(status); } - } else if (first_error.ok()) { - first_error = std::move(status); } + return first_error; } - return first_error; } void RealtimeContextImpl::CleanupReadViews() { diff --git a/src/paimon/core/realtime/realtime_context_impl.h b/src/paimon/core/realtime/realtime_context_impl.h index 7204b3834..8a1a31479 100644 --- a/src/paimon/core/realtime/realtime_context_impl.h +++ b/src/paimon/core/realtime/realtime_context_impl.h @@ -137,6 +137,7 @@ class PAIMON_EXPORT RealtimeContextImpl final : public RealtimeContext { std::shared_ptr factory_; mutable std::mutex mutex_; + std::mutex refresh_mutex_; std::mutex progress_mutex_; std::map stores_; // Full-table progress used as the initial offset when a store is created lazily. diff --git a/src/paimon/core/realtime/realtime_context_test.cpp b/src/paimon/core/realtime/realtime_context_test.cpp index 155bb5866..87d0cb104 100644 --- a/src/paimon/core/realtime/realtime_context_test.cpp +++ b/src/paimon/core/realtime/realtime_context_test.cpp @@ -17,6 +17,7 @@ */ #include +#include #include #include #include @@ -28,16 +29,22 @@ #include "arrow/api.h" #include "arrow/c/bridge.h" #include "arrow/c/helpers.h" +#include "arrow/ipc/json_simple.h" #include "paimon/common/table/special_fields.h" +#include "paimon/common/utils/checked_cast.h" +#include "paimon/common/utils/scope_guard.h" #include "paimon/core/core_options.h" +#include "paimon/core/io/data_file_meta.h" #include "paimon/core/io/data_file_path_factory.h" #include "paimon/core/realtime/arrow_deduplicate_realtime_store.h" +#include "paimon/core/realtime/data_file_key_offset_index_writer.h" #include "paimon/core/realtime/realtime_context_impl.h" #include "paimon/core/realtime/realtime_deduplicate_state.h" #include "paimon/core/realtime/realtime_offset_file_index_lookup.h" #include "paimon/fs/local/local_file_system.h" #include "paimon/memory/memory_pool.h" #include "paimon/realtime/arrow_realtime_store_factory.h" +#include "paimon/testing/utils/blocking_file_system.h" #include "paimon/testing/utils/testharness.h" namespace paimon::test { @@ -166,6 +173,138 @@ Result GetOrCreateAppendStore( } // namespace +class RealtimeRefreshTest : public testing::Test { + protected: + void SetUp() override { + directory_ = UniqueTestDirectory::Create(); + ASSERT_NE(nullptr, directory_); + fs_ = std::make_shared(directory_->GetFileSystem()); + path_factory_ = std::make_shared(); + ASSERT_OK(path_factory_->Init(directory_->Str(), "parquet", "data-", nullptr)); + schema_ = MakeDeduplicateSchema(); + ASSERT_OK_AND_ASSIGN(context_, + CreateContext(std::make_shared())); + ASSERT_OK_AND_ASSIGN(state_, CreateStore(0)); + ASSERT_OK(context_->AdvanceCommittedProgress(1, {{bucket_, 0}}, {})); + } + + Result CreateStore(int32_t bucket) const { + RealtimeStoreCreateRequest request{ExportSchema(schema_), options_, GetDefaultPool(), + RealtimeStoreMode::DEDUPLICATE}; + request.deduplicate_key_fields = {"id"}; + PAIMON_ASSIGN_OR_RAISE(RealtimeStoreState state, + context_->GetOrCreateRealtimeStore( + std::move(request), RealtimePartitionBucket({}, bucket))); + PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(options_, fs_)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr lookup, + RealtimeOffsetFileIndexLookup::Create(schema_, 0, schema_->field(1), {}, path_factory_, + fs_, options, nullptr, GetDefaultPool())); + PAIMON_RETURN_NOT_OK(state.deduplicate_state->AttachFileIndexLookup(lookup, state.store)); + return state; + } + + Result> CreateFile(const std::string& name, + const std::string& json) const { + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::shared_ptr array, + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(schema_->fields()), json)); + const auto batch = checked_pointer_cast(array); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr file, + DataFileMeta::ForAppend(name, 0, batch->length(), SimpleStats::EmptyStats(), 0, 0, 0, + std::nullopt, std::nullopt, std::nullopt, std::nullopt, + std::nullopt)); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr writer, + DataFileKeyOffsetIndexWriter::Create(schema_->field(1), fs_, GetDefaultPool())); + PAIMON_RETURN_NOT_OK(writer->AddBatch(batch)); + PAIMON_ASSIGN_OR_RAISE(std::string sidecar, + writer->Finish(path_factory_->ToPath(file), file->row_count)); + return file->CopyWithExtraFiles({sidecar}); + } + + const RealtimePartitionBucket bucket_{{}, 0}; + const std::map options_ = { + {"file-index.bitmap.columns", SpecialFields::RealtimeOffset().Name()}}; + std::unique_ptr directory_; + std::shared_ptr fs_; + std::shared_ptr path_factory_; + std::shared_ptr schema_; + std::shared_ptr context_; + RealtimeStoreState state_; +}; + +TEST_F(RealtimeRefreshTest, TestRechecksProgressForStoresCreatedDuringPreparation) { + const RealtimePartitionBucket second_bucket({}, 1); + ASSERT_OK(context_->AdvanceCommittedProgress(2, {{bucket_, 0}, {second_bucket, 10}}, {})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr file, + CreateFile("first.parquet", R"([[0, 1, "old"]])")); + auto open_gate = fs_->BlockNext(BlockingFileSystem::Operation::OPEN, + path_factory_->ToPath(file) + ".offset"); + auto refreshing = std::async(std::launch::async, [&]() { + return context_->AdvanceCommittedProgress(3, {{bucket_, 1}, {second_bucket, 5}}, + {{bucket_, {file}}}); + }); + std::future> registering; + ScopeGuard finish([&]() { + open_gate->Release(); + if (registering.valid()) { + registering.wait(); + } + if (refreshing.valid()) { + refreshing.wait(); + } + }); + ASSERT_TRUE(open_gate->WaitForArrival()); + registering = std::async(std::launch::async, [&]() { return CreateStore(1); }); + ASSERT_EQ(std::future_status::ready, registering.wait_for(std::chrono::seconds(5))); + ASSERT_OK_AND_ASSIGN(RealtimeStoreState added, registering.get()); + ASSERT_EQ(10, added.initial_offset); + open_gate->Release(); + ASSERT_NOK_WITH_MSG(refreshing.get(), "recreate RealtimeContext"); + ASSERT_OK_AND_ASSIGN(RealtimeReadState unchanged, context_->AcquireReadState()); + ASSERT_EQ(0, unchanged.committed_offsets.at(bucket_)); + ASSERT_EQ(10, unchanged.committed_offsets.at(second_bucket)); + ASSERT_TRUE(state_.deduplicate_state->committed_file_lookup_->DataFiles().empty()); +} + +TEST_F(RealtimeRefreshTest, TestSerializesConcurrentRefreshes) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr first, + CreateFile("first.parquet", R"([[0, 1, "old"]])")); + ASSERT_OK_AND_ASSIGN(std::shared_ptr second, + CreateFile("second.parquet", R"([[1, 1, "new"]])")); + auto open_gate = fs_->BlockNext(BlockingFileSystem::Operation::OPEN, + path_factory_->ToPath(first) + ".offset"); + auto refreshing = std::async(std::launch::async, [&]() { + return context_->AdvanceCommittedProgress(2, {{bucket_, 1}}, {{bucket_, {first}}}); + }); + std::future next; + ScopeGuard finish([&]() { + open_gate->Release(); + if (next.valid()) { + next.wait(); + } + if (refreshing.valid()) { + refreshing.wait(); + } + }); + ASSERT_TRUE(open_gate->WaitForArrival()); + next = std::async(std::launch::async, [&]() { + return context_->AdvanceCommittedProgress(3, {{bucket_, 2}}, {{bucket_, {second}}}); + }); + ASSERT_EQ(std::future_status::timeout, next.wait_for(std::chrono::milliseconds(100))); + open_gate->Release(); + ASSERT_OK(refreshing.get()); + ASSERT_OK(next.get()); + ASSERT_OK_AND_ASSIGN(RealtimeReadState after, context_->AcquireReadState()); + ASSERT_EQ(2, after.committed_offsets.at(bucket_)); + ASSERT_EQ(std::vector>{second}, + state_.deduplicate_state->committed_file_lookup_->DataFiles()); + ASSERT_NOK_WITH_MSG(context_->AdvanceCommittedProgress(2, {{bucket_, 1}}, {{bucket_, {first}}}), + "committed snapshot cannot move backwards"); +} + TEST(RealtimeContextTest, TestReusesStoreAndCapturesRegisteredViews) { auto factory = std::make_shared(); ASSERT_OK_AND_ASSIGN(std::shared_ptr context, CreateContext(factory)); @@ -393,6 +532,38 @@ TEST(RealtimeContextTest, TestDeduplicateStoreWrapsAppendDelegateFromFactory) { ASSERT_NE(nullptr, read_state.views[0].offset_deletions); } +TEST(RealtimeContextTest, TestDeduplicateSpillRequiresTwoPhaseDelegate) { + const auto directory = UniqueTestDirectory::Create(); + ASSERT_NE(nullptr, directory); + const auto schema = MakeDeduplicateSchema(); + for (bool use_default_factory : {false, true}) { + SCOPED_TRACE(use_default_factory); + std::shared_ptr factory; + if (use_default_factory) { + factory = std::make_shared(); + } else { + factory = std::make_shared(); + } + ASSERT_OK_AND_ASSIGN(std::shared_ptr context, CreateContext(factory)); + RealtimeStoreCreateRequest request{ExportSchema(schema), + {{Options::REALTIME_SPILL_ENABLED, "true"}}, + GetDefaultPool(), + RealtimeStoreMode::DEDUPLICATE}; + request.deduplicate_key_fields = {"id"}; + request.temp_directory = directory->Str(); + request.file_system = directory->GetFileSystem(); + auto result = context->GetOrCreateRealtimeStore( + std::move(request), RealtimePartitionBucket(/*partition=*/{}, /*bucket=*/0)); + if (use_default_factory) { + ASSERT_OK_AND_ASSIGN(RealtimeStoreState state, std::move(result)); + ASSERT_NE(nullptr, + std::dynamic_pointer_cast(state.store)); + } else { + ASSERT_NOK_WITH_MSG(result, "requires a delegate with two-phase sealing"); + } + } +} + TEST(RealtimeContextTest, TestNewDeduplicateSnapshotRefreshesLookupAtSameOffset) { auto factory = std::make_shared(); ASSERT_OK_AND_ASSIGN(std::shared_ptr context, CreateContext(factory)); @@ -410,11 +581,11 @@ TEST(RealtimeContextTest, TestNewDeduplicateSnapshotRefreshesLookupAtSameOffset) ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options)); auto path_factory = std::make_shared(); auto file_system = std::make_shared(); - ASSERT_OK_AND_ASSIGN( - std::shared_ptr initial_lookup, - RealtimeOffsetFileIndexLookup::Create( - schema, /*data_schema_id=*/0, schema->GetFieldByName("id"), - /*data_files=*/{}, path_factory, file_system, GetDefaultPool(), core_options)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr initial_lookup, + RealtimeOffsetFileIndexLookup::Create( + schema, /*data_schema_id=*/0, schema->GetFieldByName("id"), + /*data_files=*/{}, path_factory, file_system, core_options, + /*schema_manager=*/nullptr, GetDefaultPool())); ASSERT_OK( store_state.deduplicate_state->AttachFileIndexLookup(initial_lookup, store_state.store)); diff --git a/src/paimon/core/realtime/realtime_deduplicate_state.cpp b/src/paimon/core/realtime/realtime_deduplicate_state.cpp index d913b03a8..1cb714256 100644 --- a/src/paimon/core/realtime/realtime_deduplicate_state.cpp +++ b/src/paimon/core/realtime/realtime_deduplicate_state.cpp @@ -50,18 +50,9 @@ Status RealtimeDeduplicateState::AttachFileIndexLookup( return Status::OK(); } -Status RealtimeDeduplicateState::InstallCommittedSnapshot( - int64_t committed_end_offset, - const std::vector>& active_data_files, - const std::shared_ptr& store) { - if (committed_end_offset < 0 || !store) { - return Status::Invalid("invalid deduplicate committed advancement"); - } - std::shared_ptr deduplicate_store = - std::dynamic_pointer_cast(store); - if (!deduplicate_store) { - return Status::Invalid("deduplicate state requires ArrowDeduplicateRealtimeStore"); - } +Result +RealtimeDeduplicateState::PrepareCommittedSnapshot( + const std::vector>& active_data_files) const { std::shared_ptr current_file_lookup; { std::lock_guard lock(mutex_); @@ -74,13 +65,31 @@ Status RealtimeDeduplicateState::InstallCommittedSnapshot( current_file_lookup->WithDataFiles(active_data_files)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr next_key_lookup, next_file_lookup->CreateKeyOffsetLookup()); - { - std::lock_guard lock(mutex_); - PAIMON_RETURN_NOT_OK(deduplicate_store->AdvanceCommittedOffsetAndLookup( - committed_end_offset, next_key_lookup)); - committed_file_lookup_ = std::move(next_file_lookup); + return PreparedSnapshot{std::move(current_file_lookup), std::move(next_file_lookup), + std::move(next_key_lookup)}; +} + +Result> RealtimeDeduplicateState::InstallPreparedSnapshot( + int64_t committed_end_offset, const PreparedSnapshot& prepared, + const std::shared_ptr& store) { + if (committed_end_offset < 0 || !store || !prepared.previous_file_lookup || + !prepared.file_lookup || !prepared.key_lookup) { + return Status::Invalid("invalid deduplicate committed advancement"); } - return Status::OK(); + std::shared_ptr deduplicate_store = + std::dynamic_pointer_cast(store); + if (!deduplicate_store) { + return Status::Invalid("deduplicate state requires ArrowDeduplicateRealtimeStore"); + } + std::lock_guard lock(mutex_); + if (committed_file_lookup_ != prepared.previous_file_lookup) { + return Status::Invalid("deduplicate committed lookup changed during refresh preparation"); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr retired, + deduplicate_store->RetireCommittedOffsetAndLookup(committed_end_offset, + prepared.key_lookup)); + committed_file_lookup_ = prepared.file_lookup; + return retired; } } // namespace paimon diff --git a/src/paimon/core/realtime/realtime_deduplicate_state.h b/src/paimon/core/realtime/realtime_deduplicate_state.h index 50e703ee0..51013c452 100644 --- a/src/paimon/core/realtime/realtime_deduplicate_state.h +++ b/src/paimon/core/realtime/realtime_deduplicate_state.h @@ -28,6 +28,8 @@ namespace paimon { class RealtimeStore; +class RealtimeReadView; +class KeyOffsetLookup; struct DataFileMeta; class RealtimeOffsetFileIndexLookup; @@ -38,12 +40,23 @@ class RealtimeOffsetFileIndexLookup; /// RealtimeCommitProgress carries them to commit, which materializes snapshot-specific DVs. class RealtimeDeduplicateState { public: + struct PreparedSnapshot { + // Current snapshot's file lookup. + std::shared_ptr previous_file_lookup; + // Next snapshot's file lookup. + std::shared_ptr file_lookup; + // Next snapshot's key-to-offset lookup. + std::shared_ptr key_lookup; + }; + Status AttachFileIndexLookup(const std::shared_ptr& file_lookup, const std::shared_ptr& store); - Status InstallCommittedSnapshot( - int64_t committed_end_offset, - const std::vector>& active_data_files, + Result PrepareCommittedSnapshot( + const std::vector>& active_data_files) const; + + Result> InstallPreparedSnapshot( + int64_t committed_end_offset, const PreparedSnapshot& prepared, const std::shared_ptr& store); private: diff --git a/src/paimon/core/realtime/realtime_deduplicate_state_test.cpp b/src/paimon/core/realtime/realtime_deduplicate_state_test.cpp index d15a179d0..9f7ddc9a0 100644 --- a/src/paimon/core/realtime/realtime_deduplicate_state_test.cpp +++ b/src/paimon/core/realtime/realtime_deduplicate_state_test.cpp @@ -67,16 +67,16 @@ TEST(RealtimeDeduplicateStateTest, TestAttachAndInstallSnapshot) { ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({{"file-index.bitmap.columns", SpecialFields::RealtimeOffset().Name()}})); - ASSERT_OK_AND_ASSIGN( - std::shared_ptr initial_lookup, - RealtimeOffsetFileIndexLookup::Create( - schema, /*data_schema_id=*/0, schema->GetFieldByName("id"), - /*data_files=*/{}, path_factory, directory->GetFileSystem(), pool, options)); - ASSERT_OK_AND_ASSIGN( - std::shared_ptr ignored_lookup, - RealtimeOffsetFileIndexLookup::Create( - schema, /*data_schema_id=*/0, schema->GetFieldByName("id"), - /*data_files=*/{}, path_factory, directory->GetFileSystem(), pool, options)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr initial_lookup, + RealtimeOffsetFileIndexLookup::Create( + schema, /*data_schema_id=*/0, schema->GetFieldByName("id"), + /*data_files=*/{}, path_factory, directory->GetFileSystem(), options, + /*schema_manager=*/nullptr, pool)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr ignored_lookup, + RealtimeOffsetFileIndexLookup::Create( + schema, /*data_schema_id=*/0, schema->GetFieldByName("id"), + /*data_files=*/{}, path_factory, directory->GetFileSystem(), options, + /*schema_manager=*/nullptr, pool)); RealtimeDeduplicateState state; ASSERT_EQ(nullptr, state.committed_file_lookup_); @@ -113,16 +113,25 @@ TEST(RealtimeDeduplicateStateTest, TestAttachAndInstallSnapshot) { sidecar_writer->Finish(path_factory->ToPath(first_file), first_file->row_count)); first_file = first_file->CopyWithExtraFiles({sidecar}); - ASSERT_NOK_WITH_MSG(state.InstallCommittedSnapshot(-1, {}, store), - "invalid deduplicate committed advancement"); - ASSERT_NOK_WITH_MSG(state.InstallCommittedSnapshot(1, {}, delegate), - "requires ArrowDeduplicateRealtimeStore"); ASSERT_OK_AND_ASSIGN(std::shared_ptr old_keys, store->AcquireKeyOffsetLookup()); ASSERT_OK_AND_ASSIGN(RoaringBitmap64 old_offsets, old_keys->LookupOffsets(keys)); ASSERT_TRUE(old_offsets.IsEmpty()); - ASSERT_OK(state.InstallCommittedSnapshot(/*committed_end_offset=*/1, {first_file}, store)); + ASSERT_OK_AND_ASSIGN(RealtimeDeduplicateState::PreparedSnapshot prepared, + state.PrepareCommittedSnapshot({first_file})); + ASSERT_NOK_WITH_MSG(state.InstallPreparedSnapshot(-1, prepared, store), + "invalid deduplicate committed advancement"); + ASSERT_NOK_WITH_MSG(state.InstallPreparedSnapshot(1, prepared, delegate), + "requires ArrowDeduplicateRealtimeStore"); + ASSERT_EQ(initial_lookup, state.committed_file_lookup_); + ASSERT_OK_AND_ASSIGN(std::shared_ptr before_install, + store->AcquireKeyOffsetLookup()); + ASSERT_OK_AND_ASSIGN(old_offsets, before_install->LookupOffsets(keys)); + ASSERT_TRUE(old_offsets.IsEmpty()); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr retired, + state.InstallPreparedSnapshot(/*committed_end_offset=*/1, prepared, store)); const std::shared_ptr installed = state.committed_file_lookup_; ASSERT_NE(initial_lookup, installed); ASSERT_EQ(initial_lookup->cache_manager_, installed->cache_manager_); @@ -134,9 +143,15 @@ TEST(RealtimeDeduplicateStateTest, TestAttachAndInstallSnapshot) { ASSERT_EQ("{0}", new_offsets.ToString()); ASSERT_OK_AND_ASSIGN(old_offsets, old_keys->LookupOffsets(keys)); ASSERT_TRUE(old_offsets.IsEmpty()); + ASSERT_NOK_WITH_MSG(state.InstallPreparedSnapshot(1, prepared, store), + "committed lookup changed during refresh preparation"); + ASSERT_NOK_WITH_MSG(state.PrepareCommittedSnapshot({first_file->CopyWithExtraFiles({})}), + "required .offset sidecar"); + ASSERT_EQ(installed, state.committed_file_lookup_); // A new snapshot may replace its active file set without advancing the durable offset. - ASSERT_OK(state.InstallCommittedSnapshot(/*committed_end_offset=*/1, {}, store)); + ASSERT_OK_AND_ASSIGN(prepared, state.PrepareCommittedSnapshot({})); + ASSERT_OK(state.InstallPreparedSnapshot(/*committed_end_offset=*/1, prepared, store)); ASSERT_TRUE(state.committed_file_lookup_->DataFiles().empty()); ASSERT_EQ(installed->cache_manager_, state.committed_file_lookup_->cache_manager_); ASSERT_EQ(std::vector>{first_file}, installed->DataFiles()); diff --git a/src/paimon/core/realtime/realtime_deduplicate_writer.cpp b/src/paimon/core/realtime/realtime_deduplicate_writer.cpp index 15eb93355..e1ae88b1c 100644 --- a/src/paimon/core/realtime/realtime_deduplicate_writer.cpp +++ b/src/paimon/core/realtime/realtime_deduplicate_writer.cpp @@ -67,7 +67,8 @@ Result> RealtimeDeduplicateWriter::Cr const std::vector& deduplicate_key_fields, const std::vector>& restored_data_files, const std::shared_ptr& data_file_path_factory, const CoreOptions& options, - const std::string& temp_directory, const std::shared_ptr& memory_pool) { + const std::string& temp_directory, const std::shared_ptr& schema_manager, + const std::shared_ptr& memory_pool) { if (!realtime_context || !file_writer || !schema_layout || !data_file_path_factory || !memory_pool) { return Status::Invalid("deduplicate real-time writer is missing a dependency"); @@ -103,7 +104,7 @@ Result> RealtimeDeduplicateWriter::Cr RealtimeOffsetFileIndexLookup::Create( schema_layout->CommitSchema(), schema_id, schema_layout->StoreWriteSchema()->field(key_position), restored_data_files, - data_file_path_factory, options.GetFileSystem(), memory_pool, options)); + data_file_path_factory, options.GetFileSystem(), options, schema_manager, memory_pool)); PAIMON_RETURN_NOT_OK( store_state.deduplicate_state->AttachFileIndexLookup(created_lookup, store)); return std::shared_ptr(new RealtimeDeduplicateWriter( @@ -169,9 +170,10 @@ Status RealtimeDeduplicateWriter::Write(std::unique_ptr&& batch) { return Status::OK(); } -Status RealtimeDeduplicateWriter::SealCurrentSegment() { - PAIMON_ASSIGN_OR_RAISE(std::optional> segment, - realtime_store_->SealForCommit()); +Status RealtimeDeduplicateWriter::SealCurrentSegment(bool spill) { + PAIMON_ASSIGN_OR_RAISE( + std::optional> segment, + spill ? realtime_store_->SealForCommit() : realtime_store_->FreezeBuildingSegment()); std::lock_guard lock(realtime_store_mutex_); if (!segment) { return Status::OK(); @@ -186,12 +188,12 @@ Status RealtimeDeduplicateWriter::SealCurrentSegment() { Status RealtimeDeduplicateWriter::Seal() { std::lock_guard lock(prepare_mutex_); - return SealCurrentSegment(); + return SealCurrentSegment(/*spill=*/true); } Result RealtimeDeduplicateWriter::PrepareCommit(bool wait_compaction) { std::lock_guard prepare_lock(prepare_mutex_); - PAIMON_RETURN_NOT_OK(SealCurrentSegment()); + PAIMON_RETURN_NOT_OK(SealCurrentSegment(/*spill=*/false)); std::vector> segments; { std::lock_guard store_lock(realtime_store_mutex_); diff --git a/src/paimon/core/realtime/realtime_deduplicate_writer.h b/src/paimon/core/realtime/realtime_deduplicate_writer.h index 90d2551b4..e602404dd 100644 --- a/src/paimon/core/realtime/realtime_deduplicate_writer.h +++ b/src/paimon/core/realtime/realtime_deduplicate_writer.h @@ -41,10 +41,10 @@ class MemoryPool; class RealtimeContext; class RealtimeSchemaLayout; class RealtimeSegmentHandle; +class SchemaManager; struct DataFileMeta; /// Writer for an append table whose user-defined deduplicate key uses last-write-wins semantics. -/// TODO(xinyu.lxy): Currently only the happy path is supported. class RealtimeDeduplicateWriter final : public BatchWriter { public: static Result> Create( @@ -56,6 +56,7 @@ class RealtimeDeduplicateWriter final : public BatchWriter { const std::vector>& restored_data_files, const std::shared_ptr& data_file_path_factory, const CoreOptions& options, const std::string& temp_directory, + const std::shared_ptr& schema_manager, const std::shared_ptr& memory_pool); Status Write(std::unique_ptr&& batch) override; @@ -77,7 +78,7 @@ class RealtimeDeduplicateWriter final : public BatchWriter { std::shared_ptr deduplicate_key_field, int64_t next_offset, std::shared_ptr memory_pool); - Status SealCurrentSegment(); + Status SealCurrentSegment(bool spill); Status FlushSegment(const std::shared_ptr& segment); std::shared_ptr arrow_pool_; diff --git a/src/paimon/core/realtime/realtime_file_schema.cpp b/src/paimon/core/realtime/realtime_file_schema.cpp new file mode 100644 index 000000000..9748f4b37 --- /dev/null +++ b/src/paimon/core/realtime/realtime_file_schema.cpp @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/realtime/realtime_file_schema.h" + +#include +#include +#include + +#include "arrow/api.h" +#include "paimon/common/table/special_fields.h" +#include "paimon/common/types/data_field.h" +#include "paimon/core/core_options.h" +#include "paimon/core/io/data_file_meta.h" +#include "paimon/core/realtime/realtime_utils.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/utils/roaring_bitmap64.h" + +namespace paimon { + +RealtimeFileSchema::RealtimeFileSchema(const std::shared_ptr& schema, + int64_t schema_id, + const std::shared_ptr& schema_manager) + : schema_(schema), schema_id_(schema_id), schema_manager_(schema_manager) {} + +Result> RealtimeFileSchema::GetFileSchema( + const DataFileMeta& file) const { + PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateOffsetField(schema_)); + if (file.schema_id == schema_id_) { + return schema_; + } + if (!schema_manager_) { + return Status::Invalid("historical realtime file schema requires a schema manager"); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr file_schema, + schema_manager_->ReadSchema(file.schema_id)); + std::shared_ptr schema = + DataField::ConvertDataFieldsToArrowSchema(file_schema->Fields()); + PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateOffsetField(schema)); + const std::string& offset_name = SpecialFields::RealtimeOffset().Name(); + PAIMON_ASSIGN_OR_RAISE(DataField current_offset, DataField::ConvertArrowFieldToDataField( + schema_->GetFieldByName(offset_name))); + PAIMON_ASSIGN_OR_RAISE(DataField file_offset, file_schema->GetField(offset_name)); + if (current_offset.Id() != file_offset.Id()) { + return Status::Invalid("historical realtime offset field ID is incompatible"); + } + return schema; +} + +Status RealtimeFileSchema::ValidateKey(const DataFileMeta& file, + const std::shared_ptr& key_field) const { + if (file.schema_id == schema_id_) { + return Status::OK(); + } + PAIMON_RETURN_NOT_OK(GetFileSchema(file).status()); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr file_schema, + schema_manager_->ReadSchema(file.schema_id)); + PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(file_schema->Options())); + const std::vector& key_fields = options.GetRealtimeDeduplicateKeyFields(); + if (key_fields.size() != 1 || key_fields.front().empty()) { + return Status::Invalid( + "historical file schema must define exactly one realtime deduplicate key field"); + } + PAIMON_ASSIGN_OR_RAISE(DataField file_key, file_schema->GetField(key_fields.front())); + PAIMON_ASSIGN_OR_RAISE(DataField current_key, + DataField::ConvertArrowFieldToDataField(key_field)); + if (file_key.Id() != current_key.Id() || + !file_key.Type()->Equals(current_key.Type(), /*check_metadata=*/true) || + file_key.ArrowField()->nullable() || current_key.ArrowField()->nullable()) { + return Status::Invalid("historical realtime deduplicate key is incompatible"); + } + return Status::OK(); +} + +Result> RealtimeFileSchema::SelectOffsets( + const DataFileMeta& file, const RoaringBitmap64& offsets) const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, GetFileSchema(file)); + const std::string& offset_name = SpecialFields::RealtimeOffset().Name(); + PAIMON_ASSIGN_OR_RAISE(std::vector fields, + DataField::ConvertArrowSchemaToDataFields(schema)); + PAIMON_ASSIGN_OR_RAISE(std::vector write_fields, + DataField::ProjectFields(fields, file.write_cols)); + PAIMON_ASSIGN_OR_RAISE(std::vector stats_fields, + DataField::ProjectFields(write_fields, file.value_stats_cols)); + std::optional position; + auto iter = std::find_if( + stats_fields.begin(), stats_fields.end(), + [&offset_name](const DataField& field) { return field.Name() == offset_name; }); + if (iter != stats_fields.end()) { + position = static_cast(iter - stats_fields.begin()); + } + const BinaryRow& min_values = file.value_stats.MinValues(); + const BinaryRow& max_values = file.value_stats.MaxValues(); + const bool has_stats = position && *position >= 0 && *position < min_values.GetFieldCount() && + *position < max_values.GetFieldCount() && + !min_values.IsNullAt(*position) && !max_values.IsNullAt(*position) && + min_values.GetLong(*position) >= 0 && + min_values.GetLong(*position) <= max_values.GetLong(*position); + std::vector literals; + if (has_stats) { + for (RoaringBitmap64::Iterator iter = offsets.EqualOrLarger(min_values.GetLong(*position)); + iter != offsets.End() && *iter <= max_values.GetLong(*position); ++iter) { + literals.emplace_back(*iter); + } + } else { + literals.reserve(offsets.Cardinality()); + for (RoaringBitmap64::Iterator iter = offsets.Begin(); iter != offsets.End(); ++iter) { + literals.emplace_back(*iter); + } + } + return literals; +} + +} // namespace paimon diff --git a/src/paimon/core/realtime/realtime_file_schema.h b/src/paimon/core/realtime/realtime_file_schema.h new file mode 100644 index 000000000..d49b9e527 --- /dev/null +++ b/src/paimon/core/realtime/realtime_file_schema.h @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include + +#include "arrow/type_fwd.h" +#include "paimon/predicate/literal.h" +#include "paimon/result.h" + +namespace paimon { +class SchemaManager; +class RoaringBitmap64; +struct DataFileMeta; + +/// Resolves immutable file schemas and offset statistics without interpreting old columns using +/// the current table layout. SchemaManager provides the shared, thread-safe schema cache. +class RealtimeFileSchema { + public: + RealtimeFileSchema(const std::shared_ptr& schema, int64_t schema_id, + const std::shared_ptr& schema_manager); + + Result> GetFileSchema(const DataFileMeta& file) const; + + /// Verifies the historical deduplicate definition and key encoding before opening a sidecar. + Status ValidateKey(const DataFileMeta& file, + const std::shared_ptr& key_field) const; + + /// Returns offsets inside the file's inclusive min/max range. Missing or unusable statistics + /// conservatively return all offsets; missing or incompatible schemas fail the read. + Result> SelectOffsets(const DataFileMeta& file, + const RoaringBitmap64& offsets) const; + + private: + std::shared_ptr schema_; + int64_t schema_id_; + std::shared_ptr schema_manager_; +}; +} // namespace paimon diff --git a/src/paimon/core/realtime/realtime_file_schema_test.cpp b/src/paimon/core/realtime/realtime_file_schema_test.cpp new file mode 100644 index 000000000..3f10f294f --- /dev/null +++ b/src/paimon/core/realtime/realtime_file_schema_test.cpp @@ -0,0 +1,207 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/realtime/realtime_file_schema.h" + +#include +#include +#include +#include +#include +#include + +#include "arrow/api.h" +#include "paimon/common/table/special_fields.h" +#include "paimon/common/types/data_field.h" +#include "paimon/core/io/data_file_meta.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/defs.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/testing/utils/binary_row_generator.h" +#include "paimon/testing/utils/testharness.h" +#include "paimon/utils/roaring_bitmap64.h" + +namespace paimon::test { + +class RealtimeFileSchemaTest : public testing::Test { + public: + void SetUp() override { + directory_ = UniqueTestDirectory::Create(); + ASSERT_NE(nullptr, directory_); + schema_ = DataField::ConvertDataFieldsToArrowSchema( + {DataField(0, SpecialFields::RealtimeOffset().ArrowField()), + DataField(1, arrow::field("id", arrow::int64(), /*nullable=*/false))}); + schema_manager_ = + std::make_shared(directory_->GetFileSystem(), directory_->Str()); + } + + protected: + std::shared_ptr MakeDataFile() const { + return std::make_shared( + "data-0.parquet", /*file_size=*/0, /*row_count=*/2, DataFileMeta::EmptyMinKey(), + DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), + /*min_sequence_number=*/0, /*max_sequence_number=*/0, /*schema_id=*/0, + DataFileMeta::DUMMY_LEVEL, /*extra_files=*/std::vector>(), + Timestamp(0, 0), /*delete_row_count=*/std::nullopt, /*embedded_index=*/nullptr, + /*file_source=*/std::nullopt, /*value_stats_cols=*/std::nullopt, + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); + } + + std::unique_ptr directory_; + std::shared_ptr schema_; + std::shared_ptr schema_manager_; +}; + +TEST_F(RealtimeFileSchemaTest, TestOffsetStatsBoundsAndConservativeFallback) { + std::shared_ptr file = MakeDataFile(); + file->value_stats = BinaryRowGenerator::GenerateStats({10}, {20}, {0}, GetDefaultPool().get()); + file->value_stats_cols = std::vector{"_REALTIME_OFFSET"}; + RealtimeFileSchema file_schema(schema_, 0, schema_manager_); + RoaringBitmap64 offsets; + for (int64_t offset : {9, 10, 20, 21}) { + offsets.Add(offset); + } + ASSERT_OK_AND_ASSIGN(std::vector selected, file_schema.SelectOffsets(*file, offsets)); + ASSERT_EQ(std::vector({Literal(int64_t{10}), Literal(int64_t{20})}), selected); + + file->value_stats_cols = std::vector{"id"}; + ASSERT_OK_AND_ASSIGN(selected, file_schema.SelectOffsets(*file, offsets)); + ASSERT_EQ(4, selected.size()); + file->value_stats_cols = std::vector{"_REALTIME_OFFSET"}; + file->value_stats = BinaryRowGenerator::GenerateStats({20}, {10}, {0}, GetDefaultPool().get()); + ASSERT_OK_AND_ASSIGN(selected, file_schema.SelectOffsets(*file, offsets)); + ASSERT_EQ(4, selected.size()); + file->value_stats = SimpleStats::EmptyStats(); + ASSERT_OK_AND_ASSIGN(selected, file_schema.SelectOffsets(*file, offsets)); + ASSERT_EQ(4, selected.size()); + file->value_stats = BinaryRowGenerator::GenerateStats({10}, {20}, {0}, GetDefaultPool().get()); + file->value_stats_cols.reset(); + file->write_cols = std::vector{"id"}; + ASSERT_OK_AND_ASSIGN(selected, file_schema.SelectOffsets(*file, offsets)); + ASSERT_EQ(4, selected.size()); +} + +TEST_F(RealtimeFileSchemaTest, TestWriteColumnsStatsProjection) { + const auto payload = DataField::ConvertDataFieldToArrowField( + DataField(2, arrow::field("payload", arrow::int64()))); + const auto schema = arrow::schema({schema_->field(0), schema_->field(1), payload}); + RealtimeFileSchema file_schema(schema, 0, schema_manager_); + std::shared_ptr file = MakeDataFile(); + const RoaringBitmap64 offsets = RoaringBitmap64::From({9, 10, 20, 21, 100, 200}); + const std::vector expected = {Literal(int64_t{10}), Literal(int64_t{20})}; + + file->write_cols = std::vector{"payload", "_REALTIME_OFFSET"}; + file->value_stats = + BinaryRowGenerator::GenerateStats({100, 10}, {200, 20}, {0, 0}, GetDefaultPool().get()); + ASSERT_OK_AND_ASSIGN(std::vector selected, file_schema.SelectOffsets(*file, offsets)); + ASSERT_EQ(expected, selected); + + file->value_stats_cols = std::vector{"_REALTIME_OFFSET", "payload"}; + file->value_stats = + BinaryRowGenerator::GenerateStats({10, 100}, {20, 200}, {0, 0}, GetDefaultPool().get()); + ASSERT_OK_AND_ASSIGN(selected, file_schema.SelectOffsets(*file, offsets)); + ASSERT_EQ(expected, selected); + + file->value_stats_cols.reset(); + file->write_cols = std::vector{"_REALTIME_OFFSET"}; + file->value_stats = BinaryRowGenerator::GenerateStats({10}, {20}, {0}, GetDefaultPool().get()); + ASSERT_OK_AND_ASSIGN(selected, file_schema.SelectOffsets(*file, offsets)); + ASSERT_EQ(expected, selected); +} + +TEST_F(RealtimeFileSchemaTest, TestWriteColumnsMissingOffsetStatsFallback) { + RealtimeFileSchema file_schema(schema_, 0, schema_manager_); + std::shared_ptr file = MakeDataFile(); + const RoaringBitmap64 offsets = RoaringBitmap64::From({9, 10, 20, 21}); + const std::vector expected = {Literal(int64_t{9}), Literal(int64_t{10}), + Literal(int64_t{20}), Literal(int64_t{21})}; + file->write_cols = std::vector{"id", "_REALTIME_OFFSET"}; + file->value_stats_cols = std::vector{"id"}; + file->value_stats = BinaryRowGenerator::GenerateStats({10}, {20}, {0}, GetDefaultPool().get()); + ASSERT_OK_AND_ASSIGN(std::vector selected, file_schema.SelectOffsets(*file, offsets)); + ASSERT_EQ(expected, selected); + + file->value_stats_cols.reset(); + ASSERT_OK_AND_ASSIGN(selected, file_schema.SelectOffsets(*file, offsets)); + ASSERT_EQ(expected, selected); + + file->write_cols = std::vector{"id"}; + ASSERT_OK_AND_ASSIGN(selected, file_schema.SelectOffsets(*file, offsets)); + ASSERT_EQ(expected, selected); +} + +TEST_F(RealtimeFileSchemaTest, TestHistoricalKeyRenameAndFieldReorder) { + const std::map options = { + {Options::REALTIME_DEDUPLICATE_KEY_FIELDS, "id"}}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr historical_schema, + TableSchema::Create(0, schema_, {}, {}, options)); + ASSERT_OK(directory_->GetFileSystem()->Mkdirs(schema_manager_->SchemaDirectory())); + ASSERT_OK_AND_ASSIGN(std::string content, historical_schema->ToJsonString()); + ASSERT_OK(directory_->GetFileSystem()->AtomicStore( + schema_manager_->SchemaDirectory() + "/schema-0", content)); + + const auto historical_arrow_schema = + DataField::ConvertDataFieldsToArrowSchema(historical_schema->Fields()); + const auto renamed_key = historical_arrow_schema->field(1)->WithName("renamed_id"); + // Swap key and offset positions without changing their field IDs or the offset name. + const auto current_schema = arrow::schema({renamed_key, historical_arrow_schema->field(0)}); + RealtimeFileSchema file_schema(current_schema, /*schema_id=*/1, schema_manager_); + std::shared_ptr file = MakeDataFile(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr resolved_schema, + file_schema.GetFileSchema(*file)); + ASSERT_TRUE(resolved_schema->Equals(*historical_arrow_schema, /*check_metadata=*/true)); + ASSERT_OK(file_schema.ValidateKey(*file, renamed_key)); + + // Historical statistics are [offset, id], not the current [renamed_id, offset]. + file->value_stats = + BinaryRowGenerator::GenerateStats({10, 100}, {20, 200}, {0, 0}, GetDefaultPool().get()); + RoaringBitmap64 offsets; + for (int64_t offset : {9, 10, 20, 21, 100, 200}) { + offsets.Add(offset); + } + ASSERT_OK_AND_ASSIGN(std::vector selected, file_schema.SelectOffsets(*file, offsets)); + const std::vector expected = {Literal(int64_t{10}), Literal(int64_t{20})}; + ASSERT_EQ(expected, selected); + + // Explicit statistics column names take precedence over the historical schema order. + file->value_stats_cols = std::vector{"id", "_REALTIME_OFFSET"}; + file->value_stats = + BinaryRowGenerator::GenerateStats({100, 10}, {200, 20}, {0, 0}, GetDefaultPool().get()); + ASSERT_OK_AND_ASSIGN(selected, file_schema.SelectOffsets(*file, offsets)); + ASSERT_EQ(expected, selected); + + file->value_stats_cols.reset(); + file->write_cols = std::vector{"id", "_REALTIME_OFFSET"}; + ASSERT_OK_AND_ASSIGN(selected, file_schema.SelectOffsets(*file, offsets)); + ASSERT_EQ(expected, selected); +} + +TEST_F(RealtimeFileSchemaTest, TestHistoricalSchemaMustBeAvailable) { + std::shared_ptr file = MakeDataFile(); + RoaringBitmap64 offsets; + offsets.Add(10); + RealtimeFileSchema without_manager(schema_, /*schema_id=*/1, nullptr); + ASSERT_NOK_WITH_MSG(without_manager.SelectOffsets(*file, offsets), "requires a schema manager"); + RealtimeFileSchema missing_schema(schema_, /*schema_id=*/1, schema_manager_); + ASSERT_NOK(missing_schema.SelectOffsets(*file, offsets)); +} + +} // namespace paimon::test diff --git a/src/paimon/core/realtime/realtime_key_offset_index.cpp b/src/paimon/core/realtime/realtime_key_offset_index.cpp index 1a4a347b4..18fbff269 100644 --- a/src/paimon/core/realtime/realtime_key_offset_index.cpp +++ b/src/paimon/core/realtime/realtime_key_offset_index.cpp @@ -44,7 +44,7 @@ Result> MutableKeyOffsetIndex::Create( PAIMON_ASSIGN_OR_RAISE(std::shared_ptr serializer, KeySerializer::Create(key_field->type(), memory_pool)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr offsets, - OffsetMap::Create(/*initial_capacity=*/1024, memory_pool)); + OffsetMap::Create(kInitialCapacity, memory_pool)); return std::shared_ptr(new MutableKeyOffsetIndex( key_field, std::move(serializer), std::move(offsets), memory_pool)); } @@ -69,7 +69,9 @@ Status MutableKeyOffsetIndex::EnsureMutable() { } PAIMON_ASSIGN_OR_RAISE( std::shared_ptr copy, - OffsetMap::Create(std::max(1, offsets_->Size()), memory_pool_)); + OffsetMap::Create( + std::max(kInitialCapacity, offsets_->Size() * kCopyCapacityMultiplier), + memory_pool_)); Status status = Status::OK(); offsets_->ForEach([&](uint64_t hash, int64_t offset) { if (status.ok() && offset >= 0) { @@ -123,11 +125,15 @@ Result MutableKeyOffsetIndex::LookupOffsets( return result; } -std::shared_ptr MutableKeyOffsetIndex::Snapshot() const { - return std::shared_ptr( +std::shared_ptr MutableKeyOffsetIndex::Snapshot() const { + return std::shared_ptr( new MutableKeyOffsetIndex(key_field_, key_serializer_, offsets_, memory_pool_)); } +uint64_t MutableKeyOffsetIndex::GetMemoryUsage() const { + return offsets_->MemoryUsage(); +} + Result CompositeKeyOffsetLookup::LookupOffsets( const std::shared_ptr& keys) const { RoaringBitmap64 result; diff --git a/src/paimon/core/realtime/realtime_key_offset_index.h b/src/paimon/core/realtime/realtime_key_offset_index.h index fe799d8fe..2010d06dc 100644 --- a/src/paimon/core/realtime/realtime_key_offset_index.h +++ b/src/paimon/core/realtime/realtime_key_offset_index.h @@ -65,11 +65,17 @@ class MutableKeyOffsetIndex final : public KeyOffsetLookup { Result LookupOffsets( const std::shared_ptr& keys) const override; - std::shared_ptr Snapshot() const; + std::shared_ptr Snapshot() const; + + /// Bucket and occupancy-bitmap bytes retained by this index, excluding older COW snapshots. + uint64_t GetMemoryUsage() const; private: using OffsetMap = HashMap; + static constexpr uint64_t kInitialCapacity = 1024; + static constexpr uint64_t kCopyCapacityMultiplier = 2; + MutableKeyOffsetIndex(std::shared_ptr key_field, std::shared_ptr key_serializer, std::shared_ptr offsets, diff --git a/src/paimon/core/realtime/realtime_key_offset_index_test.cpp b/src/paimon/core/realtime/realtime_key_offset_index_test.cpp index 5275578f8..3a810c476 100644 --- a/src/paimon/core/realtime/realtime_key_offset_index_test.cpp +++ b/src/paimon/core/realtime/realtime_key_offset_index_test.cpp @@ -115,6 +115,24 @@ TEST(RealtimeKeyOffsetIndexTest, TestMutableIndexSnapshotIsCopyOnWrite) { ASSERT_TRUE(pinned.IsEmpty()); } +TEST(RealtimeKeyOffsetIndexTest, TestMutableIndexMemoryUsageExcludesOldSnapshots) { + const auto key_field = arrow::field("id", arrow::int64()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr index, + MutableKeyOffsetIndex::Create(key_field, GetDefaultPool())); + const uint64_t initial_bytes = index->GetMemoryUsage(); + ASSERT_GT(initial_bytes, 0); + ASSERT_OK(index->Add(MakeKeys(key_field, R"([[1], [2]])"), MakeOffsets("[10, 20]"))); + const auto snapshot = index->Snapshot(); + ASSERT_EQ(initial_bytes, snapshot->GetMemoryUsage()); + ASSERT_OK(index->Add(MakeKeys(key_field, R"([[1]])"), MakeOffsets("[30]"))); + ASSERT_EQ(initial_bytes, snapshot->GetMemoryUsage()); + // COW allocates a fresh map sized for the live keys, not an extra charge for the old map. + ASSERT_GT(index->GetMemoryUsage(), 0); + ASSERT_OK_AND_ASSIGN(RoaringBitmap64 pinned, + snapshot->LookupOffsets(MakeKeys(key_field, R"([[1]])"))); + AssertOffsets(pinned, {10}); +} + TEST(RealtimeKeyOffsetIndexTest, TestMutableIndexValidatesInput) { const std::shared_ptr key_field = arrow::field("id", arrow::int64()); ASSERT_NOK_WITH_MSG(MutableKeyOffsetIndex::Create(nullptr, GetDefaultPool()), diff --git a/src/paimon/core/realtime/realtime_offset_batch_reader.cpp b/src/paimon/core/realtime/realtime_offset_batch_reader.cpp index 0237d3ec0..99a388915 100644 --- a/src/paimon/core/realtime/realtime_offset_batch_reader.cpp +++ b/src/paimon/core/realtime/realtime_offset_batch_reader.cpp @@ -85,10 +85,6 @@ Result RealtimeOffsetBatchReader::NextBatchWit output_bitmap.Add(row); } } - if (static_cast(input_bitmap.Cardinality()) != offsets->length()) { - return Status::Invalid( - "real-time store reader bitmap must cover every raw transport row"); - } if (output_bitmap.IsEmpty()) { continue; } diff --git a/src/paimon/core/realtime/realtime_offset_batch_reader_test.cpp b/src/paimon/core/realtime/realtime_offset_batch_reader_test.cpp index d8d687d26..2b353b744 100644 --- a/src/paimon/core/realtime/realtime_offset_batch_reader_test.cpp +++ b/src/paimon/core/realtime/realtime_offset_batch_reader_test.cpp @@ -68,18 +68,24 @@ TEST(RealtimeOffsetBatchReaderTest, TestFilterBitmapAndRemoveOffset) { << "expected: " << expected->ToString() << "\nactual: " << result->ToString(); } -TEST(RealtimeOffsetBatchReaderTest, TestRejectsPartialBitmap) { +TEST(RealtimeOffsetBatchReaderTest, TestIntersectsPartialBitmapWithVisibleRange) { std::shared_ptr type = MakeDataType(); - std::shared_ptr data = MakeArray(type, R"([[10, 0], [11, 1]])"); + std::shared_ptr data = + MakeArray(type, R"([[10, 0], [11, 1], [12, 2], [13, 3], [14, 4]])"); RoaringBitmap32 input_bitmap; input_bitmap.Add(0); + input_bitmap.Add(2); + input_bitmap.Add(4); auto input = std::make_unique(data, type, input_bitmap, /*read_batch_size=*/2); input->EnableRandomizeBatchSize(false); - RealtimeOffsetBatchReader reader(std::move(input), OffsetRange(0, 2), /*keep_offset=*/false); + RealtimeOffsetBatchReader reader(std::move(input), OffsetRange(1, 4), /*keep_offset=*/false); - ASSERT_NOK_WITH_MSG(ReadResultCollector::CollectResult(&reader), - "must cover every raw transport row"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, + ReadResultCollector::CollectResult(&reader)); + std::shared_ptr expected = + MakeArray(arrow::struct_({arrow::field("id", arrow::int32())}), R"([[12]])"); + ASSERT_TRUE(result->Equals(arrow::ChunkedArray(expected))) << result->ToString(); } TEST(RealtimeOffsetBatchReaderTest, TestFilterWithoutInputBitmap) { diff --git a/src/paimon/core/realtime/realtime_offset_file_index_lookup.cpp b/src/paimon/core/realtime/realtime_offset_file_index_lookup.cpp index 82d02845a..82eef3a23 100644 --- a/src/paimon/core/realtime/realtime_offset_file_index_lookup.cpp +++ b/src/paimon/core/realtime/realtime_offset_file_index_lookup.cpp @@ -18,8 +18,6 @@ #include "paimon/core/realtime/realtime_offset_file_index_lookup.h" -#include -#include #include #include "arrow/api.h" @@ -30,7 +28,9 @@ #include "paimon/core/io/data_file_meta.h" #include "paimon/core/io/data_file_path_factory.h" #include "paimon/core/io/file_index_evaluator.h" +#include "paimon/core/realtime/realtime_file_schema.h" #include "paimon/core/realtime/realtime_key_offset_index.h" +#include "paimon/core/realtime/realtime_utils.h" #include "paimon/file_index/bitmap_index_result.h" #include "paimon/fs/file_system.h" #include "paimon/predicate/literal.h" @@ -43,14 +43,14 @@ Result> RealtimeOffsetFileIndexLo const std::shared_ptr& deduplicate_key_field, const std::vector>& data_files, const std::shared_ptr& path_factory, - const std::shared_ptr& file_system, const std::shared_ptr& memory_pool, - const CoreOptions& options) { + const std::shared_ptr& file_system, const CoreOptions& options, + const std::shared_ptr& schema_manager, + const std::shared_ptr& memory_pool) { if (!data_schema || data_schema_id < 0 || !deduplicate_key_field || !path_factory || !file_system || !memory_pool) { return Status::Invalid("realtime offset file-index lookup is missing a dependency"); } - // TODO(xinyu.lxy): Bitmap is the mandatory first implementation. Select a more suitable - // high-cardinality offset index (for example BSI) after its exact row-position API is ready. + PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateOffsetField(data_schema)); for (const std::shared_ptr& file : data_files) { if (!file || file->row_count < 0 || file->row_count > RoaringBitmap32::MAX_VALUE) { return Status::Invalid("invalid data file for realtime offset lookup"); @@ -62,28 +62,28 @@ Result> RealtimeOffsetFileIndexLo auto cache_manager = std::make_shared(options.GetLookupCacheMaxMemory(), options.GetLookupCacheHighPrioPoolRatio()); return std::shared_ptr(new RealtimeOffsetFileIndexLookup( - data_schema, data_schema_id, deduplicate_key_field, - std::vector>(data_files), path_factory, file_system, - options.ToMap(), options.GetCachePageSize(), std::move(cache_manager), memory_pool)); + deduplicate_key_field, std::vector>(data_files), path_factory, + file_system, options.ToMap(), options.GetCachePageSize(), std::move(cache_manager), + std::make_shared(data_schema, data_schema_id, schema_manager), + memory_pool)); } RealtimeOffsetFileIndexLookup::RealtimeOffsetFileIndexLookup( - std::shared_ptr data_schema, int64_t data_schema_id, std::shared_ptr deduplicate_key_field, std::vector> data_files, std::shared_ptr path_factory, std::shared_ptr file_system, std::map options, uint32_t cache_page_size, - std::shared_ptr cache_manager, std::shared_ptr memory_pool) - : data_schema_(std::move(data_schema)), - data_schema_id_(data_schema_id), - deduplicate_key_field_(std::move(deduplicate_key_field)), + std::shared_ptr cache_manager, std::shared_ptr file_schema, + std::shared_ptr memory_pool) + : deduplicate_key_field_(std::move(deduplicate_key_field)), data_files_(std::move(data_files)), path_factory_(std::move(path_factory)), file_system_(std::move(file_system)), memory_pool_(std::move(memory_pool)), options_(std::move(options)), cache_page_size_(cache_page_size), - cache_manager_(std::move(cache_manager)) {} + cache_manager_(std::move(cache_manager)), + file_schema_(std::move(file_schema)) {} Result RealtimeOffsetFileIndexLookup::LookupFilePositions(const RoaringBitmap64& offsets) const { @@ -92,56 +92,20 @@ RealtimeOffsetFileIndexLookup::LookupFilePositions(const RoaringBitmap64& offset return result; } const std::string& offset_name = SpecialFields::RealtimeOffset().Name(); - const int32_t offset_position = data_schema_->GetFieldIndex(offset_name); for (const std::shared_ptr& file : data_files_) { - std::optional stats_offset_position; - if (file->value_stats_cols) { - const auto iter = std::find(file->value_stats_cols->begin(), - file->value_stats_cols->end(), offset_name); - if (iter != file->value_stats_cols->end()) { - stats_offset_position = - static_cast(iter - file->value_stats_cols->begin()); - } - } else if (file->schema_id == data_schema_id_) { - stats_offset_position = offset_position; - } - // TODO(xinyu.lxy): Resolve `file->schema_id` to its write-time schema before using stats - // from an evolved schema. `write_cols` belongs to data evolution, which real-time - // deduplicate does not support yet. - - const BinaryRow& min_values = file->value_stats.MinValues(); - const BinaryRow& max_values = file->value_stats.MaxValues(); - const bool has_offset_stats = stats_offset_position && *stats_offset_position >= 0 && - *stats_offset_position < min_values.GetFieldCount() && - *stats_offset_position < max_values.GetFieldCount() && - !min_values.IsNullAt(*stats_offset_position) && - !max_values.IsNullAt(*stats_offset_position) && - min_values.GetLong(*stats_offset_position) <= - max_values.GetLong(*stats_offset_position); - - std::vector literals; - if (has_offset_stats) { - const int64_t min_offset = min_values.GetLong(*stats_offset_position); - const int64_t max_offset = max_values.GetLong(*stats_offset_position); - for (RoaringBitmap64::Iterator iter = offsets.EqualOrLarger(min_offset); - iter != offsets.End() && *iter <= max_offset; ++iter) { - literals.emplace_back(*iter); - } - } else { - literals.reserve(offsets.Cardinality()); - for (RoaringBitmap64::Iterator iter = offsets.Begin(); iter != offsets.End(); ++iter) { - literals.emplace_back(*iter); - } - } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, + file_schema_->GetFileSchema(*file)); + PAIMON_ASSIGN_OR_RAISE(std::vector literals, + file_schema_->SelectOffsets(*file, offsets)); if (literals.empty()) { continue; } - std::shared_ptr predicate = - PredicateBuilder::In(offset_position, offset_name, FieldType::BIGINT, literals); + std::shared_ptr predicate = PredicateBuilder::In( + schema->GetFieldIndex(offset_name), offset_name, FieldType::BIGINT, literals); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr index_result, - FileIndexEvaluator::Evaluate(data_schema_, options_, predicate, path_factory_, file, + FileIndexEvaluator::Evaluate(schema, options_, predicate, path_factory_, file, file_system_, memory_pool_)); PAIMON_ASSIGN_OR_RAISE(bool remains, index_result->IsRemain()); if (!remains) { @@ -150,9 +114,9 @@ RealtimeOffsetFileIndexLookup::LookupFilePositions(const RoaringBitmap64& offset std::shared_ptr bitmap = std::dynamic_pointer_cast(index_result); if (!bitmap) { - return Status::Invalid( - fmt::format("_REALTIME_OFFSET index for data file {} is not an exact bitmap index", - file->file_name)); + return Status::Invalid(fmt::format( + "_REALTIME_OFFSET index for data file {} did not return exact row positions", + file->file_name)); } PAIMON_ASSIGN_OR_RAISE(const RoaringBitmap32* positions, bitmap->GetBitmap()); if (!positions->IsEmpty()) { @@ -164,14 +128,21 @@ RealtimeOffsetFileIndexLookup::LookupFilePositions(const RoaringBitmap64& offset Result> RealtimeOffsetFileIndexLookup::CreateKeyOffsetLookup() const { + std::lock_guard lock(key_readers_mutex_); std::vector> readers; readers.reserve(data_files_.size()); for (const std::shared_ptr& file : data_files_) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr reader, - DataFileKeyOffsetIndexReader::Create( - cache_page_size_, cache_manager_, deduplicate_key_field_, file, - path_factory_, file_system_, memory_pool_)); - readers.push_back(std::move(reader)); + const std::string path = path_factory_->ToPath(file); + auto iter = key_readers_.find(path); + if (iter == key_readers_.end()) { + PAIMON_RETURN_NOT_OK(file_schema_->ValidateKey(*file, deduplicate_key_field_)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr reader, + DataFileKeyOffsetIndexReader::Create( + cache_page_size_, cache_manager_, deduplicate_key_field_, + file, path_factory_, file_system_, memory_pool_)); + iter = key_readers_.emplace(path, KeyReader{file, std::move(reader)}).first; + } + readers.push_back(iter->second.reader); } return std::shared_ptr(new CompositeKeyOffsetLookup(std::move(readers))); } @@ -183,10 +154,21 @@ Result> RealtimeOffsetFileIndexLo return Status::Invalid("invalid data file for realtime offset lookup"); } } - return std::shared_ptr(new RealtimeOffsetFileIndexLookup( - data_schema_, data_schema_id_, deduplicate_key_field_, - std::vector>(data_files), path_factory_, file_system_, - options_, cache_page_size_, cache_manager_, memory_pool_)); + auto next = std::shared_ptr(new RealtimeOffsetFileIndexLookup( + deduplicate_key_field_, std::vector>(data_files), + path_factory_, file_system_, options_, cache_page_size_, cache_manager_, file_schema_, + memory_pool_)); + { + std::lock_guard lock(key_readers_mutex_); + for (const std::shared_ptr& file : data_files) { + const std::string path = path_factory_->ToPath(file); + auto iter = key_readers_.find(path); + if (iter != key_readers_.end() && *iter->second.file == *file) { + next->key_readers_.emplace(path, KeyReader{file, iter->second.reader}); + } + } + } + return next; } } // namespace paimon diff --git a/src/paimon/core/realtime/realtime_offset_file_index_lookup.h b/src/paimon/core/realtime/realtime_offset_file_index_lookup.h index c5735bb93..192fb2334 100644 --- a/src/paimon/core/realtime/realtime_offset_file_index_lookup.h +++ b/src/paimon/core/realtime/realtime_offset_file_index_lookup.h @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -38,13 +39,16 @@ namespace paimon { class CacheManager; class CoreOptions; class DataFilePathFactory; +class DataFileKeyOffsetIndexReader; class FileSystem; class KeyOffsetLookup; class MemoryPool; +class RealtimeFileSchema; +class SchemaManager; struct DataFileMeta; /// Uses the configured bitmap file index on `_REALTIME_OFFSET` to translate a pinned offset DV to -/// physical row positions. Instances are immutable and therefore safe to pin for lock-free I/O. +/// physical row positions. File sets are immutable; sidecar readers are reused across snapshots. class RealtimeOffsetFileIndexLookup { public: // Maps data file names to matching physical row positions within a single partition and bucket. @@ -55,8 +59,9 @@ class RealtimeOffsetFileIndexLookup { const std::shared_ptr& deduplicate_key_field, const std::vector>& data_files, const std::shared_ptr& path_factory, - const std::shared_ptr& file_system, - const std::shared_ptr& memory_pool, const CoreOptions& options); + const std::shared_ptr& file_system, const CoreOptions& options, + const std::shared_ptr& schema_manager, + const std::shared_ptr& memory_pool); Result LookupFilePositions(const RoaringBitmap64& offsets) const; @@ -70,16 +75,21 @@ class RealtimeOffsetFileIndexLookup { } private: - RealtimeOffsetFileIndexLookup( - std::shared_ptr data_schema, int64_t data_schema_id, - std::shared_ptr deduplicate_key_field, - std::vector> data_files, - std::shared_ptr path_factory, std::shared_ptr file_system, - std::map options, uint32_t cache_page_size, - std::shared_ptr cache_manager, std::shared_ptr memory_pool); - - std::shared_ptr data_schema_; - int64_t data_schema_id_; + struct KeyReader { + std::shared_ptr file; + std::shared_ptr reader; + }; + + RealtimeOffsetFileIndexLookup(std::shared_ptr deduplicate_key_field, + std::vector> data_files, + std::shared_ptr path_factory, + std::shared_ptr file_system, + std::map options, + uint32_t cache_page_size, + std::shared_ptr cache_manager, + std::shared_ptr file_schema, + std::shared_ptr memory_pool); + std::shared_ptr deduplicate_key_field_; std::vector> data_files_; std::shared_ptr path_factory_; @@ -89,6 +99,9 @@ class RealtimeOffsetFileIndexLookup { uint32_t cache_page_size_; // One budget for all sidecars and all pinned committed snapshots of this bucket. std::shared_ptr cache_manager_; + std::shared_ptr file_schema_; + mutable std::mutex key_readers_mutex_; + mutable std::map key_readers_; }; } // namespace paimon diff --git a/src/paimon/core/realtime/realtime_offset_file_index_lookup_test.cpp b/src/paimon/core/realtime/realtime_offset_file_index_lookup_test.cpp index 5d35450ce..9902ebf5d 100644 --- a/src/paimon/core/realtime/realtime_offset_file_index_lookup_test.cpp +++ b/src/paimon/core/realtime/realtime_offset_file_index_lookup_test.cpp @@ -20,6 +20,7 @@ #include "paimon/core/realtime/realtime_offset_file_index_lookup.h" #include +#include #include #include #include @@ -28,6 +29,7 @@ #include "arrow/api.h" #include "arrow/ipc/json_simple.h" +#include "fmt/format.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/utils/checked_cast.h" #include "paimon/core/core_options.h" @@ -35,8 +37,11 @@ #include "paimon/core/io/data_file_meta.h" #include "paimon/core/io/data_file_path_factory.h" #include "paimon/core/io/file_index_options.h" +#include "paimon/core/realtime/data_file_key_offset_index_writer.h" #include "paimon/core/realtime/realtime_key_offset_index.h" +#include "paimon/core/schema/schema_manager.h" #include "paimon/defs.h" +#include "paimon/memory/bytes.h" #include "paimon/memory/memory_pool.h" #include "paimon/testing/utils/binary_row_generator.h" #include "paimon/testing/utils/testharness.h" @@ -73,9 +78,13 @@ class RealtimeOffsetFileIndexLookupTest : public testing::Test { ASSERT_OK(path_factory_->Init(directory_->Str(), /*format_identifier=*/"parquet", /*data_file_prefix=*/"data-", /*external_path_provider=*/nullptr)); - key_field_ = arrow::field("id", arrow::int64()); - schema_ = arrow::schema( - {DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset()), key_field_}); + key_field_ = DataField::ConvertDataFieldToArrowField( + DataField(1, arrow::field("id", arrow::int64(), /*nullable=*/false))); + schema_ = DataField::ConvertDataFieldsToArrowSchema( + {DataField(0, SpecialFields::RealtimeOffset().ArrowField()), + DataField(1, arrow::field("id", arrow::int64(), /*nullable=*/false))}); + schema_manager_ = + std::make_shared(directory_->GetFileSystem(), directory_->Str()); options_ = {{"file-index.bitmap.columns", SpecialFields::RealtimeOffset().Name()}, {Options::FILE_INDEX_IN_MANIFEST_THRESHOLD, "1MB"}}; } @@ -87,7 +96,36 @@ class RealtimeOffsetFileIndexLookupTest : public testing::Test { CoreOptions::FromMap(options_, directory_->GetFileSystem())); return RealtimeOffsetFileIndexLookup::Create( schema_, data_schema_id, key_field_, data_files, path_factory_, - directory_->GetFileSystem(), GetDefaultPool(), options); + directory_->GetFileSystem(), options, schema_manager_, GetDefaultPool()); + } + + Result> WriteHistoricalSchema( + const std::shared_ptr& schema) const { + std::map options = options_; + options[Options::REALTIME_DEDUPLICATE_KEY_FIELDS] = "id"; + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table_schema, + TableSchema::Create(0, schema, {}, {}, options)); + PAIMON_RETURN_NOT_OK( + directory_->GetFileSystem()->Mkdirs(schema_manager_->SchemaDirectory())); + PAIMON_ASSIGN_OR_RAISE(std::string content, table_schema->ToJsonString()); + PAIMON_RETURN_NOT_OK(directory_->GetFileSystem()->AtomicStore( + schema_manager_->SchemaDirectory() + "/schema-0", content)); + return table_schema; + } + + Result> CreateKeyIndexedDataFile(const std::string& file_name, + const std::string& json) const { + std::shared_ptr batch = checked_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(schema_->fields()), json) + .ValueOrDie()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr writer, + DataFileKeyOffsetIndexWriter::Create( + key_field_, directory_->GetFileSystem(), GetDefaultPool())); + PAIMON_RETURN_NOT_OK(writer->AddBatch(batch)); + std::shared_ptr file = MakeDataFile(file_name, batch->length()); + PAIMON_ASSIGN_OR_RAISE(std::string sidecar, + writer->Finish(path_factory_->ToPath(file), file->row_count)); + return file->CopyWithExtraFiles({sidecar}); } Result> CreateIndexedDataFile( @@ -133,6 +171,7 @@ class RealtimeOffsetFileIndexLookupTest : public testing::Test { std::shared_ptr key_field_; std::shared_ptr schema_; std::map options_; + std::shared_ptr schema_manager_; }; TEST_F(RealtimeOffsetFileIndexLookupTest, TestLookupFilePositionsAcrossDataFiles) { @@ -166,6 +205,28 @@ TEST_F(RealtimeOffsetFileIndexLookupTest, TestLookupFilePositionsAcrossDataFiles ASSERT_TRUE(positions.empty()); } +TEST_F(RealtimeOffsetFileIndexLookupTest, TestMixedExactIndexTypes) { + std::vector> files; + int64_t offset = 0; + for (const std::string type : {"bitmap", "bsi", "range-bitmap"}) { + options_ = {{"file-index." + type + ".columns", "_REALTIME_OFFSET"}, + {Options::FILE_INDEX_IN_MANIFEST_THRESHOLD, "1MB"}}; + const std::string json = fmt::format("[[{}, 1]]", offset); + ASSERT_OK_AND_ASSIGN(std::shared_ptr file, + CreateIndexedDataFile(type + ".parquet", json, offset, offset)); + files.push_back(file); + ++offset; + } + ASSERT_OK_AND_ASSIGN(std::shared_ptr lookup, + CreateLookup(files)); + std::map positions; + ASSERT_OK_AND_ASSIGN(positions, lookup->LookupFilePositions(RoaringBitmap64::From({0, 1, 2}))); + ASSERT_EQ(3, positions.size()); + for (const auto& entry : positions) { + ASSERT_EQ("{0}", entry.second.ToString()); + } +} + TEST_F(RealtimeOffsetFileIndexLookupTest, TestCreateAndCloneDataFiles) { ASSERT_OK_AND_ASSIGN(std::shared_ptr lookup, CreateLookup(/*data_files=*/{})); @@ -192,14 +253,83 @@ TEST_F(RealtimeOffsetFileIndexLookupTest, TestCreateAndCloneDataFiles) { "invalid data file"); } +TEST_F(RealtimeOffsetFileIndexLookupTest, TestReusesUnchangedFileKeyReaders) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr first, + CreateKeyIndexedDataFile("first.parquet", R"([[10, 1], [12, 2]])")); + ASSERT_OK_AND_ASSIGN(std::shared_ptr second, + CreateKeyIndexedDataFile("second.parquet", R"([[20, 1], [22, 2]])")); + ASSERT_OK_AND_ASSIGN(std::shared_ptr initial, + CreateLookup({first})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr old_keys, + initial->CreateKeyOffsetLookup()); + const std::string first_path = path_factory_->ToPath(first); + const std::string second_path = path_factory_->ToPath(second); + const auto first_reader = initial->key_readers_.at(first_path).reader; + + auto first_copy = std::make_shared(*first); + ASSERT_OK_AND_ASSIGN(std::shared_ptr next, + initial->WithDataFiles({first_copy, second})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr next_keys, + next->CreateKeyOffsetLookup()); + ASSERT_EQ(first_reader, next->key_readers_.at(first_path).reader); + auto reading_old = std::async(std::launch::async, + [&]() { return old_keys->LookupOffsets(MakeKeys(key_field_)); }); + auto reading_next = std::async( + std::launch::async, [&]() { return next_keys->LookupOffsets(MakeKeys(key_field_)); }); + ASSERT_OK_AND_ASSIGN(RoaringBitmap64 old_found, reading_old.get()); + ASSERT_EQ("{10,12}", old_found.ToString()); + ASSERT_OK_AND_ASSIGN(RoaringBitmap64 next_found, reading_next.get()); + ASSERT_EQ("{10,12,20,22}", next_found.ToString()); + ASSERT_EQ(1, initial->key_readers_.size()); + ASSERT_EQ(2, next->key_readers_.size()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr repeated, + next->CreateKeyOffsetLookup()); + ASSERT_EQ(first_reader, next->key_readers_.at(first_path).reader); + ASSERT_OK_AND_ASSIGN(RoaringBitmap64 found, next_keys->LookupOffsets(MakeKeys(key_field_))); + ASSERT_EQ("{10,12,20,22}", found.ToString()); + ASSERT_OK_AND_ASSIGN(found, old_keys->LookupOffsets(MakeKeys(key_field_))); + ASSERT_EQ("{10,12}", found.ToString()); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr removed, + next->WithDataFiles({second})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr removed_keys, + removed->CreateKeyOffsetLookup()); + ASSERT_EQ(1, removed->key_readers_.size()); + ASSERT_EQ(next->key_readers_.at(second_path).reader, + removed->key_readers_.at(second_path).reader); + ASSERT_OK_AND_ASSIGN(found, removed_keys->LookupOffsets(MakeKeys(key_field_))); + ASSERT_EQ("{20,22}", found.ToString()); + ASSERT_OK_AND_ASSIGN(found, next_keys->LookupOffsets(MakeKeys(key_field_))); + ASSERT_EQ("{10,12,20,22}", found.ToString()); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr replacement, + CreateKeyIndexedDataFile("replacement.parquet", R"([[30, 1], [32, 2]])")); + std::shared_ptr changed = first->CopyWithExtraFiles(replacement->extra_files); + ASSERT_OK_AND_ASSIGN(std::shared_ptr rewritten, + initial->WithDataFiles({changed})); + ASSERT_TRUE(rewritten->key_readers_.empty()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr rewritten_keys, + rewritten->CreateKeyOffsetLookup()); + ASSERT_NE(first_reader, rewritten->key_readers_.at(first_path).reader); + ASSERT_OK_AND_ASSIGN(found, rewritten_keys->LookupOffsets(MakeKeys(key_field_))); + ASSERT_EQ("{30,32}", found.ToString()); +} + TEST_F(RealtimeOffsetFileIndexLookupTest, TestDifferentSchemaDoesNotUseCurrentStatsPosition) { - // These unnamed stats represent another field at position 0 in the old schema. The current - // schema has _REALTIME_OFFSET at position 0, so interpreting them with the current layout would - // incorrectly prune offset 10 before consulting its bitmap index. + ASSERT_OK_AND_ASSIGN(std::shared_ptr old_schema, + WriteHistoricalSchema(arrow::schema( + {arrow::field("id", arrow::int64(), false), + arrow::field("_REALTIME_OFFSET", arrow::int64(), false)}))); + key_field_ = DataField::ConvertDataFieldToArrowField(old_schema->Fields()[0]); + schema_ = arrow::schema( + {DataField::ConvertDataFieldToArrowField(old_schema->Fields()[1]), key_field_}); + // Offset is at position 1 in the old schema, but position 0 in the current schema. ASSERT_OK_AND_ASSIGN(std::shared_ptr old_schema_file, CreateIndexedDataFile("data-0.parquet", R"([[10, 1], [12, 2]])", /*min_offset=*/100, /*max_offset=*/200, /*schema_id=*/0, /*name_value_stats=*/false)); + old_schema_file->value_stats = + BinaryRowGenerator::GenerateStats({100, 10}, {200, 12}, {0, 0}, GetDefaultPool().get()); ASSERT_OK_AND_ASSIGN(std::shared_ptr lookup, CreateLookup({old_schema_file}, /*data_schema_id=*/1)); @@ -209,6 +339,59 @@ TEST_F(RealtimeOffsetFileIndexLookupTest, TestDifferentSchemaDoesNotUseCurrentSt ASSERT_OK_AND_ASSIGN(positions, lookup->LookupFilePositions(deleted_offsets)); ASSERT_EQ(1, positions.size()); ASSERT_EQ("{0}", positions.at("data-0.parquet").ToString()); + + // The old schema's actual offset bounds can also prune irrelevant lookups. + old_schema_file->embedded_index = + std::make_shared("invalid index", GetDefaultPool().get()); + RoaringBitmap64 outside; + outside.Add(100); + ASSERT_OK_AND_ASSIGN(positions, lookup->LookupFilePositions(outside)); + ASSERT_TRUE(positions.empty()); +} + +TEST_F(RealtimeOffsetFileIndexLookupTest, TestHistoricalKeyRenameAndIncompatibleKey) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr old_schema, + WriteHistoricalSchema( + arrow::schema({arrow::field("_REALTIME_OFFSET", arrow::int64(), false), + arrow::field("id", arrow::int64(), false)}))); + schema_ = DataField::ConvertDataFieldsToArrowSchema(old_schema->Fields()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr file, + CreateIndexedDataFile("data-0.parquet", R"([[10, 1], [12, 2]])", 10, 12)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr writer, + DataFileKeyOffsetIndexWriter::Create( + schema_->field(1), directory_->GetFileSystem(), GetDefaultPool())); + auto batch = checked_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(schema_->fields()), + R"([[10, 1], [12, 2]])") + .ValueOrDie()); + ASSERT_OK(writer->AddBatch(batch)); + ASSERT_OK_AND_ASSIGN(std::string sidecar, + writer->Finish(path_factory_->ToPath(file), file->row_count)); + file = file->CopyWithExtraFiles({sidecar}); + key_field_ = schema_->field(1)->WithName("renamed_id"); + schema_ = arrow::schema({schema_->field(0), key_field_}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr lookup, + CreateLookup({file}, /*data_schema_id=*/1)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr keys, + lookup->CreateKeyOffsetLookup()); + ASSERT_OK_AND_ASSIGN(RoaringBitmap64 found, keys->LookupOffsets(MakeKeys(key_field_))); + ASSERT_EQ("{10,12}", found.ToString()); + + key_field_ = key_field_->WithType(arrow::int32()); + ASSERT_OK_AND_ASSIGN(lookup, CreateLookup({file}, /*data_schema_id=*/1)); + ASSERT_NOK_WITH_MSG(lookup->CreateKeyOffsetLookup(), "deduplicate key is incompatible"); + key_field_ = DataField::ConvertDataFieldToArrowField( + DataField(2, arrow::field("renamed_id", arrow::int64(), false))); + ASSERT_OK_AND_ASSIGN(lookup, CreateLookup({file}, /*data_schema_id=*/1)); + ASSERT_NOK_WITH_MSG(lookup->CreateKeyOffsetLookup(), "deduplicate key is incompatible"); + + schema_ = DataField::ConvertDataFieldsToArrowSchema( + {DataField(3, arrow::field("_REALTIME_OFFSET", arrow::int64(), false)), + DataField(1, arrow::field("id", arrow::int64(), false))}); + ASSERT_OK_AND_ASSIGN(lookup, CreateLookup({file}, /*data_schema_id=*/1)); + RoaringBitmap64 offsets; + offsets.Add(10); + ASSERT_NOK_WITH_MSG(lookup->LookupFilePositions(offsets), "offset field ID is incompatible"); } TEST_F(RealtimeOffsetFileIndexLookupTest, TestCreateValidation) { @@ -217,12 +400,14 @@ TEST_F(RealtimeOffsetFileIndexLookupTest, TestCreateValidation) { SpecialFields::RealtimeOffset().Name()}})); ASSERT_NOK_WITH_MSG(RealtimeOffsetFileIndexLookup::Create( schema_, /*data_schema_id=*/0, key_field_, {nullptr}, path_factory_, - directory_->GetFileSystem(), GetDefaultPool(), valid_options), + directory_->GetFileSystem(), valid_options, + /*schema_manager=*/nullptr, GetDefaultPool()), "invalid data file"); - ASSERT_NOK_WITH_MSG(RealtimeOffsetFileIndexLookup::Create( - schema_, /*data_schema_id=*/0, key_field_, {}, nullptr, - directory_->GetFileSystem(), GetDefaultPool(), valid_options), - "missing a dependency"); + ASSERT_NOK_WITH_MSG( + RealtimeOffsetFileIndexLookup::Create(schema_, /*data_schema_id=*/0, key_field_, {}, + nullptr, directory_->GetFileSystem(), valid_options, + /*schema_manager=*/nullptr, GetDefaultPool()), + "missing a dependency"); } } // namespace paimon::test diff --git a/src/paimon/core/realtime/realtime_primary_key_reader_test.cpp b/src/paimon/core/realtime/realtime_primary_key_reader_test.cpp index d365de504..40b400fa0 100644 --- a/src/paimon/core/realtime/realtime_primary_key_reader_test.cpp +++ b/src/paimon/core/realtime/realtime_primary_key_reader_test.cpp @@ -251,7 +251,7 @@ TEST_F(RealtimePrimaryKeyReaderTest, TestQueryBitmapBounds) { "selected row id 1 is out of bounds for realtime query batch length 1"); } -TEST_F(RealtimePrimaryKeyReaderTest, TestQueryRejectsPartialBitmap) { +TEST_F(RealtimePrimaryKeyReaderTest, TestQueryAllowsPartialBitmap) { std::shared_ptr key = MakeField("key", arrow::int32(), 0); std::shared_ptr value_schema = arrow::schema({key}); std::shared_ptr store_write_schema = MakeStoreWriteSchema({key}); @@ -271,7 +271,15 @@ TEST_F(RealtimePrimaryKeyReaderTest, TestQueryRejectsPartialBitmap) { std::unique_ptr reader, CreateRealtimePrimaryKeyQueryReaderForTest(std::move(batch_reader), OffsetRange(0, 2), value_schema, value_schema, pool_)); - ASSERT_NOK_WITH_MSG(reader->NextBatch(), "must cover every raw transport row"); + ASSERT_OK_AND_ASSIGN( + std::vector results, + (ReadResultCollector::CollectKeyValueResult(reader.get()))); + std::vector row_kinds = {const_cast(RowKind::Insert())}; + std::vector levels = {KeyValue::UNKNOWN_LEVEL}; + std::vector expected = + KeyValueChecker::GenerateKeyValues(row_kinds, {10}, levels, {{1}}, {{1}}, pool_); + KeyValueChecker::CheckResult(expected, results, 1, 1); } TEST_F(RealtimePrimaryKeyReaderTest, TestCommitCoverageAcrossReadersAndBatches) { diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.cpp b/src/paimon/core/realtime/realtime_primary_key_writer.cpp index 05d534a1a..7c56a4e4b 100644 --- a/src/paimon/core/realtime/realtime_primary_key_writer.cpp +++ b/src/paimon/core/realtime/realtime_primary_key_writer.cpp @@ -206,9 +206,11 @@ Status RealtimePrimaryKeyWriter::Write(std::unique_ptr&& batch) { return Status::OK(); } -Status RealtimePrimaryKeyWriter::SealCurrentSegment() { +Status RealtimePrimaryKeyWriter::SealCurrentSegment(bool spill) { + auto* two_phase = dynamic_cast(realtime_store_.get()); PAIMON_ASSIGN_OR_RAISE(std::optional> segment, - realtime_store_->SealForCommit()); + !spill && two_phase ? two_phase->FreezeBuildingSegment() + : realtime_store_->SealForCommit()); std::lock_guard lock(realtime_store_mutex_); if (!segment) { return Status::OK(); @@ -223,12 +225,12 @@ Status RealtimePrimaryKeyWriter::SealCurrentSegment() { Status RealtimePrimaryKeyWriter::Seal() { std::lock_guard lock(prepare_mutex_); - return SealCurrentSegment(); + return SealCurrentSegment(/*spill=*/true); } Result RealtimePrimaryKeyWriter::PrepareCommit(bool wait_compaction) { std::lock_guard prepare_lock(prepare_mutex_); - PAIMON_RETURN_NOT_OK(SealCurrentSegment()); + PAIMON_RETURN_NOT_OK(SealCurrentSegment(/*spill=*/false)); std::vector> segments; { std::lock_guard store_lock(realtime_store_mutex_); @@ -248,7 +250,6 @@ Result RealtimePrimaryKeyWriter::PrepareCommit(bool wait_compac Status RealtimePrimaryKeyWriter::FlushSegment( const std::shared_ptr& segment) { - // TODO(xinyu.lxy): Validate row count from commit readers. PAIMON_ASSIGN_OR_RAISE(std::vector> readers, realtime_store_->CreateCommitReaders(segment)); PAIMON_ASSIGN_OR_RAISE( diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.h b/src/paimon/core/realtime/realtime_primary_key_writer.h index d3fb718bc..0d0456905 100644 --- a/src/paimon/core/realtime/realtime_primary_key_writer.h +++ b/src/paimon/core/realtime/realtime_primary_key_writer.h @@ -83,7 +83,7 @@ class RealtimePrimaryKeyWriter final : public BatchWriter { const std::shared_ptr& memory_pool); Status FlushSegment(const std::shared_ptr& segment); - Status SealCurrentSegment(); + Status SealCurrentSegment(bool spill); std::shared_ptr memory_pool_; std::shared_ptr arrow_pool_; diff --git a/src/paimon/core/realtime/realtime_utils.h b/src/paimon/core/realtime/realtime_utils.h index 4a4ce60ee..6b28a04ce 100644 --- a/src/paimon/core/realtime/realtime_utils.h +++ b/src/paimon/core/realtime/realtime_utils.h @@ -29,6 +29,9 @@ #include "paimon/core/core_options.h" #include "paimon/core/io/file_index_options.h" #include "paimon/core/schema/arrow_schema_validator.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/file_index/file_indexer.h" +#include "paimon/file_index/file_indexer_factory.h" #include "paimon/result.h" namespace paimon { @@ -38,6 +41,13 @@ class RealtimeUtils { RealtimeUtils() = delete; ~RealtimeUtils() = delete; + static Status ValidateBranch(const std::string& branch) { + if (!BranchManager::IsMainBranch(BranchManager::NormalizeBranch(branch))) { + return Status::Invalid("real-time operations only support the main branch"); + } + return Status::OK(); + } + static Result GetDeduplicateKeyPosition( const std::shared_ptr& schema, const std::vector& deduplicate_key_fields) { @@ -92,8 +102,6 @@ class RealtimeUtils { return Status::OK(); } - // TODO(xinyu.lxy): Relax the bitmap-only restriction after other exact file indexes can - // return row positions for realtime offsets. static Status ValidateDeduplicateFileIndex(const CoreOptions& options) { if (!options.FileIndexReadEnabled()) { return Status::Invalid( @@ -103,12 +111,20 @@ class RealtimeUtils { FileIndexOptions::FromCoreOptions(options)); const std::string& offset_name = SpecialFields::RealtimeOffset().Name(); for (const FileIndexDefinition& definition : file_indexes.Definitions()) { - if (definition.column_name == offset_name && definition.index_type == "bitmap") { + if (definition.column_name != offset_name) { + continue; + } + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr indexer, + FileIndexerFactory::Get(definition.index_type, definition.options)); + if (indexer && indexer->SupportsExactPredicate(Function::Type::IN, FieldType::BIGINT) && + indexer->SupportsExactPredicate(Function::Type::NOT_IN, FieldType::BIGINT)) { return Status::OK(); } } return Status::Invalid( - "real-time deduplicate mode requires file-index.bitmap.columns to include ", + "real-time deduplicate mode requires an index supporting exact BIGINT IN and NOT_IN " + "row positions on ", offset_name); } }; diff --git a/src/paimon/core/realtime/realtime_utils_test.cpp b/src/paimon/core/realtime/realtime_utils_test.cpp index 685c51835..54f3efd27 100644 --- a/src/paimon/core/realtime/realtime_utils_test.cpp +++ b/src/paimon/core/realtime/realtime_utils_test.cpp @@ -30,6 +30,14 @@ namespace paimon::test { +TEST(RealtimeUtilsTest, TestValidateBranch) { + ASSERT_OK(RealtimeUtils::ValidateBranch("")); + ASSERT_OK(RealtimeUtils::ValidateBranch(" ")); + ASSERT_OK(RealtimeUtils::ValidateBranch("main")); + ASSERT_NOK_WITH_MSG(RealtimeUtils::ValidateBranch("dev"), + "real-time operations only support the main branch"); +} + TEST(RealtimeUtilsTest, TestValidateOffsetField) { const std::shared_ptr offset_field = DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset()); @@ -101,14 +109,28 @@ TEST(RealtimeUtilsTest, TestValidateDeduplicateFileIndex) { SpecialFields::RealtimeOffset().Name()}})); ASSERT_OK(RealtimeUtils::ValidateDeduplicateFileIndex(valid_options)); + for (const std::string type : {"bsi", "range-bitmap"}) { + ASSERT_OK_AND_ASSIGN(CoreOptions options, + CoreOptions::FromMap({{"file-index." + type + ".columns", + SpecialFields::RealtimeOffset().Name()}})); + ASSERT_OK(RealtimeUtils::ValidateDeduplicateFileIndex(options)); + } + ASSERT_OK_AND_ASSIGN( + CoreOptions approximate_options, + CoreOptions::FromMap({{"file-index.bloom-filter.columns", "_REALTIME_OFFSET"}})); + ASSERT_NOK_WITH_MSG(RealtimeUtils::ValidateDeduplicateFileIndex(approximate_options), + "requires an index supporting exact BIGINT IN and NOT_IN row positions"); + ASSERT_OK_AND_ASSIGN(CoreOptions missing_index_options, CoreOptions::FromMap({})); ASSERT_NOK_WITH_MSG(RealtimeUtils::ValidateDeduplicateFileIndex(missing_index_options), - "requires file-index.bitmap.columns to include _REALTIME_OFFSET"); + "requires an index supporting exact BIGINT IN and NOT_IN row positions on " + "_REALTIME_OFFSET"); ASSERT_OK_AND_ASSIGN(CoreOptions wrong_column_options, CoreOptions::FromMap({{"file-index.bitmap.columns", "id"}})); ASSERT_NOK_WITH_MSG(RealtimeUtils::ValidateDeduplicateFileIndex(wrong_column_options), - "requires file-index.bitmap.columns to include _REALTIME_OFFSET"); + "requires an index supporting exact BIGINT IN and NOT_IN row positions on " + "_REALTIME_OFFSET"); ASSERT_OK_AND_ASSIGN( CoreOptions disabled_read_options, diff --git a/src/paimon/core/schema/schema_validation.cpp b/src/paimon/core/schema/schema_validation.cpp index 5660fe4c1..ffed109ec 100644 --- a/src/paimon/core/schema/schema_validation.cpp +++ b/src/paimon/core/schema/schema_validation.cpp @@ -689,14 +689,14 @@ Status SchemaValidation::ValidateRealtime(const TableSchema& schema, const CoreO PAIMON_RETURN_NOT_OK(Preconditions::CheckState( !options.DeletionVectorsBitmap64(), "Real-time deduplicate mode does not support 'deletion-vectors.bitmap64=true'.")); - PAIMON_RETURN_NOT_OK(Preconditions::CheckState( - !options.RealtimeSpillEnabled(), - "Real-time deduplicate mode does not support 'realtime.spill-enabled=true'.")); - const std::shared_ptr arrow_schema = DataField::ConvertDataFieldsToArrowSchema(schema.Fields()); PAIMON_RETURN_NOT_OK( RealtimeUtils::ValidateDeduplicateSchema(arrow_schema, deduplicate_key_fields)); + PAIMON_RETURN_NOT_OK(Preconditions::CheckState( + !schema.BucketKeys().empty() && + ObjectUtils::ContainsAll(deduplicate_key_fields, schema.BucketKeys()), + "Real-time deduplicate mode requires bucket-key to use only deduplicate key fields.")); return RealtimeUtils::ValidateDeduplicateFileIndex(options); } diff --git a/src/paimon/core/schema/schema_validation_test.cpp b/src/paimon/core/schema/schema_validation_test.cpp index 631a8123a..44aaa71f5 100644 --- a/src/paimon/core/schema/schema_validation_test.cpp +++ b/src/paimon/core/schema/schema_validation_test.cpp @@ -187,8 +187,7 @@ TEST(SchemaValidationTest, TestRealtimeDeduplicate) { std::map spill_options = valid_options; spill_options[Options::REALTIME_SPILL_ENABLED] = "true"; - ASSERT_NOK_WITH_MSG(CreateAndValidateTableSchema(schema, /*primary_keys=*/{}, spill_options), - "does not support 'realtime.spill-enabled=true'"); + ASSERT_OK(CreateAndValidateTableSchema(schema, /*primary_keys=*/{}, spill_options)); std::map bitmap64_options = valid_options; bitmap64_options[Options::DELETION_VECTOR_BITMAP64] = "true"; @@ -207,7 +206,18 @@ TEST(SchemaValidationTest, TestRealtimeDeduplicate) { no_offset_index_options.erase("file-index.bitmap.columns"); ASSERT_NOK_WITH_MSG( CreateAndValidateTableSchema(schema, /*primary_keys=*/{}, no_offset_index_options), - "requires file-index.bitmap.columns to include _REALTIME_OFFSET"); + "requires an index supporting exact BIGINT IN and NOT_IN row positions on " + "_REALTIME_OFFSET"); + + for (const std::string type : {"bsi", "range-bitmap", "bloom-filter"}) { + std::map index_options = no_offset_index_options; + index_options["file-index." + type + ".columns"] = "_REALTIME_OFFSET"; + if (type == "bloom-filter") { + ASSERT_NOK(CreateAndValidateTableSchema(schema, {}, index_options)); + } else { + ASSERT_OK(CreateAndValidateTableSchema(schema, {}, index_options)); + } + } std::map disabled_file_index_read_options = valid_options; disabled_file_index_read_options[Options::FILE_INDEX_READ_ENABLED] = "false"; @@ -216,6 +226,48 @@ TEST(SchemaValidationTest, TestRealtimeDeduplicate) { "requires 'file-index.read.enabled=true'"); } +TEST(SchemaValidationTest, TestRealtimeDeduplicateBucketKey) { + const std::shared_ptr schema = arrow::schema( + {arrow::field("id", arrow::int64(), /*nullable=*/false), + arrow::field("payload", arrow::utf8()), arrow::field("pt", arrow::utf8()), + arrow::field(SpecialFields::RealtimeOffset().Name(), arrow::int64(), /*nullable=*/false)}); + for (const std::vector& partition_keys : + {std::vector{}, std::vector{"pt"}}) { + for (const std::string buckets : {"1", "2"}) { + std::map options = { + {Options::BUCKET, buckets}, + {Options::BUCKET_KEY, "id"}, + {Options::REALTIME_ENABLED, "true"}, + {Options::REALTIME_DEDUPLICATE_KEY_FIELDS, "id"}, + {Options::DELETION_VECTORS_ENABLED, "true"}, + {"file-index.bitmap.columns", SpecialFields::RealtimeOffset().Name()}}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, schema, partition_keys, + /*primary_keys=*/{}, options)); + ASSERT_OK(SchemaValidation::ValidateTableSchema(*table_schema)); + for (const std::string bucket_key : {"payload", "id,payload", "_REALTIME_OFFSET"}) { + SCOPED_TRACE(bucket_key); + options[Options::BUCKET_KEY] = bucket_key; + ASSERT_OK_AND_ASSIGN(table_schema, + TableSchema::Create(/*schema_id=*/0, schema, partition_keys, + /*primary_keys=*/{}, options)); + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options)); + ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateRealtime(*table_schema, core_options), + "bucket-key to use only deduplicate key fields"); + ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema), + "bucket-key to use only deduplicate key fields"); + } + options.erase(Options::BUCKET_KEY); + ASSERT_OK_AND_ASSIGN(table_schema, + TableSchema::Create(/*schema_id=*/0, schema, partition_keys, + /*primary_keys=*/{}, options)); + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options)); + ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateRealtime(*table_schema, core_options), + "bucket-key to use only deduplicate key fields"); + } + } +} + TEST(SchemaValidationTest, TestStartupMode) { auto validate = [](const std::map& options) -> Status { PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(options)); diff --git a/src/paimon/core/table/source/append_only_table_read.cpp b/src/paimon/core/table/source/append_only_table_read.cpp index 2e9cdb992..47f1c5a32 100644 --- a/src/paimon/core/table/source/append_only_table_read.cpp +++ b/src/paimon/core/table/source/append_only_table_read.cpp @@ -43,9 +43,11 @@ #include "paimon/core/operation/internal_read_context.h" #include "paimon/core/operation/raw_file_split_read.h" #include "paimon/core/realtime/realtime_context_impl.h" +#include "paimon/core/realtime/realtime_file_schema.h" #include "paimon/core/realtime/realtime_reader.h" #include "paimon/core/realtime/realtime_schema_layout.h" #include "paimon/core/realtime/realtime_store_read_pipeline.h" +#include "paimon/core/schema/schema_manager.h" #include "paimon/core/table/source/append_count_reader.h" #include "paimon/core/table/source/data_split_impl.h" #include "paimon/core/table/source/realtime_split.h" @@ -120,6 +122,13 @@ AppendOnlyTableRead::AppendOnlyTableRead(const std::shared_ptr& executor) : context_(context) { const auto& core_options = context->GetCoreOptions(); + if (!core_options.GetRealtimeDeduplicateKeyFields().empty()) { + realtime_file_schema_ = std::make_shared( + DataField::ConvertDataFieldsToArrowSchema(context->GetTableSchema()->Fields()), + context->GetTableSchema()->Id(), + std::make_shared(core_options.GetFileSystem(), context->GetPath(), + core_options.GetBranch())); + } if (core_options.DataEvolutionEnabled()) { // add data evolution first split_reads_.push_back( @@ -341,23 +350,23 @@ Result> AppendOnlyTableRead::CreateDeduplicateDiskR read_schema = arrow::schema(std::move(fields), read_schema->metadata()); } - // TODO(xinyu.lxy): Build a file-specific visibility predicate for each data file. Use the - // `_REALTIME_OFFSET` min/max statistics in DataFileMeta to remove deleted offsets that cannot - // intersect that file, instead of pushing the complete bucket offset DV to every file. - std::vector deleted_offsets; - deleted_offsets.reserve(offset_deletions->Cardinality()); - for (RoaringBitmap64::Iterator offset = offset_deletions->Begin(); - offset != offset_deletions->End(); ++offset) { - deleted_offsets.emplace_back(static_cast(*offset)); - } const int32_t offset_position = read_schema->GetFieldIndex(offset_name); - std::shared_ptr visibility_predicate = - PredicateBuilder::NotIn(offset_position, offset_name, FieldType::BIGINT, deleted_offsets); - std::shared_ptr predicate = visibility_predicate; - if (context_->GetPredicate()) { - PAIMON_ASSIGN_OR_RAISE(predicate, PredicateBuilder::And({context_->GetPredicate(), - std::move(visibility_predicate)})); - } + AbstractSplitRead::PredicateProvider predicate_provider = + [file_schema = realtime_file_schema_, offset_deletions, offset_position, offset_name, + user_predicate = context_->GetPredicate()]( + const std::shared_ptr& file) -> Result> { + PAIMON_ASSIGN_OR_RAISE(std::vector deleted_offsets, + file_schema->SelectOffsets(*file, *offset_deletions)); + if (deleted_offsets.empty()) { + return user_predicate; + } + std::shared_ptr visibility_predicate = PredicateBuilder::NotIn( + offset_position, offset_name, FieldType::BIGINT, deleted_offsets); + if (user_predicate) { + return PredicateBuilder::And({user_predicate, visibility_predicate}); + } + return visibility_predicate; + }; DeletionVector::Factory committed_factory = DeletionVector::CreateFactory( context_->GetCoreOptions().GetFileSystem(), @@ -365,9 +374,10 @@ Result> AppendOnlyTableRead::CreateDeduplicateDiskR context_->GetMemoryPool()); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr reader, - raw_read->CreateReader(data_split->Partition(), data_split->Bucket(), - data_split->DataFiles(), std::move(committed_factory), row_ranges, - read_schema, predicate)); + raw_read->CreateReaderWithPredicateProvider( + data_split->Partition(), data_split->Bucket(), data_split->DataFiles(), + std::move(committed_factory), row_ranges, read_schema, context_->GetPredicate(), + std::move(predicate_provider))); if (!expose_offset) { reader = std::make_unique(std::move(reader)); } diff --git a/src/paimon/core/table/source/append_only_table_read.h b/src/paimon/core/table/source/append_only_table_read.h index 0999774cd..e9becc165 100644 --- a/src/paimon/core/table/source/append_only_table_read.h +++ b/src/paimon/core/table/source/append_only_table_read.h @@ -38,6 +38,7 @@ class InternalReadContext; class MemoryPool; class RealtimeSplit; class RoaringBitmap64; +class RealtimeFileSchema; class AppendOnlyTableRead : public TableRead { public: @@ -67,6 +68,7 @@ class AppendOnlyTableRead : public TableRead { std::vector> split_reads_; std::shared_ptr context_; + std::shared_ptr realtime_file_schema_; }; } // namespace paimon diff --git a/src/paimon/core/table/table.cpp b/src/paimon/core/table/table.cpp index bba5aa150..b7da9cfba 100644 --- a/src/paimon/core/table/table.cpp +++ b/src/paimon/core/table/table.cpp @@ -69,6 +69,7 @@ #include "paimon/core/postpone/postpone_bucket_file_store_write.h" #include "paimon/core/realtime/realtime_context_impl.h" #include "paimon/core/realtime/realtime_schema_layout.h" +#include "paimon/core/realtime/realtime_utils.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/schema_validation.h" #include "paimon/core/schema/table_schema.h" @@ -267,6 +268,9 @@ Result> FileStoreWrite::Create(std::unique_ptrGetOptions(), specific_fs, ctx->GetFileSystemSchemeToIdentifierMap())); std::string branch = ctx->GetBranch(); + if (ctx->GetRealtimeContext()) { + PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateBranch(branch)); + } std::optional catalog_identifier; if (ctx->GetIdentifier()) { PAIMON_ASSIGN_OR_RAISE( @@ -755,6 +759,9 @@ Result> TableScan::Create(std::unique_ptrGetOptions(), shared_context->GetSpecificFileSystem(), {})); + if (shared_context->GetRealtimeContext()) { + PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateBranch(tmp_options.GetBranch())); + } PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, SystemTableLoader::TryParsePath(shared_context->GetPath())); if (system_table_path) { @@ -957,6 +964,9 @@ namespace { Result> CreateInternalReadContext( const std::shared_ptr& context, const std::string& branch, const std::shared_ptr& loaded_schema) { + if (context->GetRealtimeContext()) { + PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateBranch(branch)); + } std::map tmp_options = context->GetOptions(); std::shared_ptr table_schema; const auto& specific_table_schema = context->GetSpecificTableSchema(); @@ -1084,6 +1094,9 @@ Result> TableRead::Create(std::unique_ptrGetExecutor() == nullptr) { return Status::Invalid("executor is null pointer"); } + if (context->GetRealtimeContext()) { + PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateBranch(context->GetBranch())); + } // A table the caller already loaded says what it is, so nothing is read to find out. if (context->GetFormatTable() != nullptr) { if (context->HasVectorOrFullTextSearch()) { diff --git a/src/paimon/testing/utils/CMakeLists.txt b/src/paimon/testing/utils/CMakeLists.txt index 29ad1c77c..a27e94972 100644 --- a/src/paimon/testing/utils/CMakeLists.txt +++ b/src/paimon/testing/utils/CMakeLists.txt @@ -44,6 +44,7 @@ if(PAIMON_BUILD_TESTS) add_paimon_test(test_utils_test SOURCES + blocking_file_system_test.cpp data_generator_test.cpp dict_array_converter_test.cpp STATIC_LINK_LIBS diff --git a/src/paimon/testing/utils/blocking_file_system.h b/src/paimon/testing/utils/blocking_file_system.h new file mode 100644 index 000000000..36da712b1 --- /dev/null +++ b/src/paimon/testing/utils/blocking_file_system.h @@ -0,0 +1,231 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "paimon/fs/file_system.h" +#include "paimon/macros.h" + +namespace paimon::test { + +/// Blocks selected file operations on a delegated file system. +class BlockingFileSystem final : public FileSystem { + public: + enum class Operation { OPEN, CREATE, CLOSE }; + + class Gate { + public: + Gate(std::string path, uint32_t arrivals, bool auto_release) + : path_(std::move(path)), arrivals_(arrivals), auto_release_(auto_release) {} + + bool WaitForArrival() const { + std::unique_lock lock(mutex_); + return cv_.wait_for(lock, std::chrono::seconds(5), [&]() { return entered_ > 0; }); + } + + void Release() { + Release(Status::OK()); + } + + /// Releases the operation with a success or injected error status. + void Release(const Status& status) { + std::lock_guard lock(mutex_); + if (!released_) { + result_ = status; + released_ = true; + cv_.notify_all(); + } + } + + std::string MatchedPath() const { + std::lock_guard lock(mutex_); + return matched_path_; + } + + private: + friend class BlockingFileSystem; + + Status Arrive(const std::string& path) { + std::unique_lock lock(mutex_); + if ((!path_.empty() && path_ != path) || entered_ >= arrivals_) { + return Status::OK(); + } + matched_path_ = path; + ++entered_; + if (auto_release_ && entered_ == arrivals_) { + released_ = true; + } + cv_.notify_all(); + const bool released = + cv_.wait_for(lock, std::chrono::seconds(60), [&]() { return released_; }); + if (!released) { + result_ = Status::IOError("timed out waiting for blocked file operation"); + released_ = true; + cv_.notify_all(); + } + return result_; + } + + std::string path_; + uint32_t arrivals_; + bool auto_release_; + mutable std::mutex mutex_; + mutable std::condition_variable cv_; + uint32_t entered_ = 0; + bool released_ = false; + std::string matched_path_; + Status result_ = Status::OK(); + }; + + explicit BlockingFileSystem(std::shared_ptr delegate) + : delegate_(std::move(delegate)), controls_(std::make_shared()) {} + + /// Blocks the next matching operation. An empty path matches any file. + std::shared_ptr BlockNext(Operation operation, const std::string& path) { + return SetGate(operation, path, 1, false); + } + + /// Releases concurrent matching operations once all required arrivals have entered. + Result> Barrier(Operation operation, const std::string& path, + uint32_t arrivals) { + if (arrivals == 0) { + return Status::Invalid("file operation barrier requires positive arrivals"); + } + return SetGate(operation, path, arrivals, true); + } + + Result> Open(const std::string& path) const override { + PAIMON_RETURN_NOT_OK(Wait(controls_, Operation::OPEN, path)); + return delegate_->Open(path); + } + + Result> Open(const FileStatus& file_status) const override { + PAIMON_RETURN_NOT_OK(Wait(controls_, Operation::OPEN, file_status.GetPath())); + return delegate_->Open(file_status); + } + + Result> Create(const std::string& path, + bool overwrite) const override { + PAIMON_RETURN_NOT_OK(Wait(controls_, Operation::CREATE, path)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr stream, + delegate_->Create(path, overwrite)); + return std::make_unique(std::move(stream), path, controls_); + } + + Status Mkdirs(const std::string& path) const override { + return delegate_->Mkdirs(path); + } + + Status Rename(const std::string& src, const std::string& dst) const override { + return delegate_->Rename(src, dst); + } + + Status Delete(const std::string& path, bool recursive) const override { + return delegate_->Delete(path, recursive); + } + + Result GetFileStatus(const std::string& path) const override { + return delegate_->GetFileStatus(path); + } + + Status ListDir(const std::string& directory, + std::vector* file_status_list) const override { + return delegate_->ListDir(directory, file_status_list); + } + + Status ListFileStatus(const std::string& path, + std::vector* file_status_list) const override { + return delegate_->ListFileStatus(path, file_status_list); + } + + Result Exists(const std::string& path) const override { + return delegate_->Exists(path); + } + + private: + struct Controls { + std::mutex mutex; + std::map> gates; + }; + + class ControlledOutputStream final : public OutputStream { + public: + ControlledOutputStream(std::unique_ptr inner, std::string path, + std::shared_ptr controls) + : inner_(std::move(inner)), path_(std::move(path)), controls_(std::move(controls)) {} + + Result GetPos() const override { + return inner_->GetPos(); + } + Result Write(const char* buffer, int64_t size) override { + return inner_->Write(buffer, size); + } + Status Flush() override { + return inner_->Flush(); + } + Status Close() override { + PAIMON_RETURN_NOT_OK(inner_->Close()); + return Wait(controls_, Operation::CLOSE, path_); + } + Result GetUri() const override { + return inner_->GetUri(); + } + + private: + std::unique_ptr inner_; + std::string path_; + std::shared_ptr controls_; + }; + + std::shared_ptr SetGate(Operation operation, const std::string& path, uint32_t arrivals, + bool auto_release) { + auto gate = std::make_shared(path, arrivals, auto_release); + std::lock_guard lock(controls_->mutex); + controls_->gates[operation] = gate; + return gate; + } + + static Status Wait(const std::shared_ptr& controls, Operation operation, + const std::string& path) { + std::shared_ptr gate; + { + std::lock_guard lock(controls->mutex); + auto iter = controls->gates.find(operation); + if (iter != controls->gates.end()) { + gate = iter->second; + } + } + return gate ? gate->Arrive(path) : Status::OK(); + } + + std::shared_ptr delegate_; + std::shared_ptr controls_; +}; + +} // namespace paimon::test diff --git a/src/paimon/testing/utils/blocking_file_system_test.cpp b/src/paimon/testing/utils/blocking_file_system_test.cpp new file mode 100644 index 000000000..d42970a78 --- /dev/null +++ b/src/paimon/testing/utils/blocking_file_system_test.cpp @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/testing/utils/blocking_file_system.h" + +#include +#include +#include +#include +#include + +#include "paimon/common/utils/path_util.h" +#include "paimon/common/utils/scope_guard.h" +#include "paimon/testing/mock/mock_file_system.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +TEST(BlockingFileSystemTest, TestOpenFiltersPathAndBlocksOnlyOnce) { + auto fs = std::make_shared(std::make_shared()); + auto gate = fs->BlockNext(BlockingFileSystem::Operation::OPEN, "blocked"); + std::future>> opening; + ScopeGuard finish([&]() { + gate->Release(); + if (opening.valid()) { + opening.wait(); + } + }); + opening = std::async(std::launch::async, [&]() { return fs->Open(FileStatus("blocked", 0)); }); + ASSERT_TRUE(gate->WaitForArrival()); + ASSERT_EQ("blocked", gate->MatchedPath()); + ASSERT_EQ(std::future_status::timeout, opening.wait_for(std::chrono::milliseconds(20))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr other, fs->Open("other")); + ASSERT_OK(other->Close()); + gate->Release(); + ASSERT_OK_AND_ASSIGN(std::unique_ptr stream, opening.get()); + ASSERT_OK(stream->Close()); + ASSERT_OK_AND_ASSIGN(stream, fs->Open("blocked")); + ASSERT_OK(stream->Close()); +} + +TEST(BlockingFileSystemTest, TestReleaseInjectsOneShotErrorAndGateCanBeRearmed) { + auto fs = std::make_shared(std::make_shared()); + auto first = fs->BlockNext(BlockingFileSystem::Operation::OPEN, "file"); + first->Release(Status::IOError("first open failed")); + ASSERT_NOK_WITH_MSG(fs->Open("file"), "first open failed"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr stream, fs->Open("file")); + ASSERT_OK(stream->Close()); + auto second = fs->BlockNext(BlockingFileSystem::Operation::OPEN, "file"); + second->Release(Status::IOError("second open failed")); + ASSERT_NOK_WITH_MSG(fs->Open(FileStatus("file", 0)), "second open failed"); + ASSERT_OK_AND_ASSIGN(stream, fs->Open("file")); + ASSERT_OK(stream->Close()); +} + +TEST(BlockingFileSystemTest, TestCreateAndCloseKeepControlsAlive) { + auto directory = UniqueTestDirectory::Create(); + ASSERT_NE(nullptr, directory); + const std::string path = PathUtil::JoinPath(directory->Str(), "output"); + auto fs = std::make_shared(directory->GetFileSystem()); + auto create_gate = fs->BlockNext(BlockingFileSystem::Operation::CREATE, /*path=*/""); + auto close_gate = fs->BlockNext(BlockingFileSystem::Operation::CLOSE, path); + std::unique_ptr stream; + std::future>> creating; + std::future closing; + ScopeGuard finish([&]() { + create_gate->Release(); + close_gate->Release(); + if (creating.valid()) { + creating.wait(); + } + if (closing.valid()) { + closing.wait(); + } + }); + creating = + std::async(std::launch::async, [&]() { return fs->Create(path, /*overwrite=*/false); }); + ASSERT_TRUE(create_gate->WaitForArrival()); + ASSERT_EQ(path, create_gate->MatchedPath()); + ASSERT_OK_AND_ASSIGN(bool exists, fs->Exists(path)); + ASSERT_FALSE(exists); + create_gate->Release(); + ASSERT_OK_AND_ASSIGN(stream, creating.get()); + ASSERT_OK_AND_ASSIGN(exists, fs->Exists(path)); + ASSERT_TRUE(exists); + ASSERT_OK_AND_ASSIGN(int64_t written, stream->Write("data", 4)); + ASSERT_EQ(4, written); + ASSERT_OK(stream->Flush()); + ASSERT_OK_AND_ASSIGN(int64_t position, stream->GetPos()); + ASSERT_EQ(4, position); + closing = std::async(std::launch::async, [&]() { return stream->Close(); }); + ASSERT_TRUE(close_gate->WaitForArrival()); + fs.reset(); + close_gate->Release(Status::IOError("injected close failure")); + ASSERT_NOK_WITH_MSG(closing.get(), "injected close failure"); + std::string content; + ASSERT_OK(directory->GetFileSystem()->ReadFile(path, &content)); + ASSERT_EQ("data", content); +} + +TEST(BlockingFileSystemTest, TestBarrierRequiresConcurrentArrivals) { + auto fs = std::make_shared(std::make_shared()); + ASSERT_NOK_WITH_MSG(fs->Barrier(BlockingFileSystem::Operation::OPEN, "", 0), + "requires positive arrivals"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr barrier, + fs->Barrier(BlockingFileSystem::Operation::OPEN, "", 3)); + std::vector>>> openings; + ScopeGuard finish([&]() { + barrier->Release(); + for (auto& opening : openings) { + if (opening.valid()) { + opening.wait(); + } + } + }); + openings.push_back(std::async(std::launch::async, [&]() { return fs->Open("first"); })); + ASSERT_TRUE(barrier->WaitForArrival()); + ASSERT_EQ(std::future_status::timeout, openings[0].wait_for(std::chrono::milliseconds(20))); + for (int32_t i = 0; i < 2; ++i) { + openings.push_back(std::async(std::launch::async, [&]() { return fs->Open("other"); })); + } + for (auto& opening : openings) { + ASSERT_EQ(std::future_status::ready, opening.wait_for(std::chrono::seconds(5))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr stream, opening.get()); + ASSERT_OK(stream->Close()); + } +} + +} // namespace paimon::test diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp index 8b0f2f9a4..e520d43e9 100644 --- a/test/inte/realtime_write_inte_test.cpp +++ b/test/inte/realtime_write_inte_test.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -88,6 +89,8 @@ #include "paimon/table/source/table_read.h" #include "paimon/table/source/table_scan.h" #include "paimon/testing/mock/mock_index_path_factory.h" +#include "paimon/testing/utils/blocking_file_system.h" +#include "paimon/testing/utils/dict_array_converter.h" #include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/test_helper.h" #include "paimon/testing/utils/testharness.h" @@ -530,6 +533,11 @@ class RealtimeWriteInteTest : public ::testing::Test { } void ConfigureRealtimeDeduplicateTable(const std::vector& partition_keys = {}) { + ConfigureRealtimeDeduplicateTable(partition_keys, "bitmap"); + } + + void ConfigureRealtimeDeduplicateTable(const std::vector& partition_keys, + const std::string& offset_index_type) { fields_ = { arrow::field(SpecialFields::RealtimeOffset().Name(), arrow::int64(), /*nullable=*/false), @@ -541,7 +549,8 @@ class RealtimeWriteInteTest : public ::testing::Test { options_[Options::FILE_FORMAT] = "parquet"; options_[Options::DELETION_VECTORS_ENABLED] = "true"; options_[Options::REALTIME_DEDUPLICATE_KEY_FIELDS] = "id"; - options_["file-index.bitmap.columns"] = SpecialFields::RealtimeOffset().Name(); + options_["file-index." + offset_index_type + ".columns"] = + SpecialFields::RealtimeOffset().Name(); CreateTable(partition_keys); } @@ -980,6 +989,9 @@ class RealtimeWriteInteTest : public ::testing::Test { std::shared_ptr expected = arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(result_fields), expected_json) .ValueOrDie(); + // Match the dictionary and nullability normalization applied by ReadResultCollector. + ASSERT_OK_AND_ASSIGN( + expected, DictArrayConverter::ConvertDictArray(expected, arrow::default_memory_pool())); ASSERT_NE(nullptr, result); ASSERT_TRUE(std::make_shared(expected)->Equals(*result)) << result->ToString(); @@ -1462,6 +1474,174 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateRollingRestoresEveryOffsetS ASSERT_EQ(expected, rows); } +TEST_F(RealtimeWriteInteTest, TestRefreshPreparationAllowsScanWriteAndStoreRegistration) { + options_[Options::BUCKET] = "2"; + ConfigureRealtimeDeduplicateTable(); + auto fs = std::make_shared(dir_->GetFileSystem()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr context, RealtimeContext::Create()); + WriteContextBuilder builder(table_path_, commit_user_); + builder.SetOptions(options_) + .WithStreamingMode(true) + .WithRealtimeContext(context) + .WithFileSystem(fs); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr writer, + FileStoreWrite::Create(std::move(write_context))); + const std::vector initial = {{1, "old", "p0"}}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, + MakeBatch(initial, /*partitioned=*/false)); + ASSERT_OK(writer->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(std::vector progress, + writer->PrepareCommitWithProgress(/*commit_identifier=*/0)); + const auto files = NewFiles(progress); + ASSERT_EQ(1, files.size()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr pinned, CreatePlan(context, /*predicate=*/nullptr)); + + // Commit the second bucket from a separate context before this writer registers its store. + ASSERT_OK_AND_ASSIGN(std::shared_ptr seed_context, RealtimeContext::Create()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr seed_writer, + CreateRealtimeWriter(seed_context)); + ASSERT_OK_AND_ASSIGN(batch, + MakeBatch({{2, "second", "p0"}}, /*partitioned=*/false, /*bucket=*/1)); + ASSERT_OK(seed_writer->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(std::vector seeded, + seed_writer->PrepareCommitWithProgress(/*commit_identifier=*/0)); + progress.insert(progress.end(), seeded.begin(), seeded.end()); + ASSERT_OK_AND_ASSIGN(int64_t snapshot, Commit(progress, /*commit_identifier=*/0)); + ASSERT_OK(seed_writer->Close()); + + auto open_gate = fs->BlockNext( + BlockingFileSystem::Operation::OPEN, + PathUtil::JoinPath(table_path_, "bucket-0/" + files[0]->file_name + ".offset")); + auto refreshing = std::async(std::launch::async, + [&]() { return writer->RefreshCommittedSnapshot(snapshot); }); + std::future>> reading; + std::future writing; + std::future registering; + ScopeGuard finish([&]() { + open_gate->Release(); + if (reading.valid()) { + reading.wait(); + } + if (writing.valid()) { + writing.wait(); + } + if (registering.valid()) { + registering.wait(); + } + if (refreshing.valid()) { + refreshing.wait(); + } + }); + ASSERT_TRUE(open_gate->WaitForArrival()); + reading = std::async(std::launch::async, [&]() { return ReadRows(context); }); + ASSERT_EQ(std::future_status::ready, reading.wait_for(std::chrono::seconds(5))); + ASSERT_OK_AND_ASSIGN(std::vector visible, reading.get()); + std::sort(visible.begin(), visible.end()); + ASSERT_EQ((std::vector{{1, "old", "p0"}, {2, "second", "p0"}}), visible); + writing = std::async(std::launch::async, [&]() -> Status { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr update, + MakeBatch({{1, "new", "p0"}}, /*partitioned=*/false)); + return writer->Write(std::move(update)); + }); + ASSERT_EQ(std::future_status::ready, writing.wait_for(std::chrono::seconds(5))); + ASSERT_OK(writing.get()); + registering = std::async(std::launch::async, [&]() -> Status { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr update, + MakeBatch({{2, "new", "p0"}}, /*partitioned=*/false, /*bucket=*/1)); + return writer->Write(std::move(update)); + }); + ASSERT_EQ(std::future_status::ready, registering.wait_for(std::chrono::seconds(5))); + ASSERT_OK(registering.get()); + const std::vector expected = {{1, "new", "p0"}, {2, "new", "p0"}}; + reading = std::async(std::launch::async, [&]() { return ReadRows(context); }); + ASSERT_EQ(std::future_status::ready, reading.wait_for(std::chrono::seconds(5))); + ASSERT_OK_AND_ASSIGN(visible, reading.get()); + std::sort(visible.begin(), visible.end()); + ASSERT_EQ(expected, visible); + + open_gate->Release(); + ASSERT_OK(refreshing.get()); + ASSERT_OK_AND_ASSIGN(RealtimeMetricValues usage, ReadRealtimeMetrics(writer->GetMetrics())); + ASSERT_EQ(0, usage.sealed_rows); + ASSERT_EQ(2, usage.building_rows); + ASSERT_OK_AND_ASSIGN(visible, ReadRows(pinned, context)); + ASSERT_EQ(initial, visible); + ASSERT_OK_AND_ASSIGN(visible, ReadRows(context)); + std::sort(visible.begin(), visible.end()); + ASSERT_EQ(expected, visible); + + ASSERT_OK_AND_ASSIGN(progress, writer->PrepareCommitWithProgress(/*commit_identifier=*/1)); + ASSERT_EQ(2, progress.size()); + for (const auto& item : progress) { + ASSERT_EQ(OffsetRange(1, 2), item.offset_range); + ASSERT_NE(nullptr, item.deleted_offsets); + ASSERT_EQ("{0}", item.deleted_offsets->ToString()); + } + ASSERT_OK_AND_ASSIGN(snapshot, Commit(progress, /*commit_identifier=*/1)); + ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot)); + ASSERT_OK_AND_ASSIGN(visible, ReadRows()); + std::sort(visible.begin(), visible.end()); + ASSERT_EQ(expected, visible); + ASSERT_OK(writer->Close()); +} + +TEST_F(RealtimeWriteInteTest, TestRefreshPreparationFailureKeepsProgressAndCanRetry) { + ConfigureRealtimeDeduplicateTable(); + auto fs = std::make_shared(dir_->GetFileSystem()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr context, RealtimeContext::Create()); + WriteContextBuilder builder(table_path_, commit_user_); + builder.SetOptions(options_) + .WithStreamingMode(true) + .WithRealtimeContext(context) + .WithFileSystem(fs); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr writer, + FileStoreWrite::Create(std::move(write_context))); + const std::vector initial = {{1, "old", "p0"}}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, + MakeBatch(initial, /*partitioned=*/false)); + ASSERT_OK(writer->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(std::vector progress, + writer->PrepareCommitWithProgress(/*commit_identifier=*/0)); + const auto files = NewFiles(progress); + ASSERT_EQ(1, files.size()); + ASSERT_OK_AND_ASSIGN(int64_t snapshot, Commit(progress, /*commit_identifier=*/0)); + ASSERT_OK_AND_ASSIGN(RealtimeMetricValues before, ReadRealtimeMetrics(writer->GetMetrics())); + ASSERT_EQ(1, before.sealed_rows); + + auto open_gate = fs->BlockNext( + BlockingFileSystem::Operation::OPEN, + PathUtil::JoinPath(table_path_, "bucket-0/" + files[0]->file_name + ".offset")); + open_gate->Release(Status::IOError("injected sidecar open failure")); + ASSERT_NOK_WITH_MSG(writer->RefreshCommittedSnapshot(snapshot), + "injected sidecar open failure"); + ASSERT_OK_AND_ASSIGN(RealtimeMetricValues failed, ReadRealtimeMetrics(writer->GetMetrics())); + ASSERT_EQ(before.sealed_rows, failed.sealed_rows); + ASSERT_EQ(before.total_memory, failed.total_memory); + ASSERT_OK_AND_ASSIGN(std::vector visible, ReadRows(context)); + ASSERT_EQ(initial, visible); + + ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot)); + ASSERT_OK_AND_ASSIGN(RealtimeMetricValues after, ReadRealtimeMetrics(writer->GetMetrics())); + ASSERT_EQ(0, after.sealed_rows); + ASSERT_EQ(0, after.total_rows); + ASSERT_OK_AND_ASSIGN(visible, ReadRows(context)); + ASSERT_EQ(initial, visible); + ASSERT_OK_AND_ASSIGN(batch, MakeBatch({{1, "new", "p0"}}, /*partitioned=*/false)); + ASSERT_OK(writer->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(progress, writer->PrepareCommitWithProgress(/*commit_identifier=*/1)); + ASSERT_EQ(1, progress.size()); + ASSERT_EQ(OffsetRange(1, 2), progress[0].offset_range); + ASSERT_NE(nullptr, progress[0].deleted_offsets); + ASSERT_EQ("{0}", progress[0].deleted_offsets->ToString()); + ASSERT_OK_AND_ASSIGN(snapshot, Commit(progress, /*commit_identifier=*/1)); + ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot)); + ASSERT_OK_AND_ASSIGN(visible, ReadRows()); + ASSERT_EQ((std::vector{{1, "new", "p0"}}), visible); + ASSERT_OK(writer->Close()); +} + TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateWritesDeletionVector) { ConfigureRealtimeDeduplicateTable(); @@ -1534,6 +1714,82 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateWritesDeletionVector) { ASSERT_EQ(expected_rows, disk_rows); } +TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateRestoresHistoricalSchemas) { + ConfigureRealtimeDeduplicateTable(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr context, RealtimeContext::Create()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr writer, CreateRealtimeWriter(context)); + ASSERT_OK(WriteCommitAndRefresh({{1, "old-one", "p0"}, {2, "old-two", "p0"}}, + /*partitioned=*/false, /*bucket=*/0, {}, 0, writer.get())); + ASSERT_OK(WriteCommitAndRefresh({{3, "three", "p0"}}, /*partitioned=*/false, + /*bucket=*/0, {}, 1, writer.get())); + ASSERT_OK(writer->Close()); + writer.reset(); + context.reset(); + + // Keep field IDs while renaming the deduplicate key and a regular column, moving the offset + // column, and adding a nullable column. Restore old files and their .offset sidecars by ID. + const auto renamed_key = fields_[1]->WithName("renamed_id"); + const auto renamed_payload = fields_[2]->WithName("renamed_payload"); + const auto extra = arrow::field("extra", arrow::utf8()); + options_[Options::REALTIME_DEDUPLICATE_KEY_FIELDS] = renamed_key->name(); + options_[Options::BUCKET_KEY] = renamed_key->name(); + ASSERT_OK(TestHelper::WriteNextSchema( + dir_->GetFileSystem(), table_path_, + {DataField(1, renamed_key), DataField(2, renamed_payload), DataField(3, fields_[3]), + DataField(0, fields_[0]), DataField(4, extra)}, + /*highest_field_id=*/4, options_)); + fields_ = {renamed_key, renamed_payload, fields_[3], fields_[0], extra}; + schema_ = arrow::schema(fields_); + ASSERT_OK_AND_ASSIGN(context, RealtimeContext::Create()); + ASSERT_OK_AND_ASSIGN(writer, CreateRealtimeWriter(context)); + auto array = arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(schema_->fields()), R"([ + [1, "new-one", "p0", 3, "x"], + [2, "deleted", "p0", 4, null], + [4, "four", "p0", 5, "y"] + ])") + .ValueOrDie(); + ArrowArray c_array; + ASSERT_TRUE(arrow::ExportArray(*array, &c_array).ok()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr batch, + RecordBatchBuilder(&c_array) + .SetBucket(0) + .SetRowKinds({RecordBatch::RowKind::INSERT, RecordBatch::RowKind::DELETE, + RecordBatch::RowKind::INSERT}) + .Finish()); + ASSERT_OK(writer->Write(std::move(batch))); + + const auto projection = arrow::schema({fields_[0], fields_[1], fields_[4]}); + const std::string expected = R"([ + [0, 3, "three", null], + [0, 1, "new-one", "x"], + [0, 4, "four", "y"] + ])"; + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, CreatePlan(context, /*predicate=*/nullptr)); + ReadPlanWithSchemaAndCheck(plan, context, projection, expected); + ASSERT_OK_AND_ASSIGN(std::vector progress, + writer->PrepareCommitWithProgress(2)); + ASSERT_OK_AND_ASSIGN(int64_t snapshot, Commit(progress, 2)); + ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot)); + ASSERT_OK(writer->Close()); + writer.reset(); + context.reset(); + + // Persisted file-position DVs must still hide the overwritten/deleted historical rows. + ASSERT_OK_AND_ASSIGN(context, RealtimeContext::Create()); + ASSERT_OK_AND_ASSIGN(writer, CreateRealtimeWriter(context)); + ASSERT_OK_AND_ASSIGN(plan, CreatePlan(context, /*predicate=*/nullptr)); + ReadPlanWithSchemaAndCheck(plan, context, projection, expected); + + ASSERT_OK_AND_ASSIGN(Snapshot compact_snapshot, CompactAndCommit(/*partition=*/{}, /*bucket=*/0, + /*commit_identifier=*/3)); + ASSERT_EQ(Snapshot::CommitKind::Compact(), compact_snapshot.GetCommitKind()); + ASSERT_OK(writer->RefreshCommittedSnapshot(compact_snapshot.Id())); + ASSERT_OK_AND_ASSIGN(plan, CreatePlan(context, /*predicate=*/nullptr)); + ReadPlanWithSchemaAndCheck(plan, context, projection, expected); + ASSERT_OK(writer->Close()); +} + TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateCommitRetryMergesLatestDv) { ConfigureRealtimeDeduplicateTable(); ASSERT_OK_AND_ASSIGN(std::shared_ptr realtime_context, @@ -3145,9 +3401,13 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateUpdateBeforeAndAfter) { ASSERT_EQ(expected_rows, disk_rows); } -TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateAcrossMultipleFileIndexStorageForms) { +class RealtimeDeduplicateExactIndexInteTest : public RealtimeWriteInteTest, + public ::testing::WithParamInterface {}; + +TEST_P(RealtimeDeduplicateExactIndexInteTest, + TestRealtimeDeduplicateAcrossMultipleFileIndexStorageForms) { options_[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1MB"; - ConfigureRealtimeDeduplicateTable(); + ConfigureRealtimeDeduplicateTable({}, GetParam()); ASSERT_OK_AND_ASSIGN(std::shared_ptr realtime_context, RealtimeContext::Create()); @@ -3181,7 +3441,7 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateAcrossMultipleFileIndexStor std::vector> external_files = NewFiles(external_progress); ASSERT_EQ(1, external_files.size()); ASSERT_EQ(nullptr, external_files[0]->embedded_index); - // The external bitmap index precedes the deduplicate key-offset sidecar. + // The external file index precedes the deduplicate key-offset sidecar. ASSERT_EQ(2, external_files[0]->extra_files.size()); ASSERT_OK_AND_ASSIGN(int64_t external_snapshot_id, Commit(external_progress, /*commit_identifier=*/1)); @@ -3224,6 +3484,9 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateAcrossMultipleFileIndexStor ASSERT_EQ(expected_rows, disk_rows); } +INSTANTIATE_TEST_SUITE_P(ExactIndex, RealtimeDeduplicateExactIndexInteTest, + ::testing::Values("bitmap", "bsi", "range-bitmap")); + TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateReadViewIsStableAcrossWriteAndRefresh) { ConfigureRealtimeDeduplicateTable(); ASSERT_OK_AND_ASSIGN(std::shared_ptr realtime_context, @@ -6745,6 +7008,99 @@ TEST_F(RealtimeWriteInteTest, TestReaderPinsMemoryAcrossRefresh) { ASSERT_OK(writer->Close()); } +class RealtimePrepareSpillInteTest + : public RealtimeWriteInteTest, + public ::testing::WithParamInterface> {}; + +TEST_P(RealtimePrepareSpillInteTest, TestPrepareFreezesWithoutSpilling) { + const auto [primary_key, deletion_vectors, seal_before_prepare] = GetParam(); + const bool deduplicate = !primary_key && deletion_vectors; + const bool merge_versions = primary_key || deduplicate; + options_[Options::FILE_FORMAT] = "parquet"; + options_[Options::DELETION_VECTORS_ENABLED] = deletion_vectors ? "true" : "false"; + options_[Options::REALTIME_SPILL_ENABLED] = "true"; + if (primary_key) { + CreatePkTable(); + } else if (deduplicate) { + ConfigureRealtimeDeduplicateTable(); + } else { + CreateTable(/*partition_keys=*/{}); + } + const std::string temp_directory = PathUtil::JoinPath(dir_->Str(), "prepare-without-spill"); + ASSERT_OK(dir_->GetFileSystem()->Mkdirs(temp_directory)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr context, RealtimeContext::Create()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr writer, + CreateRealtimeWriter(context, temp_directory)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr initial, + MakeBatch({{1, "old", "p0"}, {2, "two", "p0"}}, + /*partitioned=*/false)); + ASSERT_OK(writer->Write(std::move(initial))); + if (seal_before_prepare) { + ASSERT_OK(writer->Seal()); + } + const int64_t expected_spill_files = seal_before_prepare ? 1 : 0; + ASSERT_EQ(expected_spill_files, + TestHelper::CountChannelFiles(dir_->GetFileSystem(), temp_directory)); + + const std::vector next_rows = + merge_versions ? std::vector{{1, "new", "p0"}, {3, "three", "p0"}} + : std::vector{{3, "three", "p0"}, {4, "four", "p0"}}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr next, + MakeBatch(next_rows, /*partitioned=*/false)); + ASSERT_OK(writer->Write(std::move(next))); + ASSERT_OK_AND_ASSIGN(std::vector progress, + writer->PrepareCommitWithProgress(/*commit_identifier=*/0)); + ASSERT_EQ(1, progress.size()); + ASSERT_EQ(OffsetRange(0, 4), progress[0].offset_range); + if (deduplicate) { + ASSERT_NE(nullptr, progress[0].deleted_offsets); + ASSERT_EQ("{0}", progress[0].deleted_offsets->ToString()); + } + ASSERT_EQ(expected_spill_files, + TestHelper::CountChannelFiles(dir_->GetFileSystem(), temp_directory)); + ASSERT_OK_AND_ASSIGN(RealtimeMetricValues usage, ReadRealtimeMetrics(writer->GetMetrics())); + ASSERT_EQ(0, usage.building_rows); + ASSERT_EQ(4, usage.sealed_rows); + ASSERT_GT(usage.sealed_memory, 0); + const std::vector expected = + merge_versions + ? std::vector{{1, "new", "p0"}, {2, "two", "p0"}, {3, "three", "p0"}} + : std::vector{ + {1, "old", "p0"}, {2, "two", "p0"}, {3, "three", "p0"}, {4, "four", "p0"}}; + ASSERT_OK_AND_ASSIGN(std::vector visible, ReadRows(context)); + std::sort(visible.begin(), visible.end()); + ASSERT_EQ(expected, visible); + ASSERT_OK(writer->Seal()); + ASSERT_OK_AND_ASSIGN(std::vector empty, + writer->PrepareCommitWithProgress(/*commit_identifier=*/1)); + ASSERT_TRUE(empty.empty()); + ASSERT_EQ(expected_spill_files, + TestHelper::CountChannelFiles(dir_->GetFileSystem(), temp_directory)); + + ASSERT_OK_AND_ASSIGN(int64_t snapshot, Commit(progress, /*commit_identifier=*/1)); + std::shared_ptr disk_context; + if (primary_key && deletion_vectors) { + // Ordinary PK DV scans skip L0; an empty context reads it without any memory tail. + ASSERT_OK_AND_ASSIGN(disk_context, RealtimeContext::Create()); + } + ASSERT_OK_AND_ASSIGN(visible, ReadRows(disk_context)); + std::sort(visible.begin(), visible.end()); + ASSERT_EQ(expected, visible); + ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot)); + ASSERT_TRUE(WaitForChannelFileCount(temp_directory, /*expected_count=*/0)); + ASSERT_OK_AND_ASSIGN(usage, ReadRealtimeMetrics(writer->GetMetrics())); + ASSERT_EQ(0, usage.sealed_memory); + ASSERT_EQ(0, usage.total_rows); + ASSERT_OK_AND_ASSIGN(visible, ReadRows(context)); + std::sort(visible.begin(), visible.end()); + ASSERT_EQ(expected, visible); + ASSERT_OK(writer->Close()); +} + +INSTANTIATE_TEST_SUITE_P(Prepare, RealtimePrepareSpillInteTest, + ::testing::Combine(::testing::Bool(), ::testing::Bool(), + ::testing::Bool())); + TEST_F(RealtimeWriteInteTest, TestAppendRealtimeSpillFileLifecycle) { options_[Options::REALTIME_SPILL_ENABLED] = "true"; CreateTable(/*partition_keys=*/{}); @@ -6828,6 +7184,58 @@ TEST_F(RealtimeWriteInteTest, TestAppendRealtimeSpillFileLifecycle) { ASSERT_OK(writer->Close()); } +TEST_F(RealtimeWriteInteTest, TestDeduplicateRealtimeSpillFileLifecycle) { + options_[Options::REALTIME_SPILL_ENABLED] = "true"; + ConfigureRealtimeDeduplicateTable(); + const std::string temp_directory = PathUtil::JoinPath(dir_->Str(), "deduplicate-spill"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr context, RealtimeContext::Create()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr writer, + CreateRealtimeWriter(context, temp_directory)); + const std::vector initial_rows = {{1, "old", "p0"}, {2, "two", "p0"}}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr initial, + MakeBatch(initial_rows, /*partitioned=*/false)); + ASSERT_OK(writer->Write(std::move(initial))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr pinned, CreatePlan(context, /*predicate=*/nullptr)); + ASSERT_OK(writer->Seal()); + ASSERT_EQ(1, TestHelper::CountChannelFiles(dir_->GetFileSystem(), temp_directory)); + ASSERT_OK_AND_ASSIGN(RealtimeMetricValues sealed_usage, + ReadRealtimeMetrics(writer->GetMetrics())); + // Arrow payloads have spilled, but the sealed key index still occupies memory. + ASSERT_GT(sealed_usage.sealed_memory, 0); + ASSERT_GT(sealed_usage.building_memory, 0); + ASSERT_EQ(2, sealed_usage.sealed_rows); + ASSERT_EQ(0, sealed_usage.building_rows); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr updates, + MakeBatch({{1, "new", "p0"}, {2, "deleted", "p0"}, {3, "three", "p0"}}, + /*partitioned=*/false, /*bucket=*/0, + {RecordBatch::RowKind::INSERT, RecordBatch::RowKind::DELETE, + RecordBatch::RowKind::INSERT})); + ASSERT_OK(writer->Write(std::move(updates))); + const std::vector expected = {{1, "new", "p0"}, {3, "three", "p0"}}; + ASSERT_OK_AND_ASSIGN(std::vector visible, ReadRows(context)); + ASSERT_EQ(expected, visible); + ASSERT_OK_AND_ASSIGN(std::vector progress, + writer->PrepareCommitWithProgress(/*commit_identifier=*/0)); + ASSERT_EQ(1, progress.size()); + ASSERT_EQ("{0,1}", progress[0].deleted_offsets->ToString()); + ASSERT_OK_AND_ASSIGN(int64_t snapshot, Commit(progress, /*commit_identifier=*/0)); + ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot)); + ASSERT_OK_AND_ASSIGN(visible, ReadRows(context)); + ASSERT_EQ(expected, visible); + ASSERT_OK_AND_ASSIGN(RealtimeMetricValues reclaimed_usage, + ReadRealtimeMetrics(writer->GetMetrics())); + ASSERT_EQ(0, reclaimed_usage.sealed_memory); + ASSERT_EQ(0, reclaimed_usage.total_rows); + // The current, empty building key index retains its allocated buckets. + ASSERT_GT(reclaimed_usage.building_memory, 0); + ASSERT_EQ(reclaimed_usage.building_memory, reclaimed_usage.total_memory); + ASSERT_OK_AND_ASSIGN(visible, ReadRows(pinned, context)); + ASSERT_EQ(initial_rows, visible); + ASSERT_TRUE(WaitForChannelFileCount(temp_directory, /*expected_count=*/0)); + ASSERT_OK(writer->Close()); +} + TEST_F(RealtimeWriteInteTest, TestPkDvRealtimeSpillFileLifecycle) { options_[Options::FILE_FORMAT] = "parquet"; options_[Options::DELETION_VECTORS_ENABLED] = "true";