From 7318b3ee9449cec5584dd76c388fef825eb6186c Mon Sep 17 00:00:00 2001 From: Ahmed Abualsaud Date: Thu, 17 Sep 2026 08:57:47 -0700 Subject: [PATCH 1/5] top api layer --- .../apache/beam/sdk/io/iceberg/IcebergIO.java | 18 + .../sdk/io/iceberg/IcebergWriteResult.java | 85 + .../sdk/io/iceberg/cdc/sink/WriteCdcRows.java | 521 ++++++ .../sdk/io/iceberg/cdc/sink/package-info.java | 233 ++- .../cdc/sink/WriteCdcRowsReadBackTest.java | 1526 +++++++++++++++++ .../io/iceberg/cdc/sink/WriteCdcRowsTest.java | 1499 ++++++++++++++++ 6 files changed, 3880 insertions(+), 2 deletions(-) create mode 100644 sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRows.java create mode 100644 sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRowsReadBackTest.java create mode 100644 sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRowsTest.java diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergIO.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergIO.java index 080c31ff9126..d2119a241e3e 100644 --- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergIO.java +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergIO.java @@ -26,6 +26,7 @@ import org.apache.beam.sdk.annotations.Internal; import org.apache.beam.sdk.io.Read; import org.apache.beam.sdk.io.iceberg.cdc.IncrementalChangelogSource; +import org.apache.beam.sdk.io.iceberg.cdc.sink.WriteCdcRows; import org.apache.beam.sdk.options.StreamingOptions; import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.transforms.PTransform; @@ -393,6 +394,23 @@ public static WriteRows writeRows(IcebergCatalogConfig catalog) { .build(); } + /** + * Returns a {@link WriteCdcRows} transform: the CDC sink, applying inserts/updates/deletes from a + * {@code PCollection} of change records (each carrying a {@link + * org.apache.beam.sdk.values.ValueKind}) to one or more Iceberg V2+ tables via equality deletes; + * superseded rows are never written. + * + * 
{@code
+   * input.apply(IcebergIO.writeCdcRows(catalogConfig)
+   *     .to(tableId)
+   *     .withSequenceNumberColumn("seq")
+   *     .withTriggeringFrequency(Duration.standardMinutes(1)));
+   * }
+ */ + public static WriteCdcRows writeCdcRows(IcebergCatalogConfig catalog) { + return WriteCdcRows.of(catalog); + } + @AutoValue public abstract static class WriteRows extends PTransform, IcebergWriteResult> { diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergWriteResult.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergWriteResult.java index 8e2549b5dadb..8db954428b3f 100644 --- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergWriteResult.java +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergWriteResult.java @@ -19,31 +19,110 @@ import java.util.Map; import org.apache.beam.sdk.Pipeline; +import org.apache.beam.sdk.annotations.Internal; import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PInput; import org.apache.beam.sdk.values.POutput; import org.apache.beam.sdk.values.PValue; +import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.checkerframework.checker.nullness.qual.Nullable; +/** + * The output of an {@code IcebergIO} write: the snapshots each destination table committed, plus + * the two diversion outputs the CDC sink can produce. + * + *

Only {@link #getSnapshots()} is always present. {@link #getDeadLetterRows()} (late-but-valid + * records) and {@link #getFailedRows()} (per-record poison rows) are non-null only for results + * built by {@link #cdc}, that is, by {@code IcebergIO.writeCdcRows}, and {@code getFailedRows()} + * additionally only when error handling was enabled. The append-only sink ({@code + * IcebergIO.writeRows}) leaves both null rather than exposing outputs that can never carry data. + */ public final class IcebergWriteResult implements POutput { private static final TupleTag> SNAPSHOTS_TAG = new TupleTag>() {}; + private static final TupleTag DEAD_LETTER_TAG = new TupleTag() {}; + + private static final TupleTag FAILED_ROWS_TAG = new TupleTag() {}; + private final Pipeline pipeline; private final PCollection> snapshots; + private final @Nullable PCollection deadLetterRows; + + private final @Nullable PCollection failedRows; + + /** + * The committed snapshots, keyed by destination. A window committed in a commit fire that later + * fails may not re-emit its {@link SnapshotInfo} on the retry (table state is unaffected). + */ public PCollection> getSnapshots() { return snapshots; } + /** + * The replayable dead-letter {@link Row}s from the CDC sink: records whose grouped pane fired + * late, i.e. after the watermark had passed their commit window's end. Schema is {@code record + * ROW + change_type STRING + sequence_number INT64 + destination STRING}. To replay, + * unnest {@code record}, map {@code change_type}/{@code sequence_number} as the sink's control + * columns, and route each row by {@code destination}. Replaying is only safe while no newer + * change for those keys has committed; a stale replay's equality delete removes the newer row. + * + * @return the dead-letter {@code PCollection} for results produced by {@code + * IcebergIO.writeCdcRows}/{@link #cdc}, or {@code null} for results produced by the + * append-only sink ({@code IcebergIO.writeRows}), which has no dead-letter output. + */ + public @Nullable PCollection getDeadLetterRows() { + return deadLetterRows; + } + + /** + * The per-record poison rows diverted by the CDC sink when error handling is enabled (see {@code + * WriteCdcRows.withErrorHandling}); schema is {@code failed_row ROW + error_message STRING} (see + * {@link org.apache.beam.sdk.schemas.transforms.providers.ErrorHandling}). Distinct from {@link + * #getDeadLetterRows()} (which carries late-but-valid records). + * + * @return the failed-rows {@code PCollection}, or {@code null} when error handling was not + * enabled (or for the append-only sink). + */ + public @Nullable PCollection getFailedRows() { + return failedRows; + } + IcebergWriteResult(Pipeline pipeline, PCollection> snapshots) { + this(pipeline, snapshots, null, null); + } + + private IcebergWriteResult( + Pipeline pipeline, + PCollection> snapshots, + @Nullable PCollection deadLetterRows, + @Nullable PCollection failedRows) { this.pipeline = pipeline; this.snapshots = snapshots; + this.deadLetterRows = deadLetterRows; + this.failedRows = failedRows; + } + + /** + * Returns an {@link IcebergWriteResult} for the CDC sink, exposing the committed-snapshot {@code + * snapshots}, the replayable {@code deadLetterRows} (see {@link #getDeadLetterRows()}), and the + * optional per-record {@code failedRows} (see {@link #getFailedRows()}, {@code null} when error + * handling is off). + */ + @Internal + public static IcebergWriteResult cdc( + Pipeline pipeline, + PCollection> snapshots, + PCollection deadLetterRows, + @Nullable PCollection failedRows) { + return new IcebergWriteResult(pipeline, snapshots, deadLetterRows, failedRows); } @Override @@ -55,6 +134,12 @@ public Pipeline getPipeline() { public Map, PValue> expand() { ImmutableMap.Builder, PValue> output = ImmutableMap.builder(); output.put(SNAPSHOTS_TAG, snapshots); + if (deadLetterRows != null) { + output.put(DEAD_LETTER_TAG, deadLetterRows); + } + if (failedRows != null) { + output.put(FAILED_ROWS_TAG, failedRows); + } return output.build(); } diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRows.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRows.java new file mode 100644 index 000000000000..20a44dc1c7cb --- /dev/null +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRows.java @@ -0,0 +1,521 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.iceberg.cdc.sink; + +import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull; + +import com.google.auto.value.AutoValue; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.stream.Stream; +import org.apache.beam.sdk.annotations.Internal; +import org.apache.beam.sdk.io.iceberg.DynamicDestinations; +import org.apache.beam.sdk.io.iceberg.IcebergCatalogConfig; +import org.apache.beam.sdk.io.iceberg.IcebergWriteResult; +import org.apache.beam.sdk.io.iceberg.SnapshotInfo; +import org.apache.beam.sdk.schemas.Schema; +import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.values.KV; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.PCollectionTuple; +import org.apache.beam.sdk.values.Row; +import org.apache.beam.sdk.values.ValueKind; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Predicates; +import org.apache.iceberg.catalog.TableIdentifier; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.joda.time.Duration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The top-level CDC sink transform: applies a collection of change records to one or more Iceberg + * V2+ tables: + * + * 

{@code
+ * changeRecords.apply(IcebergIO.writeCdcRows(catalogConfig)
+ *     .to(tableId)
+ *     .withSequenceNumberColumn("seq")
+ *     .withTriggeringFrequency(Duration.standardMinutes(1)));
+ * }
+ * + *

Input contract

+ * + *

Each input {@link Row} is one change record. Its change kind is its native {@link ValueKind}, + * but can be overridden with a string column value using {@link #withChangeTypeColumn}, optionally + * translated via {@link #withChangeTypeMap}. Each row must also carry a per-key monotonic + * (long) sequence number specified by {@link #withSequenceNumberColumn}, which orders a single + * primary key's changes; the column is required in the input schema as a non-nullable {@code + * INT64}; set defaults upstream. These control columns are stripped from the input rows before + * writing to the table. + * + *

Ordering requirement (not validated): for a given primary key, the input element's + * event-time must be non-decreasing with its (sequence number, kind rank): a higher-sequence change + * never carries an earlier event time, and equal-sequence records (an update's before and after + * images) carry equal event times, so neither half lands in an earlier commit window. A violation + * can corrupt final table state (a lower-sequence equality delete deleting a higher-sequence row + * committed in a later snapshot). + * + *

Semantics

+ * + *

The sink commits one new snapshot to the destination table per commit window, in ascending + * window order, with an idempotency token written to each snapshot's summary. A retried or + * restarted commit finds the token and skips already-committed windows, making the commit + * effectively-once on runners that honor {@code @RequiresStableInput}. Records whose grouped pane + * fires late (watermark is past their commit window's end) are diverted to the DLQ output, + * accessible with {@link IcebergWriteResult#getDeadLetterRows()}. + * + *

Each dead-lettered record nests the data row under {@code record}, beside {@code change_type}, + * {@code sequence_number}, and {@code destination}. + * + *

Sink id

+ * + *

The sink creates a fresh unique id by default. All commits in a single pipeline run share the + * same id. Commits get stamped with the sink id and a window-end millis token. Set {@link + * #withSinkId} explicitly (and keep it stable) for cross-relaunch idempotency. Don't reuse sink ids + * for batch runs though because all commits fall under a single global window, so a second load + * with the same sink id will recognize the same window-end millis token and skip the commit. A + * batch load's sink id must likewise not be carried into a streaming continuation. + */ +@Internal +@AutoValue +public abstract class WriteCdcRows extends PTransform, IcebergWriteResult> { + + private static final Logger LOG = LoggerFactory.getLogger(WriteCdcRows.class); + + public static final Duration DEFAULT_ALLOWED_LATENESS = Duration.standardHours(6); + + abstract IcebergCatalogConfig getCatalogConfig(); + + abstract @Nullable TableIdentifier getTableIdentifier(); + + abstract @Nullable DynamicDestinations getDynamicDestinations(); + + abstract @Nullable List getEqualityColumns(); + + abstract String getSequenceNumberColumn(); + + abstract @Nullable String getChangeTypeColumn(); + + abstract @Nullable Map getChangeTypeMap(); + + abstract int getNumShards(); + + abstract @Nullable Integer getShardsPerPartition(); + + abstract int getSorterMemoryMB(); + + abstract boolean getUpsert(); + + abstract @Nullable Long getTokenHeartbeatMillis(); + + abstract boolean getErrorHandlingEnabled(); + + abstract @Nullable Map getSnapshotProperties(); + + /** The sink id namespacing the {@code beam.cdc.} snapshot-summary tokens. */ + abstract String getSinkId(); + + abstract @Nullable Duration getTriggeringFrequency(); + + abstract @Nullable Duration getAllowedLateness(); + + abstract Builder toBuilder(); + + @AutoValue.Builder + abstract static class Builder { + abstract Builder setCatalogConfig(IcebergCatalogConfig catalogConfig); + + abstract Builder setTableIdentifier(TableIdentifier tableIdentifier); + + abstract Builder setDynamicDestinations(DynamicDestinations destinations); + + abstract Builder setEqualityColumns(List equalityColumns); + + abstract Builder setSequenceNumberColumn(String sequenceNumberColumn); + + abstract Builder setChangeTypeColumn(String changeTypeColumn); + + abstract Builder setChangeTypeMap(Map changeTypeMap); + + abstract Builder setNumShards(int numShards); + + abstract Builder setShardsPerPartition(Integer shardsPerPartition); + + abstract Builder setSorterMemoryMB(int sorterMemoryMB); + + abstract Builder setUpsert(boolean upsert); + + abstract Builder setTokenHeartbeatMillis(@Nullable Long tokenHeartbeatMillis); + + abstract Builder setErrorHandlingEnabled(boolean errorHandlingEnabled); + + abstract Builder setSnapshotProperties(Map snapshotProperties); + + abstract Builder setSinkId(String sinkId); + + abstract Builder setTriggeringFrequency(Duration triggeringFrequency); + + abstract Builder setAllowedLateness(Duration allowedLateness); + + abstract WriteCdcRows build(); + } + + public static WriteCdcRows of(IcebergCatalogConfig catalogConfig) { + return new AutoValue_WriteCdcRows.Builder() + .setCatalogConfig(catalogConfig) + .setSequenceNumberColumn(CdcWriteConfig.DEFAULT_SEQUENCE_NUMBER_COLUMN) + .setNumShards(CdcWriteConfig.DEFAULT_NUM_SHARDS) + .setSorterMemoryMB(CdcWriteConfig.DEFAULT_SORTER_MEMORY_MB) + .setUpsert(false) + .setErrorHandlingEnabled(false) + .setSinkId(UUID.randomUUID().toString()) + .build(); + } + + /** Writes to a single table. Mutually exclusive with {@link #to(DynamicDestinations)}. */ + public WriteCdcRows to(TableIdentifier tableIdentifier) { + return toBuilder().setTableIdentifier(tableIdentifier).build(); + } + + /** + * Writes to multiple tables. Mutually exclusive with {@link #to(TableIdentifier)}. + * + *

The sink reads the control columns from the raw element and writes {@link + * DynamicDestinations#getData}, whose schema must match the destination table's and must exclude + * control columns. + */ + public WriteCdcRows to(DynamicDestinations destinations) { + return toBuilder().setDynamicDestinations(destinations).build(); + } + + /** + * Columns that define a row's identity (the Iceberg equality-delete fields). Defaults to the + * destination table's identifier (primary-key) fields. Tables may be partitioned on non-key + * columns; partition source columns must be equality columns only under {@link #withUpsert} or a + * {@link #withShardsPerPartition} cap. + */ + public WriteCdcRows withEqualityColumns(List columns) { + return toBuilder().setEqualityColumns(columns).build(); + } + + /** + * The column holding the per-primary-key monotonic sequence number used to order a single key's + * changes. Must be declared as a non-nullable {@code INT64} in the input schema. The column is + * stripped from the written rows. Defaults to {@value + * CdcWriteConfig#DEFAULT_SEQUENCE_NUMBER_COLUMN}. + */ + public WriteCdcRows withSequenceNumberColumn(String column) { + return toBuilder().setSequenceNumberColumn(column).build(); + } + + /** + * When set, reads the change kind from this column instead of the element's native {@link + * ValueKind}. Must be declared as a non-nullable {@code STRING} in the input schema. The column + * is stripped from the written rows. + */ + public WriteCdcRows withChangeTypeColumn(String column) { + return toBuilder().setChangeTypeColumn(column).build(); + } + + /** + * Mapping from {@link #withChangeTypeColumn} values to {@link ValueKind} names (e.g. for + * Debezium: {@code {"c": "INSERT", "u": "UPDATE_AFTER", "d": "DELETE"}}). Requires {@link + * #withChangeTypeColumn} to also be set. + */ + public WriteCdcRows withChangeTypeMap(Map changeTypeMap) { + return toBuilder().setChangeTypeMap(changeTypeMap).build(); + } + + /** + * The number of deterministic primary-key-hash shards per destination. Controls the sink's + * write-parallelism knob. Defaults to {@value CdcWriteConfig#DEFAULT_NUM_SHARDS}. + * + *

Too low may cause a write bottleneck with a growing commit backlog, too high increases the + * sink's output file count ({@code num_shards x touched partitions} files per commit window). + * + *

On a partitioned table, a commit window writes up to this many files per touched partition. + * {@link #withShardsPerPartition} lowers that per-partition count while leaving this number + * unchanged, so the file count drops without lowering the ceiling on total write parallelism. + * Each individual partition is then written by at most that many shards. + */ + public WriteCdcRows withNumShards(int numShards) { + return toBuilder().setNumShards(numShards).build(); + } + + /** + * Caps how many shards one partition may occupy (default: {@code num_shards}, i.e. all shards may + * write to all partitions). A lower cap reduces write parallelism per partition but also + * concentrates the writes in fewer files. A cap of {@code 1} pins each partition to a single + * writer. Ignored for unpartitioned tables. A cap below {@code num_shards} requires every + * partition source column to be an equality column, making the partition (and with it the shard) + * a pure function of the primary key. + */ + public WriteCdcRows withShardsPerPartition(int shardsPerPartition) { + return toBuilder().setShardsPerPartition(shardsPerPartition).build(); + } + + /** + * The in-memory buffer size (MB) for the sorter that orders each shard's records by primary key, + * then sequence number, then change kind, before writing; groups larger than this spill to disk. + * Must be {@code >= 1}. Defaults to {@value CdcWriteConfig#DEFAULT_SORTER_MEMORY_MB}. + */ + public WriteCdcRows withSorterMemoryMB(int sorterMemoryMB) { + Preconditions.checkArgument( + sorterMemoryMB >= 1, "sorter_memory_mb must be >= 1, got %s", sorterMemoryMB); + return toBuilder().setSorterMemoryMB(sorterMemoryMB).build(); + } + + /** + * If {@code true}, only the after-image of each change ({@code INSERT}/{@code UPDATE_AFTER}) is + * required; {@code UPDATE_BEFORE} records are dropped and {@code INSERT}/{@code UPDATE_AFTER} are + * applied as upserts (equality-delete-then-insert on the primary key). Defaults to {@code false}. + * Requires every partition source column to be an equality column: with before-images dropped, a + * row that moved partitions could never be deleted from its old one. + */ + public WriteCdcRows withUpsert(boolean upsert) { + return toBuilder().setUpsert(upsert).build(); + } + + /** + * Enables a periodic empty token-refresh (heartbeat) commit for each destination with a + * committed-through token, whether committed in this run or recovered from the table: while idle, + * the committer re-writes the token into a fresh snapshot every {@code interval}, so the + * token-bearing snapshot stays recent and is less likely to be lost to {@code expire_snapshots} + * before the sink resumes. Disabled by default, and ignored for bounded (batch) input. + * + *

With heartbeat enabled prefer cancel-and-resubmit over drain: the self-re-arming + * processing-time timer can keep a drain from completing. + */ + public WriteCdcRows withTokenHeartbeat(Duration interval) { + Preconditions.checkArgument( + interval != null && interval.getMillis() > 0, "token heartbeat interval must be positive"); + return toBuilder().setTokenHeartbeatMillis(interval.getMillis()).build(); + } + + /** + * Enables per-record error handling. Poison records (unknown change type, missing/null sequence + * number, null equality value, an unresolvable destination) are diverted to {@link + * IcebergWriteResult#getFailedRows()} (schema {@code failed_row ROW + error_message STRING}). + * Defaults to off where the DoFn throws an error instead. Diversion covers poison detectable in + * the assignment stage; a failure later in the pipeline fails and retries its bundle as usual. + */ + public WriteCdcRows withErrorHandling() { + return toBuilder().setErrorHandlingEnabled(true).build(); + } + + /** + * Extra user properties to add to every commit's Iceberg snapshot summary. Keys prefixed with + * {@code beam.cdc.} are reserved for the sink's own idempotency/diagnostic tokens and are + * rejected at construction. + */ + public WriteCdcRows withSnapshotProperties(Map snapshotProperties) { + return toBuilder().setSnapshotProperties(snapshotProperties).build(); + } + + /** + * A stable identifier for this sink, used to namespace the idempotency tokens written to each + * commit's Iceberg snapshot summary ({@code beam.cdc.sink-id}, {@code + * beam.cdc.committed-through-ms.}). Mostly useful when re-using the same {@code sinkId} + * across subsequent streaming relaunches to maintain exactly-once commit idempotency from the + * same source. If unset, a fresh UUID is used per launch. + * + *

Also useful in batch to make runs idempotent for the same {@code sinkId}. A stable {@code + * sinkId} should only be used once though. In batch, the idempotency token is always + * global-window end, so re-using it will lead to no-op for subsequent loads (they will be skipped + * at commit time). Do not carry a batch job's {@code sinkId} into a streaming job, + * as it would skip every commit window forever. + */ + public WriteCdcRows withSinkId(String sinkId) { + return toBuilder().setSinkId(sinkId).build(); + } + + /** The size of each event-time commit window (streaming only). */ + public WriteCdcRows withTriggeringFrequency(Duration triggeringFrequency) { + return toBuilder().setTriggeringFrequency(triggeringFrequency).build(); + } + + /** + * How far behind the watermark an element's event-time may be before it is dropped entirely, + * otherwise routed to {@link IcebergWriteResult#getDeadLetterRows()} as a late firing. Defaults + * to {@link #DEFAULT_ALLOWED_LATENESS} (6 hours) if unset. + */ + public WriteCdcRows withAllowedLateness(Duration allowedLateness) { + return toBuilder().setAllowedLateness(allowedLateness).build(); + } + + private CdcWriteConfig cdcWriteConfig() { + @Nullable Integer shardsPerPartition = getShardsPerPartition(); + return CdcWriteConfig.builder() + .setEqualityColumns(getEqualityColumns()) + .setSequenceNumberColumn(getSequenceNumberColumn()) + .setChangeTypeColumn(getChangeTypeColumn()) + .setChangeTypeMap(getChangeTypeMap()) + .setNumShards(getNumShards()) + .setShardsPerPartition(shardsPerPartition == null ? getNumShards() : shardsPerPartition) + .setSorterMemoryMB(getSorterMemoryMB()) + .setUpsert(getUpsert()) + .setTokenHeartbeatMillis(getTokenHeartbeatMillis()) + .setSinkId(getSinkId()) + .setSnapshotProperties(getSnapshotProperties()) + .setErrorHandling(getErrorHandlingEnabled()) + .build(); + } + + /** Construction-time validation. */ + private void validate(PCollection input, CdcWriteConfig config) { + Preconditions.checkArgument( + 1 + == Stream.of(getTableIdentifier(), getDynamicDestinations()) + .filter(Predicates.notNull()) + .count(), + "Must set exactly one of to(TableIdentifier) or to(DynamicDestinations)."); + + // Structural config invariants (value bounds, mutually exclusive options, reserved names). + config.validate(); + + boolean unbounded = input.isBounded() == PCollection.IsBounded.UNBOUNDED; + @Nullable Duration triggeringFrequency = getTriggeringFrequency(); + if (unbounded) { + if (triggeringFrequency == null) { + throw new IllegalArgumentException( + "Streaming CDC writes require withTriggeringFrequency(...)"); + } + Preconditions.checkArgument( + triggeringFrequency.isLongerThan(Duration.ZERO), + "triggering frequency must be positive for streaming CDC writes, got %s", + triggeringFrequency); + } + + @Nullable String changeTypeColumn = config.getChangeTypeColumn(); + if (changeTypeColumn != null) { + if (!input.getSchema().hasField(changeTypeColumn)) { + throw new IllegalArgumentException( + "CDC input schema is missing the change-type column '" + + changeTypeColumn + + "' named by change_type_column. Check the spelling, or drop change_type_column " + + "to use native change kind. Input schema: " + + input.getSchema()); + } + Schema.FieldType changeTypeType = input.getSchema().getField(changeTypeColumn).getType(); + if (changeTypeType.getTypeName() != Schema.TypeName.STRING) { + throw new IllegalArgumentException( + "CDC change-type column '" + + changeTypeColumn + + "' must be STRING, but the input schema declares it " + + changeTypeType + + "."); + } + if (changeTypeType.getNullable()) { + throw new IllegalArgumentException( + "CDC change-type column '" + + changeTypeColumn + + "' must be non-nullable, but the input schema declares it nullable."); + } + } + + String seqCol = config.getSequenceNumberColumn(); + if (!input.getSchema().hasField(seqCol)) { + throw new IllegalArgumentException( + "CDC input schema is missing the sequence-number column '" + + seqCol + + "', required to order each primary key's changes. Input schema: " + + input.getSchema()); + } + Schema.FieldType seqType = input.getSchema().getField(seqCol).getType(); + if (seqType.getTypeName() != Schema.TypeName.INT64) { + throw new IllegalArgumentException( + "CDC sequence-number column '" + + seqCol + + "' must be INT64, but the input schema declares it " + + seqType + + "."); + } + if (seqType.getNullable()) { + throw new IllegalArgumentException( + "CDC sequence-number column '" + + seqCol + + "' must be non-nullable, but the input schema declares it nullable."); + } + + // The idle token-refresh heartbeat is a streaming-only concept. + if (!unbounded && config.getTokenHeartbeatMillis() != null) { + LOG.warn( + "Token heartbeat is ignored for bounded input. Batch loads will commit once and exit."); + } + } + + @Override + public IcebergWriteResult expand(PCollection input) { + CdcWriteConfig config = cdcWriteConfig(); + validate(input, config); + + @Nullable DynamicDestinations destinations = getDynamicDestinations(); + if (destinations == null) { + destinations = + SingleTableDestinations.of( + checkStateNotNull(getTableIdentifier()), input.getSchema(), config); + } + + boolean unbounded = input.isBounded() == PCollection.IsBounded.UNBOUNDED; + @Nullable Duration allowedLateness = getAllowedLateness(); + + String runId = UUID.randomUUID().toString(); + + // Stage 1: destination + shard/sort-key assignment, and handle error rows + PCollectionTuple assigned = + input.apply( + "AssignCdcKeys", new AssignCdcKeys(getCatalogConfig(), config, destinations, runId)); + PCollection>> keyed = + assigned.get(AssignCdcKeys.KEYED); + // getFailedRows() is null when error handling is disabled + @Nullable PCollection failedRows = + getErrorHandlingEnabled() ? assigned.get(AssignCdcKeys.FAILED) : null; + + // Stage 2: create commit windows, group by (destination, shard) and split late rows to the + // replayable dead-letter, and sort groups by (pk, seq, kind) + CommitWindows.Result windowed = + keyed.apply( + "CommitWindows", + new CommitWindows( + config, + getTriggeringFrequency(), + allowedLateness != null ? allowedLateness : DEFAULT_ALLOWED_LATENESS)); + + // Stage 3: write delta files and emit their serialized metadata, one element per shard. + PCollection written = + windowed + .getSortedGroups() + .apply("WriteDeltas", new WriteDeltas(getCatalogConfig(), config, destinations, runId)); + + // Commit stage: ordered, idempotent commit + PCollection> snapshots = + written.apply( + "CommitDeltas", + new CommitDeltas( + getCatalogConfig(), + getSinkId(), + config.getSnapshotProperties(), + unbounded ? config.getTokenHeartbeatMillis() : null, + runId)); + + return IcebergWriteResult.cdc( + input.getPipeline(), snapshots, windowed.getDeadLetterRows(), failedRows); + } +} diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/package-info.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/package-info.java index 2bcd8b5f6222..c25496648f98 100644 --- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/package-info.java +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/package-info.java @@ -17,7 +17,236 @@ */ /** - * A change data capture (CDC) write sink for Apache Iceberg: applies inserts, updates, and deletes - * to a table, identifying rows by equality columns. + * Iceberg CDC sink: applies a stream or batch of inserts, updates and deletes to Iceberg V2+ + * tables, exposed as {@code IcebergIO.writeCdcRows(catalogConfig)}. For the transform's API and + * semantics see {@code WriteCdcRows}. + * + *

This is a merge-on-read sink. Within one commit window the writer collapses each primary key's + * changes to their final state, so a superseded intermediate row is never written at all; a change + * that reaches back into an already-committed snapshot is written as a PK-only equality + * delete. No position deletes or deletion vectors are ever written. + * + *

Monitoring

+ * + *

All metrics below are queryable through {@code PipelineResult.metrics()}; the namespace is the + * owning class's fully-qualified name, so filter on {@code + * org.apache.beam.sdk.io.iceberg.cdc.sink.*}. + * + *

Committer volume and latency

+ * + *

The normal throughput picture. None of these indicate a problem by themselves; watch their + * shape over time. + * + *

    + *
  • snapshotsCreated: CDC snapshots committed. In streaming this should track one per + * commit window per destination. A flat line while data is flowing means commits have + * stalled. + *
  • committedDataFiles / committedDeleteFiles: files published. Divide by {@code + * snapshotsCreated} for files per commit. A large ratio points to the small-file problem + * described below. + *
  • committedRecords: rows committed. + *
  • committedEqualityDeleteRecords: equality-delete rows committed. + *
  • committedBytes: total bytes of committed files. + *
  • commitDurationMs (distribution): wall-clock time spent waiting on the Iceberg commit + * operation. Its tail is your catalog's health: a growing maximum usually means catalog + * contention, not a Beam problem. + *
  • commitFailures: fires whenever the commit path throws. Iceberg's own optimistic + * retry runs underneath, so a genuine commit collision is counted only once that is + * exhausted. The failure is rethrown, so the bundle fails and the runner retries it; + * occasional increments under heavy concurrent writing are survivable. A sustained rate means + * the table has too many concurrent writers, and the sink will not make progress past the + * failing window (commits are strictly ordered). + *
  • heartbeatCommits: empty token-refresh commits emitted while a destination is idle + * (only if {@code withTokenHeartbeat} is configured). Each one is a real snapshot. With + * heartbeat enabled prefer cancel-and-resubmit over drain: the self-re-arming processing-time + * timer can keep a drain from completing. + *
+ * + *

Safety tripwires

+ * + *

These should be zero. A nonzero value is not necessarily an outage, but each one means + * something specific happened that you want to know about. + * + *

    + *
  • alreadyCommittedWindowsSkipped: a commit window was skipped because the table + * already carried a committed-through token at or past its end. Expected and benign after a + * restart or a retried bundle (we skip to ensure idempotency). Alarming in two cases: (1) in + * batch, where the whole load commits under one window, a nonzero value on a fresh + * load means a stable {@code sink_id} matched a previous load and this run wrote + * nothing at all. Use a unique {@code sink_id} per load, or omit it. Never carry a batch + * load's {@code sink_id} into a subsequent streaming run (the batch token is the + * global-window end, above every streaming window; the streaming committer will refuse a + * recovered batch token); and (2) in streaming, a persistently nonzero rate means data keeps + * arriving for windows that already closed, which is a source-lateness problem. The token + * proves that a window with that end committed, not that the skipped rows are the rows that + * committed. A logged warning names up to 5 of the window's files plus a count: your + * handle on the skipped rows until {@code remove_orphan_files} reclaims them. + *
  • orphanFiles: files (data and delete) carried by a skipped window: potential + * orphans (a pure redelivery's files are the committed live ones). Wasted storage until + * {@code remove_orphan_files} runs. + *
  • tokenParseFailures: an unparseable {@code beam.cdc.*} value was found while scanning + * snapshot ancestry. The committed-through scan continues to older ancestors rather than + * crash-looping; a bad max-seq or run-spec stamp just reads as absent. Nonzero means + * something wrote a malformed value into a snapshot summary; investigate before trusting the + * recovered position. + *
  • suspectedTokenExpiry: the sink's own {@code beam.cdc.sink-id} marker was found in + * the ancestry but no committed-through token was. Strongly implies that {@code + * expire_snapshots} removed the token-bearing snapshots while the pipeline was down. Recovery + * then falls back to the beginning and may re-apply retained windows. If you see this, either + * lengthen snapshot retention or enable {@code withTokenHeartbeat} so the token stays young. + *
  • crossWindowSequenceInversions: a committed window's minimum source sequence number + * was below an earlier window's committed maximum. This is the detector for a violated + * ordering contract (see below) and the only tripwire here that can mean silently wrong + * table contents. It has benign false positives when the two windows touch entirely + * disjoint primary keys, so treat it as "go and check the source's ordering", not as proof of + * corruption. + *
  • specMismatchedWindows: signals the partition spec was evolved mid-run. Incremented + * when a committed window carries equality deletes and uses partition-spec ids different from + * the run's pinned spec. Those deletes may not reach rows written under the other spec. + *
+ * + *

Input health

+ * + *

These describe what is arriving, not what the sink did with it. + * + *

    + *
  • deadLetterRecords: records whose grouped pane fired late (the watermark had already + * passed their commit window's end), diverted to the replayable dead-letter output ({@code + * IcebergWriteResult.getDeadLetterRows()}) instead of being applied out of order. + * Every late pane is diverted, including a window's first ({@code SplitLateData}'s + * javadoc explains why). Nonzero means your source is lagging past the commit window. The + * records are not lost, but nothing consumes them unless you wire that output somewhere. + * Records later than {@code withAllowedLateness} are a different case: see below. + *
  • failedRecords: poison records diverted to {@code IcebergWriteResult.getFailedRows()} + * when {@code withErrorHandling()} is on: unknown change type, missing sequence number, null + * equality value, unresolvable destination. Without error handling these fail the pipeline + * instead. Like the dead letters above, the records are not lost, but you need to wire the + * output somewhere to avoid dropping them. + *
  • upsertUpdateBeforeDropped: {@code UPDATE_BEFORE} records discarded because the sink + * is in upsert mode, which needs only after-images. Tells you how much of your input was + * redundant. + *
+ * + *

Records more than {@code withAllowedLateness} behind the watermark are dropped by the runner + * at the {@code GroupByShardKey} step. Raising {@code withAllowedLateness} is the only way to + * capture them (at the cost of more live window state per destination). + * + *

Files and maintenance

+ * + *

File count per commit is governed by {@code num_shards} × touched partitions, and + * total file count by that times the number of commit windows. A table with many partitions and a + * short {@code triggering_frequency_seconds} produces a lot of small files very quickly, regardless + * of the data rate. {@code num_shards} (default=16) is the sink's write-parallelism knob. Too low + * may lead to a write bottleneck and commit backlog. Too high is fine for the sink but may lead to + * slower downstream reads due to small files. + * + *

There are three levers that you can use: + * + *

    + *
  • {@code triggering_frequency_seconds}: increasing it leads to fewer, larger commits, + * at the cost of end-to-end latency. + *
  • {@code num_shards}: trades throughput for file count. Lowering it reduces file + * count, but also the write-parallelism ceiling. + *
  • {@code shards_per_partition} (default={@code num_shards}, i.e. no cap): on a + * partitioned table, cap how many shards one partition's rows may occupy. A touched + * partition then writes about {@code min(shards_per_partition, distinct keys)} files per file + * kind per commit, with its write parallelism capped to match; {@code 1} is the pure + * partition-affine endpoint (one writer, about one data file, per partition). + *
+ * + *

The trade-off a lowered {@code shards_per_partition} makes, stated plainly: effective + * write parallelism per destination becomes at most the cap times the number of distinct partitions + * receiving data in a window: at {@code 1}, one writer per partition. A cap of {@code 1} is right + * for a table with many partitions and wrong for a table with three, which it would hold at + * three concurrent writers however many workers the pipeline has; such a table wants an + * intermediate cap, sized so cap × touched partitions still covers the pipeline's write + * parallelism. The cap is moot for an unpartitioned table, where it is simply ignored (with a WARN) + * rather than funnelling everything through a single shard. At {@code 1} this is the same + * limitation Flink's {@code hash} write-distribution mode carries; the dial exists because the + * alternative (leaving {@code num_shards} as the only lever) forces the same trade with none of the + * benefit. It is safe with {@code upsert}: both options require every partition source column to be + * an equality column (see the partitioning section below), which makes a row's partition a pure + * function of its primary key. + * + *

Snapshot expiry bounds the sink's own cost, not just read performance

+ * + *

This sink creates one snapshot per commit window per table. The following sink costs + * grow as snapshot count grows: + * + *

    + *
  • Commit latency: Every commit fire loads the destination's metadata once, then walks its + * ancestry to find the latest committed-through token. Each commit within that fire publishes + * a snapshot, which requires refreshing and re-parsing the metadata. Both scale with the + * number of snapshots in that file. + *
  • Worker heap: Each cached table pins a {@code TableMetadata} holding every retained + * snapshot, per worker, multiplied by the number of dynamic destinations that worker touches. + *
+ * + *

Orphan files

+ * + *

The writer stage writes data files and delete files and passes their metadata to the committer + * stage in the pipeline. Every orphan is an ordinary data or delete file that no snapshot ended up + * referencing. Orphan files can be produced in three ways: + * + *

    + *
  • A retried or failed writer bundle. Each attempt names its files with a fresh UUID, + * so whichever attempt's output element loses the retry race leaves its files unreferenced. + *
  • A window skipped because the committed-through token already covers its end. This + * happens when a relaunch reuses a stable {@code sink_id}. The incoming element is targeting + * a window that has already been committed with different files. If the incoming element's + * files contain different content than the committed ones, they may hold rows the table never + * received. Those files are left in place so you can recover them. Counted by {@code + * orphanFiles}; the skip WARN names up to 5 of them plus a count. + *
  • A failed commit attempt. Iceberg normally deletes the manifests it wrote when a + * commit fails, but it cannot when the outcome is unknown (i.e. {@code + * CommitStateUnknownException}). The commit may in fact have succeeded, and deleting those + * manifests would corrupt a live snapshot. Only the manifests are left orphaned. The window's + * own data and delete files are untouched, because the pending bag survives the failure and + * the retry commits those same files. + *
+ * + *

All of these are reclaimed by Iceberg's {@code remove_orphan_files}. The sink writes + * everything through the table's own write path. Give the procedure an age threshold much longer + * than your longest in-flight bundle to avoid deleting files a live pipeline is about to reference. + * + *

Partition-spec evolution mid-run

+ * + *

Iceberg matches an equality delete to its data files by {@code (spec id, partition)}, so if a + * table's spec changes during streaming writes, the sink will choose to keep writing with the old + * spec. Otherwise, it could leave new deletes that never reach old-spec rows, which is data + * corruption. If a window happens to still mix specs while carrying equality deletes, it commits + * anyway, WARNs, and increments the {@code specMismatchedWindows} counter. + * + *

If you update a table's spec, run {@code rewrite_data_files} first so old-spec rows are + * rewritten under the new spec. A relaunched pipeline or in-place update will adopt the new spec. + * + *

Partitioning

+ * + *

Tables may be partitioned on any columns, key or not, with any transforms. Two options still + * require every partition source column to be an equality column: {@code upsert} (before-images are + * dropped, so a row that moved partitions could never be deleted from its old one) and a {@code + * shards_per_partition} cap below {@code num_shards} (the shard is derived from the partition + * tuple, which must therefore be a pure function of the primary key). + * + *

Non-key partitioning sharpens the input contract: every update must carry its {@code + * UPDATE_BEFORE}, because a moved row whose before-image never arrives leaves a permanent duplicate + * in the old partition that no later delete reaches; and {@code UPDATE_BEFORE}/{@code DELETE} rows + * must carry the row's actual old values in the partition source columns, because a nulled non-key + * column routes the equality delete to the null partition. + * + *

The ordering contract

+ * + *

The sink orders each primary key's changes by the sequence-number column, and commits one + * snapshot per commit window in ascending window order. That is only sound if the two orderings + * agree at the source: for a given key, an element's event time must be non-decreasing with its + * (sequence number, kind rank), so equal-sequence records (an update's before and after images) + * carry equal event times. If a higher-sequence change lands in an earlier commit window + * than a lower-sequence one, a stale equality delete can be committed after the row it should not + * have touched, and the table's final contents are wrong. + * + *

This is a source contract. Violations are detected at commit time and visible with the {@code + * crossWindowSequenceInversions} counter and warning logs, but the commit still proceeds. The fix + * is in how event times are assigned upstream. Replaying dead letters is bound by the same + * contract: see the replay caveat at {@code IcebergWriteResult#getDeadLetterRows}. */ package org.apache.beam.sdk.io.iceberg.cdc.sink; diff --git a/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRowsReadBackTest.java b/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRowsReadBackTest.java new file mode 100644 index 000000000000..6e18e903e7e7 --- /dev/null +++ b/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRowsReadBackTest.java @@ -0,0 +1,1526 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.iceberg.cdc.sink; + +import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull; +import static org.apache.beam.sdk.values.ValueKind.DELETE; +import static org.apache.beam.sdk.values.ValueKind.INSERT; +import static org.apache.beam.sdk.values.ValueKind.UPDATE_AFTER; +import static org.apache.beam.sdk.values.ValueKind.UPDATE_BEFORE; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.notNullValue; + +import java.io.IOException; +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.time.Instant; +import java.time.LocalDate; +import java.time.OffsetDateTime; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.stream.Collectors; +import org.apache.beam.sdk.PipelineResult; +import org.apache.beam.sdk.coders.CoderException; +import org.apache.beam.sdk.coders.KvCoder; +import org.apache.beam.sdk.coders.RowCoder; +import org.apache.beam.sdk.coders.SerializableCoder; +import org.apache.beam.sdk.io.iceberg.DynamicDestinations; +import org.apache.beam.sdk.io.iceberg.IcebergCatalogConfig; +import org.apache.beam.sdk.io.iceberg.IcebergIO; +import org.apache.beam.sdk.io.iceberg.IcebergUtils; +import org.apache.beam.sdk.metrics.MetricNameFilter; +import org.apache.beam.sdk.metrics.MetricResult; +import org.apache.beam.sdk.metrics.MetricsFilter; +import org.apache.beam.sdk.schemas.Schema; +import org.apache.beam.sdk.testing.TestPipeline; +import org.apache.beam.sdk.testing.TestStream; +import org.apache.beam.sdk.transforms.Create; +import org.apache.beam.sdk.transforms.windowing.FixedWindows; +import org.apache.beam.sdk.transforms.windowing.GlobalWindow; +import org.apache.beam.sdk.transforms.windowing.Window; +import org.apache.beam.sdk.util.CoderUtils; +import org.apache.beam.sdk.values.KV; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.Row; +import org.apache.beam.sdk.values.TimestampedValue; +import org.apache.beam.sdk.values.ValueKind; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileContent; +import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.SortOrder; +import org.apache.iceberg.StructLike; +import org.apache.iceberg.Table; +import org.apache.iceberg.catalog.Catalog; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.data.IcebergGenerics; +import org.apache.iceberg.expressions.Expressions; +import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.DateTimeUtil; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.joda.time.Duration; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Read-back end-to-end tests for the assembled CDC sink: each case drives the full public transform + * against a real HadoopCatalog table and asserts the table's final committed contents via {@link + * IcebergGenerics#read}: the sink's ground truth, where a composition bug the per-stage suites + * cannot see shows up as wrong rows. Every case asserts the COMPLETE expected row set, all columns, + * never a count. The partition-type cases pin ground the path-rendered predecessor banned or + * crashed on; the typed-JSON transport carries partition tuples with no rendered path to parse. + * Every test creates a uniquely named table (TableCache is process-wide). + */ +@RunWith(JUnit4.class) +public class WriteCdcRowsReadBackTest { + + @Rule public transient TestPipeline p = TestPipeline.create(); + @Rule public transient TemporaryFolder tmp = new TemporaryFolder(); + + /** Canonical test table schema, shared with the other {@code cdc/sink} suites. */ + private static final org.apache.iceberg.Schema ICEBERG_SCHEMA = + new org.apache.iceberg.Schema( + Types.NestedField.required(1, "id", Types.IntegerType.get()), + Types.NestedField.optional(2, "name", Types.StringType.get()), + Types.NestedField.optional(3, "data", Types.StringType.get())); + + private static final Schema DATA_SCHEMA = + Schema.builder() + .addInt32Field("id") + .addNullableField("name", Schema.FieldType.STRING) + .addNullableField("data", Schema.FieldType.STRING) + .build(); + + /** Input schema = the canonical data schema + a sequence-number column named {@code seq}. */ + private static final Schema INPUT_SCHEMA = + Schema.builder().addFields(DATA_SCHEMA.getFields()).addInt64Field("seq").build(); + + /** Event-time origin for the streaming (TestStream) cases. */ + private static final org.joda.time.Instant BASE = new org.joda.time.Instant(0); + + private static final Duration WINDOW = Duration.standardSeconds(60); + + private Catalog catalog; + + @Before + public void setUp() { + catalog = CdcSinkTestUtils.hadoopCatalog(tmp.getRoot()); + } + + private IcebergCatalogConfig catalogConfig() { + return CdcSinkTestUtils.catalogConfig(tmp.getRoot()); + } + + // ----------------------------------------------------------------------------------------------- + // Fixtures + // ----------------------------------------------------------------------------------------------- + + /** Creates a uniquely named table and returns its identifier. */ + private TableIdentifier createTable( + String prefix, + org.apache.iceberg.Schema schema, + Set identifierFieldIds, + int formatVersion, + PartitionSpec spec) { + TableIdentifier id = TableIdentifier.of("db", prefix + "_" + System.nanoTime()); + CdcSinkTestUtils.createTable(catalog, id, schema, identifierFieldIds, formatVersion, spec); + return id; + } + + /** A uniquely named unpartitioned table over {@link #ICEBERG_SCHEMA} (PK = {@code id}). */ + private TableIdentifier createCanonicalTable(String prefix, int formatVersion) { + return createTable( + prefix, ICEBERG_SCHEMA, ImmutableSet.of(1), formatVersion, PartitionSpec.unpartitioned()); + } + + /** The sink's input schema for {@code table}: its Beam schema plus the {@code seq} column. */ + private static Schema inputSchemaFor(Table table) { + return Schema.builder() + .addFields(IcebergUtils.icebergSchemaToBeamSchema(table.schema()).getFields()) + .addInt64Field("seq") + .build(); + } + + /** Builds an input row over the canonical {@link #INPUT_SCHEMA}. */ + private static Row row(int id, String name, String data, long seq) { + return Row.withSchema(INPUT_SCHEMA).addValues(id, name, data, seq).build(); + } + + /** Builds an input row over an arbitrary schema (the per-test partitioned/timestamp tables). */ + private static Row row(Schema schema, Object... values) { + return Row.withSchema(schema).addValues(values).build(); + } + + /** A bounded CDC input of the given kind-tagged rows. */ + @SafeVarargs + private final PCollection boundedInput(Schema schema, KV... rows) { + return boundedInput(schema, ImmutableList.copyOf(rows)); + } + + /** A bounded CDC input of the given kind-tagged rows. */ + private PCollection boundedInput(Schema schema, List> rows) { + return CdcSinkTestUtils.withKinds(p.apply(Create.of(rows).withCoder(taggedRowCoder(schema)))) + .setRowSchema(schema); + } + + /** An unbounded CDC input driven by {@code stream}. */ + private PCollection streamingInput(Schema schema, TestStream> stream) { + return CdcSinkTestUtils.withKinds(p.apply(stream)).setRowSchema(schema); + } + + private static KvCoder taggedRowCoder(Schema schema) { + return KvCoder.of(SerializableCoder.of(ValueKind.class), RowCoder.of(schema)); + } + + private static TestStream.Builder> testStream(Schema schema) { + return TestStream.create(taggedRowCoder(schema)); + } + + private static TimestampedValue> at(ValueKind kind, Row row, int atSeconds) { + return TimestampedValue.of(KV.of(kind, row), BASE.plus(Duration.standardSeconds(atSeconds))); + } + + // ----------------------------------------------------------------------------------------------- + // Ground truth: read the committed table back + // ----------------------------------------------------------------------------------------------- + + /** + * Every live row, the named columns joined by {@code ':'}, sorted; {@code timestamptz} rendered + * as micros-since-epoch so a truncated or shifted timestamp cannot hide behind formatting. + */ + private static List readColumns(Table table, String... columns) { + table.refresh(); + return ImmutableList.copyOf(IcebergGenerics.read(table).build()).stream() + .map( + record -> + Arrays.stream(columns) + .map(column -> render(record.getField(column))) + .collect(Collectors.joining(":"))) + .sorted() + .collect(ImmutableList.toImmutableList()); + } + + /** {@link #readColumns} over the canonical {@code (id, name, data)} schema. */ + private static List readRows(Table table) { + return readColumns(table, "id", "name", "data"); + } + + private static String render(@Nullable Object value) { + if (value instanceof OffsetDateTime) { + return String.valueOf(DateTimeUtil.microsFromTimestamptz((OffsetDateTime) value)); + } + return String.valueOf(value); + } + + /** {@code instant} as micros since epoch, matching how {@link #readColumns} renders it. */ + private static String micros(Instant instant) { + return String.valueOf(ChronoUnit.MICROS.between(Instant.EPOCH, instant)); + } + + /** + * The distinct partition tuples of the live data files: proves WHERE rows physically landed, + * which the row contents alone cannot show. Binary values render as hex so a byte-lossy + * round-trip is visible. + */ + private static List scannedPartitions(Table table, int numFields) throws IOException { + table.refresh(); + Set partitions = new TreeSet<>(); + try (CloseableIterable tasks = table.newScan().planFiles()) { + for (FileScanTask task : tasks) { + StructLike partition = task.file().partition(); + List values = new ArrayList<>(); + for (int i = 0; i < numFields; i++) { + Object value = partition.get(i, Object.class); + values.add( + value instanceof ByteBuffer || value instanceof byte[] + ? hex(value) + : String.valueOf(value)); + } + partitions.add(String.join(":", values)); + } + } + return ImmutableList.copyOf(partitions); + } + + private static int snapshotCount(Table table) { + table.refresh(); + return Lists.newArrayList(table.snapshots()).size(); + } + + private static List addedDeleteFiles(Table table) { + table.refresh(); + return Lists.newArrayList( + checkStateNotNull(table.currentSnapshot()).addedDeleteFiles(table.io())); + } + + // ----------------------------------------------------------------------------------------------- + // 1. Same-commit insert + delete (V2) + // ----------------------------------------------------------------------------------------------- + + /** + * A key inserted then deleted inside one commit window leaves no trace, and the same-commit dedup + * touches nothing else. + */ + @Test + public void v2SameCommitInsertThenDeleteLeavesNoRowForTheKey() { + TableIdentifier id = createCanonicalTable("v2dedup", 2); + Table t = catalog.loadTable(id); + + boundedInput( + INPUT_SCHEMA, + KV.of(INSERT, row(1, "a", "x", 1L)), + KV.of(DELETE, row(1, "a", "x", 2L)), + KV.of(INSERT, row(2, "b", "y", 1L)), + KV.of(INSERT, row(3, "c", "z", 1L))) + .apply(IcebergIO.writeCdcRows(catalogConfig()).to(id).withSequenceNumberColumn("seq")); + p.run().waitUntilFinish(); + + assertThat(readRows(t), containsInAnyOrder("2:b:y", "3:c:z")); + } + + // ----------------------------------------------------------------------------------------------- + // 2. Same-commit dedup on a format-version-3 table + // ----------------------------------------------------------------------------------------------- + + /** + * Same-commit churn on V3 collapses inside the writer: the commit carries only each key's final + * row and no delete files at all. Structurally case 1 on a V3 table, kept because V3 is the + * format version the sink used to special-case, and it must keep working end to end. + */ + @Test + public void v3SameCommitChangesCollapseWithoutDeleteFiles() { + TableIdentifier id = createCanonicalTable("v3dedup", 3); + Table t = catalog.loadTable(id); + + boundedInput( + INPUT_SCHEMA, + KV.of(INSERT, row(1, "a", "x", 1L)), + KV.of(UPDATE_BEFORE, row(1, "a", "x", 2L)), + KV.of(UPDATE_AFTER, row(1, "a2", "z", 2L)), + KV.of(INSERT, row(2, "b", "y", 1L)), + KV.of(DELETE, row(2, "b", "y", 2L)), + KV.of(INSERT, row(3, "c", "w", 1L))) + .apply(IcebergIO.writeCdcRows(catalogConfig()).to(id).withSequenceNumberColumn("seq")); + p.run().waitUntilFinish(); + + assertThat(readRows(t), containsInAnyOrder("1:a2:z", "3:c:w")); + + // Every change here is same-window churn, so the collapse leaves nothing to delete. + assertThat(addedDeleteFiles(t), empty()); + } + + // ----------------------------------------------------------------------------------------------- + // 3. Cross-commit delete + // ----------------------------------------------------------------------------------------------- + + /** + * A later window's delete removes a row committed by an earlier snapshot: the cross-commit + * equality delete reaches back, and only the deleted key disappears. + */ + @Test + public void crossCommitDeleteRemovesRowCommittedByEarlierWindow() { + TableIdentifier id = createCanonicalTable("xcommit", 2); + Table t = catalog.loadTable(id); + + TestStream> stream = + testStream(INPUT_SCHEMA) + .advanceWatermarkTo(BASE) + .addElements( + at(INSERT, row(1, "a", "x", 1L), 1), + at(INSERT, row(2, "b", "y", 1L), 2), + at(INSERT, row(3, "c", "z", 1L), 3)) + .advanceWatermarkTo(BASE.plus(Duration.standardSeconds(70))) // window 0 commits + .addElements(at(DELETE, row(1, "a", "x", 2L), 61)) + .advanceWatermarkTo(BASE.plus(Duration.standardSeconds(130))) // window 1 commits + .advanceWatermarkToInfinity(); + + streamingInput(INPUT_SCHEMA, stream) + .apply( + IcebergIO.writeCdcRows(catalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withTriggeringFrequency(WINDOW)); + p.run().waitUntilFinish(); + + // Two windows really did commit separately; otherwise this would be same-commit dedup again. + assertThat(snapshotCount(t), equalTo(2)); + assertThat(readRows(t), containsInAnyOrder("2:b:y", "3:c:z")); + } + + // ----------------------------------------------------------------------------------------------- + // 4. Upsert across commit windows + // ----------------------------------------------------------------------------------------------- + + /** + * In upsert mode, a later window's image of a key replaces the one an earlier window committed: + * one row survives, carrying the later values. + */ + @Test + public void upsertCrossWindowReplacesEarlierCommittedImage() { + TableIdentifier id = createCanonicalTable("upsertxwin", 2); + Table t = catalog.loadTable(id); + + TestStream> stream = + testStream(INPUT_SCHEMA) + .advanceWatermarkTo(BASE) + .addElements( + at(INSERT, row(1, "a", "v1", 1L), 1), at(INSERT, row(2, "b", "keep", 1L), 2)) + .advanceWatermarkTo(BASE.plus(Duration.standardSeconds(70))) // window 0 commits + .addElements(at(UPDATE_AFTER, row(1, "a2", "v2", 2L), 61)) + .advanceWatermarkTo(BASE.plus(Duration.standardSeconds(130))) // window 1 commits + .advanceWatermarkToInfinity(); + + streamingInput(INPUT_SCHEMA, stream) + .apply( + IcebergIO.writeCdcRows(catalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withUpsert(true) + .withTriggeringFrequency(WINDOW)); + p.run().waitUntilFinish(); + + assertThat(snapshotCount(t), equalTo(2)); + assertThat(readRows(t), containsInAnyOrder("1:a2:v2", "2:b:keep")); + } + + // ----------------------------------------------------------------------------------------------- + // 5. Adversarial element order within one window + // ----------------------------------------------------------------------------------------------- + + /** + * Element arrival order inside a commit window is irrelevant: a DELETE fed before its own INSERT + * and out-of-order after-images still produce the sequence-order final state. + */ + @Test + public void streamingShuffledWithinWindowAppliesSequenceOrder() { + TableIdentifier id = createCanonicalTable("shuffled", 2); + Table t = catalog.loadTable(id); + + // All eight changes land in window [0s, 60s), arrival order deliberately adversarial. Event + // times are equal, so the per-key event-time/sequence source contract still holds. + TestStream> stream = + testStream(INPUT_SCHEMA) + .advanceWatermarkTo(BASE) + .addElements( + at(DELETE, row(1, "a2", "x2", 3L), 5), + at(UPDATE_AFTER, row(2, "v3", "c", 3L), 5), + at(UPDATE_AFTER, row(1, "a2", "x2", 2L), 5), + at(INSERT, row(3, "c", "w", 1L), 5), + at(INSERT, row(2, "v1", "a", 1L), 5), + at(UPDATE_BEFORE, row(1, "a", "x", 2L), 5), + at(UPDATE_AFTER, row(2, "v2", "b", 2L), 5), + at(INSERT, row(1, "a", "x", 1L), 5)) + .advanceWatermarkTo( + BASE.plus(Duration.standardSeconds(70))) // the single window commits + .advanceWatermarkToInfinity(); + + streamingInput(INPUT_SCHEMA, stream) + .apply( + IcebergIO.writeCdcRows(catalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withTriggeringFrequency(WINDOW)); + p.run().waitUntilFinish(); + + // One window => one snapshot, so this really is intra-window ordering, not commit ordering. + assertThat(snapshotCount(t), equalTo(1)); + // In sequence order id=1 ends deleted, id=2's seq-3 after-image wins, id=3 is untouched. + assertThat(readRows(t), containsInAnyOrder("2:v3:c", "3:c:w")); + } + + // ----------------------------------------------------------------------------------------------- + // 7. day(timestamptz) partitioning, through the full sink + // ----------------------------------------------------------------------------------------------- + + /** + * A {@code day(timestamptz)}-partitioned table takes inserts, updates and deletes through the + * whole sink, and every row lands in the day partition its timestamp implies, with no rendered + * partition path anywhere (the predecessor crashed here until DATE got a renderer special-case). + */ + @Test + public void dayPartitionedTableAppliesChangesAndLandsInExpectedPartitions() throws IOException { + org.apache.iceberg.Schema icebergSchema = + new org.apache.iceberg.Schema( + Types.NestedField.required(1, "id", Types.IntegerType.get()), + Types.NestedField.required(2, "ts", Types.TimestampType.withZone()), + Types.NestedField.optional(3, "name", Types.StringType.get())); + // ts is also an equality column: the key-derived-partition shape (case 16 covers non-key). + TableIdentifier id = + createTable( + "daypart", + icebergSchema, + ImmutableSet.of(1, 2), + 2, + PartitionSpec.builderFor(icebergSchema).day("ts").build()); + Table t = catalog.loadTable(id); + Schema inputSchema = inputSchemaFor(t); + + Instant march = Instant.parse("2024-03-15T10:30:00.123456Z"); + Instant june = Instant.parse("2024-06-01T00:00:00Z"); + Instant marchLate = Instant.parse("2024-03-15T23:59:59.999999Z"); + + boundedInput( + inputSchema, + KV.of(INSERT, row(inputSchema, 1, march, "a", 1L)), + KV.of(INSERT, row(inputSchema, 2, june, "b", 1L)), + KV.of(INSERT, row(inputSchema, 3, marchLate, "c", 1L)), + // Update inside the March partition (ts unchanged, so the key is unchanged). + KV.of(UPDATE_BEFORE, row(inputSchema, 1, march, "a", 2L)), + KV.of(UPDATE_AFTER, row(inputSchema, 1, march, "a2", 2L)), + // Delete the other March row. + KV.of(DELETE, row(inputSchema, 3, marchLate, "c", 2L))) + .apply(IcebergIO.writeCdcRows(catalogConfig()).to(id).withSequenceNumberColumn("seq")); + p.run().waitUntilFinish(); + + assertThat( + readColumns(t, "id", "ts", "name"), + containsInAnyOrder("1:" + micros(march) + ":a2", "2:" + micros(june) + ":b")); + + // Rows landed in the day partitions their timestamps imply, not in one lump. + assertThat( + scannedPartitions(t, 1), + containsInAnyOrder( + String.valueOf((int) LocalDate.of(2024, 3, 15).toEpochDay()), + String.valueOf((int) LocalDate.of(2024, 6, 1).toEpochDay()))); + } + + // ----------------------------------------------------------------------------------------------- + // 8. Multi-transform partition spec, through the full sink + // ----------------------------------------------------------------------------------------------- + + /** + * A spec combining {@code hour(ts)} and {@code identity(DATE)} applies end to end: contents + * right, each row in its {@code (hour, date)} partition pair. + */ + @Test + public void multiTransformPartitionSpecAppliesChangesEndToEnd() throws IOException { + org.apache.iceberg.Schema icebergSchema = + new org.apache.iceberg.Schema( + Types.NestedField.required(1, "id", Types.IntegerType.get()), + Types.NestedField.required(2, "ts", Types.TimestampType.withZone()), + Types.NestedField.required(3, "d", Types.DateType.get()), + Types.NestedField.optional(4, "name", Types.StringType.get())); + TableIdentifier id = + createTable( + "multipart", + icebergSchema, + ImmutableSet.of(1, 2, 3), + 2, + PartitionSpec.builderFor(icebergSchema).hour("ts").identity("d").build()); + Table t = catalog.loadTable(id); + Schema inputSchema = inputSchemaFor(t); + + Instant hour10 = Instant.parse("2024-03-15T10:30:00Z"); + Instant hour10b = Instant.parse("2024-03-15T10:45:00Z"); + Instant hour11 = Instant.parse("2024-03-15T11:05:00Z"); + LocalDate day15 = LocalDate.of(2024, 3, 15); + LocalDate day16 = LocalDate.of(2024, 3, 16); + + boundedInput( + inputSchema, + KV.of(INSERT, row(inputSchema, 1, hour10, day15, "a", 1L)), + KV.of(INSERT, row(inputSchema, 2, hour11, day16, "b", 1L)), + KV.of(INSERT, row(inputSchema, 3, hour10b, day15, "c", 1L)), + KV.of(UPDATE_BEFORE, row(inputSchema, 1, hour10, day15, "a", 2L)), + KV.of(UPDATE_AFTER, row(inputSchema, 1, hour10, day15, "a2", 2L)), + KV.of(DELETE, row(inputSchema, 3, hour10b, day15, "c", 2L))) + .apply(IcebergIO.writeCdcRows(catalogConfig()).to(id).withSequenceNumberColumn("seq")); + p.run().waitUntilFinish(); + + assertThat( + readColumns(t, "id", "ts", "d", "name"), + containsInAnyOrder( + "1:" + micros(hour10) + ":" + day15 + ":a2", + "2:" + micros(hour11) + ":" + day16 + ":b")); + + assertThat( + scannedPartitions(t, 2), + containsInAnyOrder( + hoursFromEpoch(hour10) + ":" + (int) day15.toEpochDay(), + hoursFromEpoch(hour11) + ":" + (int) day16.toEpochDay())); + } + + private static long hoursFromEpoch(Instant instant) { + return ChronoUnit.HOURS.between(Instant.EPOCH, instant); + } + + // ----------------------------------------------------------------------------------------------- + // 9. timestamptz round-trip at microsecond precision + // ----------------------------------------------------------------------------------------------- + + /** + * {@code timestamptz} survives the whole sink at microsecond precision, including pre-epoch + * instants (negative seconds, non-negative sub-second part). + */ + @Test + public void timestamptzRoundTripsAtMicrosecondPrecision() { + org.apache.iceberg.Schema icebergSchema = + new org.apache.iceberg.Schema( + Types.NestedField.required(1, "id", Types.IntegerType.get()), + Types.NestedField.required(2, "ts", Types.TimestampType.withZone()), + Types.NestedField.optional(3, "name", Types.StringType.get())); + TableIdentifier id = + createTable("tstz", icebergSchema, ImmutableSet.of(1), 2, PartitionSpec.unpartitioned()); + Table t = catalog.loadTable(id); + Schema inputSchema = inputSchemaFor(t); + + Instant afterEpoch = Instant.parse("2024-03-15T10:30:00.123456Z"); + Instant beforeEpoch = Instant.parse("1969-07-20T20:17:40.000001Z"); + Instant longBeforeEpoch = Instant.parse("1901-12-13T20:45:52.654321Z"); + Instant updated = Instant.parse("1955-11-05T06:15:00.000123Z"); + Instant deleted = Instant.parse("1962-10-16T12:00:00.999999Z"); + + boundedInput( + inputSchema, + KV.of(INSERT, row(inputSchema, 1, afterEpoch, "after", 1L)), + KV.of(INSERT, row(inputSchema, 2, beforeEpoch, "before", 1L)), + KV.of(INSERT, row(inputSchema, 3, longBeforeEpoch, "long-before", 1L)), + KV.of(INSERT, row(inputSchema, 4, deleted, "deleted", 1L)), + // The timestamp itself is updated (ts is not part of the key here). + KV.of(UPDATE_BEFORE, row(inputSchema, 2, beforeEpoch, "before", 2L)), + KV.of(UPDATE_AFTER, row(inputSchema, 2, updated, "before-updated", 2L)), + // The DELETE is what fails a sink that ignores change kinds: same-commit dedup would + // reproduce id=2's expected row regardless, but a DELETE has no such cover. + KV.of(DELETE, row(inputSchema, 4, deleted, "deleted", 2L))) + .apply(IcebergIO.writeCdcRows(catalogConfig()).to(id).withSequenceNumberColumn("seq")); + p.run().waitUntilFinish(); + + assertThat( + readColumns(t, "id", "ts", "name"), + containsInAnyOrder( + "1:" + micros(afterEpoch) + ":after", + "2:" + micros(updated) + ":before-updated", + "3:" + micros(longBeforeEpoch) + ":long-before")); + } + + // ----------------------------------------------------------------------------------------------- + // 10. '/' inside a string partition value + // ----------------------------------------------------------------------------------------------- + + /** + * A {@code '/'} inside an identity-partitioned string value round-trips: the tuple travels as + * typed JSON, so a path-separator lookalike is just a value. The predecessor banned this outright + * ({@code SerializableDataFileTest} still pins the NaN path-fallback's encoded form). + */ + @Test + public void slashInStringPartitionValueAppliesChangesEndToEnd() throws IOException { + org.apache.iceberg.Schema icebergSchema = + new org.apache.iceberg.Schema( + Types.NestedField.required(1, "id", Types.IntegerType.get()), + Types.NestedField.required(2, "name", Types.StringType.get()), + Types.NestedField.optional(3, "data", Types.StringType.get())); + TableIdentifier id = + createTable( + "slashpart", + icebergSchema, + ImmutableSet.of(1, 2), + 2, + PartitionSpec.builderFor(icebergSchema).identity("name").build()); + Table t = catalog.loadTable(id); + Schema inputSchema = inputSchemaFor(t); + + boundedInput( + inputSchema, + KV.of(INSERT, row(inputSchema, 1, "a/b", "x", 1L)), + KV.of(INSERT, row(inputSchema, 2, "c/d/e", "y", 1L)), + KV.of(INSERT, row(inputSchema, 3, "plain", "z", 1L)), + KV.of(INSERT, row(inputSchema, 4, "a/b", "w", 1L)), + // Update and delete inside the slash-bearing partition. + KV.of(UPDATE_BEFORE, row(inputSchema, 1, "a/b", "x", 2L)), + KV.of(UPDATE_AFTER, row(inputSchema, 1, "a/b", "x2", 2L)), + KV.of(DELETE, row(inputSchema, 4, "a/b", "w", 2L))) + .apply(IcebergIO.writeCdcRows(catalogConfig()).to(id).withSequenceNumberColumn("seq")); + p.run().waitUntilFinish(); + + assertThat(readRows(t), containsInAnyOrder("1:a/b:x2", "2:c/d/e:y", "3:plain:z")); + assertThat(scannedPartitions(t, 1), containsInAnyOrder("a/b", "c/d/e", "plain")); + } + + // ----------------------------------------------------------------------------------------------- + // 11. shards_per_partition + // ----------------------------------------------------------------------------------------------- + + /** {@code (id INT, ts TIMESTAMPTZ)} both required and both identifier fields, plus a name. */ + private static final org.apache.iceberg.Schema DAY_PARTITIONED_SCHEMA = + new org.apache.iceberg.Schema( + Types.NestedField.required(1, "id", Types.IntegerType.get()), + Types.NestedField.required(2, "ts", Types.TimestampType.withZone()), + Types.NestedField.optional(3, "name", Types.StringType.get())); + + /** A uniquely named {@code day(ts)}-partitioned table over {@link #DAY_PARTITIONED_SCHEMA}. */ + private TableIdentifier createDayPartitionedTable(String prefix) { + return createTable( + prefix, + DAY_PARTITIONED_SCHEMA, + ImmutableSet.of(1, 2), + 2, + PartitionSpec.builderFor(DAY_PARTITIONED_SCHEMA).day("ts").build()); + } + + /** The number of live data files in {@code table}: the quantity the option exists to reduce. */ + private static int dataFileCount(Table table) throws IOException { + table.refresh(); + try (CloseableIterable tasks = table.newScan().planFiles()) { + return Lists.newArrayList(tasks).size(); + } + } + + /** + * The payoff, measured: 48 rows over 8 day partitions written twice at {@code num_shards = 8}. + * {@code shards_per_partition = 1} yields exactly one data file per day, the uncapped default + * several per day. Same rows, same partitions; only the file layout differs. + */ + @Test + public void shardsPerPartitionOneCollapsesFilesToThePartitionCount() throws IOException { + int numDays = 8; + int rowsPerDay = 6; + int numShards = 8; + + TableIdentifier affineId = createDayPartitionedTable("pas_files_affine"); + TableIdentifier pkId = createDayPartitionedTable("pas_files_pk"); + Table affine = catalog.loadTable(affineId); + Table pk = catalog.loadTable(pkId); + Schema inputSchema = inputSchemaFor(affine); + + List> rows = new ArrayList<>(); + int id = 0; + for (int day = 0; day < numDays; day++) { + Instant ts = Instant.parse("2024-03-01T12:00:00Z").plus(day, ChronoUnit.DAYS); + for (int i = 0; i < rowsPerDay; i++) { + id++; + rows.add(KV.of(INSERT, row(inputSchema, id, ts, "n" + id, 1L))); + } + } + + // One input, two sinks: the only difference between them is the sharding mode. + PCollection in = boundedInput(inputSchema, rows); + in.apply( + "WriteAffine", + IcebergIO.writeCdcRows(catalogConfig()) + .to(affineId) + .withSequenceNumberColumn("seq") + .withNumShards(numShards) + .withShardsPerPartition(1)); + in.apply( + "WritePk", + IcebergIO.writeCdcRows(catalogConfig()) + .to(pkId) + .withSequenceNumberColumn("seq") + .withNumShards(numShards)); + p.run().waitUntilFinish(); + + // Same rows, same partitions: the tables are indistinguishable to a reader. + assertThat(readColumns(affine, "id", "ts", "name"), hasSize(numDays * rowsPerDay)); + assertThat( + readColumns(affine, "id", "ts", "name"), equalTo(readColumns(pk, "id", "ts", "name"))); + assertThat(scannedPartitions(affine, 1), hasSize(numDays)); + assertThat(scannedPartitions(affine, 1), equalTo(scannedPartitions(pk, 1))); + + // Exactly one data file per partition, whatever num_shards is. + assertThat(dataFileCount(affine), equalTo(numDays)); + // Uncapped, several shards write each partition (36 files here, deterministically); the bound + // is deliberately loose so this asserts the scaling claim, not a hash fixture. + assertThat(dataFileCount(pk), greaterThanOrEqualTo(2 * numDays)); + } + + /** The {@code (id, ts)} primary-key schema exactly as {@code TableSetup} derives it. */ + private static Schema dayPartitionedPkSchema(Table table) { + Schema dataSchema = IcebergUtils.icebergSchemaToBeamSchema(table.schema()); + return Schema.builder() + .addField(dataSchema.getField("id")) + .addField(dataSchema.getField("ts")) + .build(); + } + + /** {@code pkBytes} for one {@code (id, ts)} key, byte-identical to the sink's own encoding. */ + private static byte[] encodePk(Schema pkSchema, int id, Instant ts) throws CoderException { + return CoderUtils.encodeToByteArray( + RowCoder.of(pkSchema), Row.withSchema(pkSchema).addValues(id, ts).build()); + } + + /** + * Picks 16 ids whose pk hashes cover all 12 {@code mod 12} residues plus all 4 {@code mod 4} + * residues once more: exact shard coverage at both {@code shards_per_partition = 4} and the + * uncapped default, no probabilistic spread. + */ + private static List idsCoveringAllShardResidues(Schema pkSchema, Instant ts, int firstId) + throws CoderException { + Map idByShardResidue = new TreeMap<>(); + Map extraIdByOffsetResidue = new TreeMap<>(); + for (int candidate = firstId; + idByShardResidue.size() < 12 || extraIdByOffsetResidue.size() < 4; + candidate++) { + if (candidate - firstId >= 10_000) { + throw new AssertionError("residue search did not converge; the PK encoding drifted"); + } + int hash = TableSetup.pkHash(encodePk(pkSchema, candidate, ts)); + if (idByShardResidue.putIfAbsent(Math.floorMod(hash, 12), candidate) != null) { + extraIdByOffsetResidue.putIfAbsent(Math.floorMod(hash, 4), candidate); + } + } + List ids = new ArrayList<>(idByShardResidue.values()); + ids.addAll(extraIdByOffsetResidue.values()); + return ids; + } + + /** + * The dial between the endpoints, measured: three day partitions at {@code num_shards = 12}, + * written at {@code shards_per_partition = 4} and uncapped. {@link #idsCoveringAllShardResidues} + * makes both counts exact (3 x 4 = 12 files dialed, 3 x 12 = 36 uncapped) and both tables must + * hold identical contents: the dial moves files, never rows. + */ + @Test + public void shardsPerPartitionDialsFileCountBetweenTheEndpoints() throws IOException { + int numShards = 12; + int shardsPerPartition = 4; + int numDays = 3; + + TableIdentifier dialId = createDayPartitionedTable("spp_dial"); + TableIdentifier uncappedId = createDayPartitionedTable("spp_dial_uncapped"); + Table dial = catalog.loadTable(dialId); + Table uncapped = catalog.loadTable(uncappedId); + Schema inputSchema = inputSchemaFor(dial); + Schema pkSchema = dayPartitionedPkSchema(dial); + + List> rows = new ArrayList<>(); + List expected = new ArrayList<>(); + for (int day = 0; day < numDays; day++) { + Instant ts = Instant.parse("2024-06-01T12:00:00Z").plus(day, ChronoUnit.DAYS); + for (int id : idsCoveringAllShardResidues(pkSchema, ts, 1000 * (day + 1))) { + rows.add(KV.of(INSERT, row(inputSchema, id, ts, "n" + id, 1L))); + expected.add(id + ":" + micros(ts) + ":n" + id); + } + } + + PCollection in = boundedInput(inputSchema, rows); + in.apply( + "WriteDial", + IcebergIO.writeCdcRows(catalogConfig()) + .to(dialId) + .withSequenceNumberColumn("seq") + .withNumShards(numShards) + .withShardsPerPartition(shardsPerPartition)); + in.apply( + "WriteUncapped", + IcebergIO.writeCdcRows(catalogConfig()) + .to(uncappedId) + .withSequenceNumberColumn("seq") + .withNumShards(numShards)); + p.run().waitUntilFinish(); + + // Identical, complete contents in identical partitions under both settings. + List expectedSorted = + expected.stream().sorted().collect(ImmutableList.toImmutableList()); + assertThat(readColumns(dial, "id", "ts", "name"), equalTo(expectedSorted)); + assertThat(readColumns(uncapped, "id", "ts", "name"), equalTo(expectedSorted)); + assertThat(scannedPartitions(dial, 1), hasSize(numDays)); + assertThat(scannedPartitions(dial, 1), equalTo(scannedPartitions(uncapped, 1))); + + // The measured dial: exactly spp shards per partition, against numShards shards uncapped. + assertThat(dataFileCount(dial), equalTo(numDays * shardsPerPartition)); + assertThat(dataFileCount(uncapped), equalTo(numDays * numShards)); + } + + /** + * End-to-end correctness at the tightest cap ({@code shards_per_partition = 1}) over cross-window + * updates and deletes: the final contents must be identical to the uncapped write of the same + * stream. + */ + @Test + public void shardsPerPartitionOneMatchesPrimaryKeyShardingEndToEnd() throws IOException { + TableIdentifier affineId = createDayPartitionedTable("pas_e2e_affine"); + TableIdentifier pkId = createDayPartitionedTable("pas_e2e_pk"); + Table affine = catalog.loadTable(affineId); + Table pk = catalog.loadTable(pkId); + Schema inputSchema = inputSchemaFor(affine); + + Instant march = Instant.parse("2024-03-15T10:30:00Z"); + Instant april = Instant.parse("2024-04-02T08:00:00Z"); + Instant may = Instant.parse("2024-05-20T23:00:00Z"); + + PCollection in = + streamingInput( + inputSchema, + testStream(inputSchema) + .advanceWatermarkTo(BASE) + // Window 1: four inserts across three day partitions. + .addElements( + at(INSERT, row(inputSchema, 1, march, "a", 1L), 1), + at(INSERT, row(inputSchema, 2, march, "b", 1L), 1), + at(INSERT, row(inputSchema, 3, april, "c", 1L), 2), + at(INSERT, row(inputSchema, 4, may, "d", 2L), 2)) + .advanceWatermarkTo(BASE.plus(WINDOW)) + // Window 2: update one March row, delete the other; delete the May row. + .addElements( + at(UPDATE_BEFORE, row(inputSchema, 1, march, "a", 3L), 61), + at(UPDATE_AFTER, row(inputSchema, 1, march, "a2", 3L), 61), + at(DELETE, row(inputSchema, 2, march, "b", 3L), 62), + at(DELETE, row(inputSchema, 4, may, "d", 4L), 62)) + .advanceWatermarkTo(BASE.plus(WINDOW).plus(WINDOW)) + // Window 3: update the April row and re-insert into the emptied May partition. + .addElements( + at(UPDATE_BEFORE, row(inputSchema, 3, april, "c", 5L), 121), + at(UPDATE_AFTER, row(inputSchema, 3, april, "c2", 5L), 121), + at(INSERT, row(inputSchema, 5, may, "e", 5L), 122)) + .advanceWatermarkToInfinity()); + + in.apply( + "WriteAffine", + IcebergIO.writeCdcRows(catalogConfig()) + .to(affineId) + .withSequenceNumberColumn("seq") + .withTriggeringFrequency(WINDOW) + .withShardsPerPartition(1)); + in.apply( + "WritePk", + IcebergIO.writeCdcRows(catalogConfig()) + .to(pkId) + .withSequenceNumberColumn("seq") + .withTriggeringFrequency(WINDOW)); + p.run().waitUntilFinish(); + + List expected = + ImmutableList.of( + "1:" + micros(march) + ":a2", "3:" + micros(april) + ":c2", "5:" + micros(may) + ":e"); + assertThat(readColumns(affine, "id", "ts", "name"), equalTo(expected)); + assertThat(readColumns(pk, "id", "ts", "name"), equalTo(expected)); + + // Rows physically landed in their day partitions under both modes. + assertThat(scannedPartitions(affine, 1), equalTo(scannedPartitions(pk, 1))); + assertThat( + scannedPartitions(affine, 1), + containsInAnyOrder( + String.valueOf((int) LocalDate.of(2024, 3, 15).toEpochDay()), + String.valueOf((int) LocalDate.of(2024, 4, 2).toEpochDay()), + String.valueOf((int) LocalDate.of(2024, 5, 20).toEpochDay()))); + } + + // ----------------------------------------------------------------------------------------------- + // 12. DECIMAL and BINARY partition values, through the full sink + // ----------------------------------------------------------------------------------------------- + + /** + * {@code identity(DECIMAL)} + {@code identity(BINARY)} partitions apply changes end to end: the + * types the path-rendered predecessor banned outright (base64 out, raw UTF-8 back in). The binary + * values are deliberately invalid UTF-8 with {@code 0x00} and {@code '/'}, so a byte-lossy + * round-trip anywhere changes the partition tuple and shows in {@link #scannedPartitions}. + */ + @Test + public void decimalAndBinaryPartitionValuesApplyChangesEndToEnd() throws IOException { + org.apache.iceberg.Schema icebergSchema = + new org.apache.iceberg.Schema( + Types.NestedField.required(1, "id", Types.IntegerType.get()), + Types.NestedField.required(2, "amount", Types.DecimalType.of(9, 3)), + Types.NestedField.required(3, "bin", Types.BinaryType.get()), + Types.NestedField.optional(4, "name", Types.StringType.get())); + TableIdentifier id = + createTable( + "decbinpart", + icebergSchema, + ImmutableSet.of(1, 2, 3), + 2, + PartitionSpec.builderFor(icebergSchema).identity("amount").identity("bin").build()); + Table t = catalog.loadTable(id); + Schema inputSchema = inputSchemaFor(t); + + BigDecimal negative = new BigDecimal("-12345.678"); + BigDecimal positive = new BigDecimal("0.001"); + byte[] gnarly = new byte[] {0x00, (byte) 0xFF, 0x2F, (byte) 0x80}; + byte[] plain = new byte[] {0x01, 0x02}; + + boundedInput( + inputSchema, + KV.of(INSERT, row(inputSchema, 1, negative, gnarly, "a", 1L)), + KV.of(INSERT, row(inputSchema, 2, positive, plain, "b", 1L)), + KV.of(INSERT, row(inputSchema, 3, negative, gnarly, "c", 1L)), + // Update and delete inside the gnarly-binary partition. + KV.of(UPDATE_BEFORE, row(inputSchema, 1, negative, gnarly, "a", 2L)), + KV.of(UPDATE_AFTER, row(inputSchema, 1, negative, gnarly, "a2", 2L)), + KV.of(DELETE, row(inputSchema, 3, negative, gnarly, "c", 2L))) + .apply(IcebergIO.writeCdcRows(catalogConfig()).to(id).withSequenceNumberColumn("seq")); + p.run().waitUntilFinish(); + + assertThat( + readColumns(t, "id", "amount", "name"), + containsInAnyOrder("1:" + negative + ":a2", "2:" + positive + ":b")); + // Both partition tuples survived byte-for-byte: the decimal keeps its scale and sign, and the + // binary keeps its 0x00 / 0xFF / '/' bytes. + assertThat( + scannedPartitions(t, 2), + containsInAnyOrder(negative + ":" + hex(gnarly), positive + ":" + hex(plain))); + } + + /** Renders a partition value that may be a {@link ByteBuffer} or {@code byte[]} as hex. */ + private static String hex(Object value) { + byte[] bytes; + if (value instanceof ByteBuffer) { + ByteBuffer view = ((ByteBuffer) value).duplicate(); + bytes = new byte[view.remaining()]; + view.get(bytes); + } else { + bytes = (byte[]) value; + } + StringBuilder sb = new StringBuilder(); + for (byte b : bytes) { + sb.append(String.format("%02x", b)); + } + return sb.toString(); + } + + // ----------------------------------------------------------------------------------------------- + // 13. Table CREATED with a sort order + // ----------------------------------------------------------------------------------------------- + + /** + * A cross-window DELETE commits against a table CREATED with a sort order. The sink's equality + * deletes carry sort-order id 0 (unsorted), and a table born sorted stores no id 0 at all; + * resolving the id straight out of {@code table.sortOrders()} threw here, and the committer's + * rethrow-before-state-write re-fired the identical window forever. Structurally case 3 on a + * sorted-on-create table; the order is on a NON-key column on purpose. + */ + @Test + public void crossWindowDeleteCommitsAgainstTableCreatedWithASortOrder() { + TableIdentifier tableId = TableIdentifier.of("db", "sortedcreate_" + System.nanoTime()); + CdcSinkTestUtils.createSortedTable( + catalog, + tableId, + ICEBERG_SCHEMA, + ImmutableSet.of(1), + 2, + PartitionSpec.unpartitioned(), + SortOrder.builderFor(ICEBERG_SCHEMA).asc("name").desc("data").build()); + Table t = catalog.loadTable(tableId); + + // The premise: this table has a sort order, and it does NOT have id 0. + assertThat(t.sortOrder().isSorted(), is(true)); + assertThat(t.sortOrders().keySet(), not(hasItem(SortOrder.unsorted().orderId()))); + + TestStream> stream = + testStream(INPUT_SCHEMA) + .advanceWatermarkTo(BASE) + .addElements( + at(INSERT, row(1, "a", "x", 1L), 1), + at(INSERT, row(2, "b", "y", 1L), 2), + at(INSERT, row(3, "c", "z", 1L), 3)) + .advanceWatermarkTo(BASE.plus(Duration.standardSeconds(70))) // window 0 commits + .addElements(at(DELETE, row(1, "a", "x", 2L), 61)) + .advanceWatermarkTo(BASE.plus(Duration.standardSeconds(130))) // window 1 commits + .advanceWatermarkToInfinity(); + + streamingInput(INPUT_SCHEMA, stream) + .apply( + IcebergIO.writeCdcRows(catalogConfig()) + .to(tableId) + .withSequenceNumberColumn("seq") + .withTriggeringFrequency(WINDOW)); + p.run().waitUntilFinish(); + + // The delete window really committed; a stalled destination leaves one snapshot, three rows. + assertThat(snapshotCount(t), equalTo(2)); + assertThat(readRows(t), containsInAnyOrder("2:b:y", "3:c:z")); + + // The equality delete landed under the unsorted order, not silently relabelled as the table's. + List deletes = addedDeleteFiles(t); + assertThat(deletes, not(empty())); + for (DeleteFile delete : deletes) { + assertThat(delete.content(), equalTo(FileContent.EQUALITY_DELETES)); + assertThat(delete.sortOrderId(), equalTo(SortOrder.unsorted().orderId())); + } + } + + // ----------------------------------------------------------------------------------------------- + // 14. Mid-run partition-spec evolution: the pinned worker keeps writing, correctly. + // ----------------------------------------------------------------------------------------------- + + /** The {@code id}-only primary-key schema of the {@link #DATA_SCHEMA} fixture tables. */ + private static final Schema ID_PK_SCHEMA = + Schema.builder().addField(DATA_SCHEMA.getField("id")).build(); + + /** + * A sorted-group element for driving {@link WriteDeltas.WriteDeltasFn} directly. The collapse + * writer blocks on the sort key's pk prefix, so it carries the row's encoded {@code id}, the + * fixture tables' primary key. + */ + private static KV cdc(Row data, long seq, ValueKind kind) + throws CoderException { + Row pk = Row.withSchema(ID_PK_SCHEMA).addValues(data.getInt32("id")).build(); + byte[] pkBytes = CoderUtils.encodeToByteArray(RowCoder.of(ID_PK_SCHEMA), pk); + return KV.of(CdcSortKey.encode(pkBytes, seq, kind), CdcRecord.of(data, kind, seq)); + } + + /** Sums the named {@link CommitDeltas} counter committed by the pipeline. */ + private static long committerCounter(PipelineResult result, String name) { + long total = 0; + for (MetricResult c : + result + .metrics() + .queryMetrics( + MetricsFilter.builder() + .addNameFilter(MetricNameFilter.named(CommitDeltas.class, name)) + .build()) + .getCounters()) { + total += c.getCommitted(); + } + return total; + } + + /** + * A worker that resolved before a mid-run spec evolution keeps writing under the pinned spec, + * equality deletes included, so they reach the earlier window's data. The write stage runs at the + * {@link WriteDeltas.WriteDeltasFn} level so both windows provably share one worker instance (the + * pin is per worker); the commit stage and read-back are real. + */ + @Test + public void evolvedSpecKeepsWritingUnderPinnedSpec() throws IOException { + TableIdentifier id = + createTable( + "pinnedspec", + ICEBERG_SCHEMA, + ImmutableSet.of(1), + 2, + PartitionSpec.builderFor(ICEBERG_SCHEMA).bucket("id", 4).build()); + Table t = catalog.loadTable(id); + int pinnedSpecId = t.spec().specId(); + + CdcWriteConfig config = CdcWriteConfig.builder().setSinkId("pin-sink").build(); + TableSetup setup = + new TableSetup( + catalogConfig(), config, DynamicDestinations.singleTable(id, DATA_SCHEMA), "px"); + WriteDeltas.WriteDeltasFn fn = new WriteDeltas.WriteDeltasFn(setup, config, "px", DATA_SCHEMA); + + // Window 1: three inserts, resolved and written under the original spec. + List w1 = new ArrayList<>(); + fn.process( + KV.of( + DestinationShard.of(id.toString(), 0), + ImmutableList.of( + cdc(row(DATA_SCHEMA, 1, "a", "x"), 1L, INSERT), + cdc(row(DATA_SCHEMA, 2, "b", "y"), 1L, INSERT), + cdc(row(DATA_SCHEMA, 3, "c", "z"), 1L, INSERT))), + GlobalWindow.INSTANCE, + CdcSinkTestUtils.collectInto(w1)); + + // The operator evolves the partition spec; the worker's shared Table instance sees it. + TableSetup.Dest dest = setup.get(id.toString(), DATA_SCHEMA); + dest.table().updateSpec().addField(Expressions.bucket("id", 8)).commit(); + dest.table().refresh(); + assertThat(dest.table().spec().specId(), not(equalTo(pinnedSpecId))); + + // Window 2 through the SAME worker instance: update id=1, delete id=2. + List w2 = new ArrayList<>(); + fn.process( + KV.of( + DestinationShard.of(id.toString(), 0), + ImmutableList.of( + cdc(row(DATA_SCHEMA, 1, "a", "x"), 2L, UPDATE_BEFORE), + cdc(row(DATA_SCHEMA, 1, "a2", "x2"), 2L, UPDATE_AFTER), + cdc(row(DATA_SCHEMA, 2, "b", "y"), 3L, DELETE))), + GlobalWindow.INSTANCE, + CdcSinkTestUtils.collectInto(w2)); + assertThat(w1, hasSize(1)); + assertThat(w2, hasSize(1)); + + // Commit both windows, in order, through the real committer. + TestStream.Builder stream = + TestStream.create(ShardDeltaFiles.coder()).advanceWatermarkTo(BASE); + stream = + stream.addElements(TimestampedValue.of(w1.get(0), BASE.plus(Duration.standardSeconds(1)))); + stream = + stream.addElements(TimestampedValue.of(w2.get(0), BASE.plus(Duration.standardSeconds(61)))); + PipelineResult result = + p.apply(stream.advanceWatermarkToInfinity()) + .apply(Window.into(FixedWindows.of(WINDOW))) + .apply(new CommitDeltas(catalogConfig(), "pin-sink")) + .getPipeline() + .run(); + result.waitUntilFinish(); + + // Two windows committed; every committed file carries the pinned original spec id. + assertThat(snapshotCount(t), equalTo(2)); + for (Snapshot snap : t.snapshots()) { + for (DataFile file : snap.addedDataFiles(t.io())) { + assertThat(file.specId(), equalTo(pinnedSpecId)); + } + for (DeleteFile file : snap.addedDeleteFiles(t.io())) { + assertThat(file.specId(), equalTo(pinnedSpecId)); + } + } + // The pinned-spec equality deletes applied to the pinned-spec data: exact final contents. + assertThat(readRows(t), containsInAnyOrder("1:a2:x2", "3:c:z")); + assertThat(committerCounter(result, "specMismatchedWindows"), equalTo(0L)); + } + + // ----------------------------------------------------------------------------------------------- + // 15. A worker joining mid-run after a spec evolution adopts the run's stamped spec. + // ----------------------------------------------------------------------------------------------- + + /** + * A worker that first resolves AFTER a mid-run spec evolution (a fresh {@link TableSetup} under + * the same runId) adopts the spec the first commit stamped, so its files still carry the original + * spec id and its deletes reach the first pass's data. Two pipelines because the stamp must be in + * the table before the joining worker resolves. + */ + @Test + public void joiningWorkerScenarioStaysConsistent() throws IOException { + TableIdentifier id = + createTable( + "joiningworker", + ICEBERG_SCHEMA, + ImmutableSet.of(1), + 2, + PartitionSpec.builderFor(ICEBERG_SCHEMA).bucket("id", 4).build()); + Table t = catalog.loadTable(id); + int stampedSpecId = t.spec().specId(); + String runId = "join-runId"; + CdcWriteConfig config = CdcWriteConfig.builder().setSinkId("join-sink").build(); + + // Pass 1: an original worker writes window 1 under the initial spec ... + TableSetup setup1 = + new TableSetup( + catalogConfig(), config, DynamicDestinations.singleTable(id, DATA_SCHEMA), runId); + WriteDeltas.WriteDeltasFn fn1 = + new WriteDeltas.WriteDeltasFn(setup1, config, runId, DATA_SCHEMA); + List w1 = new ArrayList<>(); + fn1.process( + KV.of( + DestinationShard.of(id.toString(), 0), + ImmutableList.of( + cdc(row(DATA_SCHEMA, 1, "a", "x"), 1L, INSERT), + cdc(row(DATA_SCHEMA, 2, "b", "y"), 1L, INSERT), + cdc(row(DATA_SCHEMA, 3, "c", "z"), 1L, INSERT))), + GlobalWindow.INSTANCE, + CdcSinkTestUtils.collectInto(w1)); + + // ... and the committer commits it, stamping the run's pinned spec into the summary. + PipelineResult first = + p.apply( + TestStream.create(ShardDeltaFiles.coder()) + .advanceWatermarkTo(BASE) + .addElements( + TimestampedValue.of(w1.get(0), BASE.plus(Duration.standardSeconds(1)))) + .advanceWatermarkToInfinity()) + .apply(Window.into(FixedWindows.of(WINDOW))) + .apply(new CommitDeltas(catalogConfig(), "join-sink", null, null, runId)) + .getPipeline() + .run(); + first.waitUntilFinish(); + + // The spec evolves after the stamp landed; the process-shared Table sees it. + Table shared = setup1.get(id.toString(), DATA_SCHEMA).table(); + shared.updateSpec().addField(Expressions.bucket("id", 8)).commit(); + shared.refresh(); + assertThat(shared.spec().specId(), not(equalTo(stampedSpecId))); + + // Pass 2: the joining worker resolves the STAMPED spec and writes window 2 under it: + // an update and a delete against window 1's rows. + TableSetup setup2 = + new TableSetup( + catalogConfig(), config, DynamicDestinations.singleTable(id, DATA_SCHEMA), runId); + WriteDeltas.WriteDeltasFn fn2 = + new WriteDeltas.WriteDeltasFn(setup2, config, runId, DATA_SCHEMA); + List w2 = new ArrayList<>(); + fn2.process( + KV.of( + DestinationShard.of(id.toString(), 0), + ImmutableList.of( + cdc(row(DATA_SCHEMA, 1, "a", "x"), 2L, UPDATE_BEFORE), + cdc(row(DATA_SCHEMA, 1, "a2", "x2"), 2L, UPDATE_AFTER), + cdc(row(DATA_SCHEMA, 2, "b", "y"), 3L, DELETE))), + GlobalWindow.INSTANCE, + CdcSinkTestUtils.collectInto(w2)); + assertThat(setup2.get(id.toString(), DATA_SCHEMA).spec().specId(), equalTo(stampedSpecId)); + + TestPipeline second = TestPipeline.create(); + second.enableAbandonedNodeEnforcement(false); + PipelineResult secondResult = + second + .apply( + TestStream.create(ShardDeltaFiles.coder()) + .advanceWatermarkTo(BASE) + .addElements( + TimestampedValue.of(w2.get(0), BASE.plus(Duration.standardSeconds(61)))) + .advanceWatermarkToInfinity()) + .apply(Window.into(FixedWindows.of(WINDOW))) + .apply(new CommitDeltas(catalogConfig(), "join-sink", null, null, runId)) + .getPipeline() + .run(); + secondResult.waitUntilFinish(); + + // Every committed file of both passes carries the stamped spec; no mismatch was flagged. + t.refresh(); + assertThat(snapshotCount(t), equalTo(2)); + for (Snapshot snap : t.snapshots()) { + for (DataFile file : snap.addedDataFiles(t.io())) { + assertThat(file.specId(), equalTo(stampedSpecId)); + } + for (DeleteFile file : snap.addedDeleteFiles(t.io())) { + assertThat(file.specId(), equalTo(stampedSpecId)); + } + } + assertThat(readRows(t), containsInAnyOrder("1:a2:x2", "3:c:z")); + assertThat( + committerCounter(first, "specMismatchedWindows") + + committerCounter(secondResult, "specMismatchedWindows"), + equalTo(0L)); + } + + // ----------------------------------------------------------------------------------------------- + // 16. Non-key partitioning: day(ts) with only id as the key + // ----------------------------------------------------------------------------------------------- + + /** {@link #DAY_PARTITIONED_SCHEMA} with {@code day(ts)} partitioning but ONLY {@code id} key. */ + private TableIdentifier createNonKeyDayPartitionedTable(String prefix) { + return createTable( + prefix, + DAY_PARTITIONED_SCHEMA, + ImmutableSet.of(1), + 2, + PartitionSpec.builderFor(DAY_PARTITIONED_SCHEMA).day("ts").build()); + } + + private static final Instant MARCH = Instant.parse("2024-03-15T10:30:00Z"); + private static final Instant JUNE = Instant.parse("2024-06-01T09:00:00Z"); + private static final int MARCH_DAY = (int) LocalDate.of(2024, 3, 15).toEpochDay(); + private static final int JUNE_DAY = (int) LocalDate.of(2024, 6, 1).toEpochDay(); + + /** + * On a {@code day(ts)}-partitioned table whose key is {@code id} alone, an update that moves a + * row across partitions within one window deletes it from the partition it actually occupied: a + * delete routed by the after-image would land in the new partition and leave the old row alive + * beside the new one. + */ + @Test + public void nonKeyPartitionedMoveDeletesTheRowFromItsOldPartition() throws IOException { + TableIdentifier id = createNonKeyDayPartitionedTable("nonkeymove"); + Table t = catalog.loadTable(id); + Schema inputSchema = inputSchemaFor(t); + + TestStream> stream = + testStream(inputSchema) + .advanceWatermarkTo(BASE) + .addElements( + at(INSERT, row(inputSchema, 1, MARCH, "a", 1L), 1), + at(INSERT, row(inputSchema, 2, MARCH, "b", 1L), 2)) + .advanceWatermarkTo(BASE.plus(WINDOW)) // window 1 commits both March rows + .addElements( + at(UPDATE_BEFORE, row(inputSchema, 1, MARCH, "a", 2L), 61), + at(UPDATE_AFTER, row(inputSchema, 1, JUNE, "a2", 2L), 61)) + .advanceWatermarkToInfinity(); + + streamingInput(inputSchema, stream) + .apply( + IcebergIO.writeCdcRows(catalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withTriggeringFrequency(WINDOW)); + p.run().waitUntilFinish(); + + assertThat(snapshotCount(t), equalTo(2)); + assertThat( + readColumns(t, "id", "ts", "name"), + containsInAnyOrder("1:" + micros(JUNE) + ":a2", "2:" + micros(MARCH) + ":b")); + + // Physical corroboration: March still holds id=2's live file, June holds the moved row, and + // window 2's only delete file sits in MARCH, where the old row lives. + assertThat( + scannedPartitions(t, 1), + containsInAnyOrder(String.valueOf(MARCH_DAY), String.valueOf(JUNE_DAY))); + List deletes = addedDeleteFiles(t); + assertThat(deletes, hasSize(1)); + assertThat(deletes.get(0).content(), equalTo(FileContent.EQUALITY_DELETES)); + assertThat(deletes.get(0).partition().get(0, Integer.class), equalTo(MARCH_DAY)); + } + + /** + * A move whose {@code UPDATE_BEFORE} and {@code UPDATE_AFTER} land in DIFFERENT commit windows + * still converges: the bare-UB window commits the equality delete in the old partition, the + * bare-UA window commits the row in the new one. This is why a bare-UA block must write without + * being rejected: its delete can already sit in an earlier commit. + */ + @Test + public void nonKeyPartitionedMoveSplitAcrossWindowsConverges() { + TableIdentifier id = createNonKeyDayPartitionedTable("nonkeysplit"); + Table t = catalog.loadTable(id); + Schema inputSchema = inputSchemaFor(t); + + TestStream> stream = + testStream(inputSchema) + .advanceWatermarkTo(BASE) + .addElements(at(INSERT, row(inputSchema, 1, MARCH, "a", 1L), 1)) + .advanceWatermarkTo(BASE.plus(WINDOW)) // window 1 commits the March row + .addElements(at(UPDATE_BEFORE, row(inputSchema, 1, MARCH, "a", 2L), 61)) + .advanceWatermarkTo(BASE.plus(WINDOW).plus(WINDOW)) // window 2: the bare UB + .addElements(at(UPDATE_AFTER, row(inputSchema, 1, JUNE, "a2", 2L), 121)) + .advanceWatermarkToInfinity(); // window 3: the bare UA + + streamingInput(inputSchema, stream) + .apply( + IcebergIO.writeCdcRows(catalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withTriggeringFrequency(WINDOW)); + p.run().waitUntilFinish(); + + t.refresh(); + assertThat(readColumns(t, "id", "ts", "name"), contains("1:" + micros(JUNE) + ":a2")); + + // The per-commit file trail: data at March; the bare UB's delete at March; data at June. + List snapshots = + Lists.newArrayList(t.snapshots()).stream() + .sorted(Comparator.comparingLong(Snapshot::sequenceNumber)) + .collect(Collectors.toList()); + assertThat(snapshots, hasSize(3)); + + List w1Data = Lists.newArrayList(snapshots.get(0).addedDataFiles(t.io())); + assertThat(w1Data, hasSize(1)); + assertThat(w1Data.get(0).partition().get(0, Integer.class), equalTo(MARCH_DAY)); + + assertThat(Lists.newArrayList(snapshots.get(1).addedDataFiles(t.io())), empty()); + List w2Deletes = Lists.newArrayList(snapshots.get(1).addedDeleteFiles(t.io())); + assertThat(w2Deletes, hasSize(1)); + assertThat(w2Deletes.get(0).content(), equalTo(FileContent.EQUALITY_DELETES)); + assertThat(w2Deletes.get(0).partition().get(0, Integer.class), equalTo(MARCH_DAY)); + + assertThat(Lists.newArrayList(snapshots.get(2).addedDeleteFiles(t.io())), empty()); + List w3Data = Lists.newArrayList(snapshots.get(2).addedDataFiles(t.io())); + assertThat(w3Data, hasSize(1)); + assertThat(w3Data.get(0).partition().get(0, Integer.class), equalTo(JUNE_DAY)); + } + + /** + * The bare-UA outcome on a non-key-partitioned table, pinned as intended behavior: without its + * {@code UPDATE_BEFORE} (the input contract for such tables, see the package-info partitioning + * section), a moved row cannot be deleted from the partition it occupied, so the old image stays + * live beside the new one and no delete file is written. + */ + @Test + public void nonKeyPartitionedBareUpdateAfterLeavesTheOldRowLive() throws IOException { + TableIdentifier id = createNonKeyDayPartitionedTable("nonkeybareua"); + Table t = catalog.loadTable(id); + Schema inputSchema = inputSchemaFor(t); + + TestStream> stream = + testStream(inputSchema) + .advanceWatermarkTo(BASE) + .addElements(at(INSERT, row(inputSchema, 1, MARCH, "a", 1L), 1)) + .advanceWatermarkTo(BASE.plus(WINDOW)) // window 1 commits the March row + .addElements(at(UPDATE_AFTER, row(inputSchema, 1, JUNE, "a2", 2L), 61)) + .advanceWatermarkToInfinity(); // window 2: the bare UA + + streamingInput(inputSchema, stream) + .apply( + IcebergIO.writeCdcRows(catalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withTriggeringFrequency(WINDOW)); + p.run().waitUntilFinish(); + + assertThat( + readColumns(t, "id", "ts", "name"), + containsInAnyOrder("1:" + micros(MARCH) + ":a", "1:" + micros(JUNE) + ":a2")); + assertThat( + scannedPartitions(t, 1), + containsInAnyOrder(String.valueOf(MARCH_DAY), String.valueOf(JUNE_DAY))); + assertThat(addedDeleteFiles(t), empty()); + } + + // ----------------------------------------------------------------------------------------------- + // 17. One runId end to end + // ----------------------------------------------------------------------------------------------- + + /** + * The committed run-spec stamp, the writer's file names, and spec adoption agree on one runId: + * the stamp's runId prefix is the runId embedded in the committed data file names, and a {@link + * TableSetup} built with it adopts the stamped spec over a live evolution. Fails if {@code + * WriteCdcRows.expand()} hands its stages different runIds. + */ + @Test + public void runIdThreadsIdenticallyThroughTheAssembledTransform() { + TableIdentifier id = + createTable( + "runid", + ICEBERG_SCHEMA, + ImmutableSet.of(1), + 2, + PartitionSpec.builderFor(ICEBERG_SCHEMA).bucket("id", 4).build()); + Table t = catalog.loadTable(id); + + boundedInput(INPUT_SCHEMA, KV.of(INSERT, row(1, "a", "x", 1L))) + .apply( + IcebergIO.writeCdcRows(catalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withSinkId("runid-sink")); + p.run().waitUntilFinish(); + + t.refresh(); + Snapshot committed = checkStateNotNull(t.currentSnapshot()); + String stamp = checkStateNotNull(committed.summary()).get("beam.cdc.run-spec.runid-sink"); + assertThat(stamp, notNullValue()); + int cut = stamp.lastIndexOf(':'); + String runId = stamp.substring(0, cut); + int stampedSpecId = Integer.parseInt(stamp.substring(cut + 1)); + + // The commit stage's stamped runId is the writer stage's runId, part of every file name. + List dataFiles = Lists.newArrayList(committed.addedDataFiles(t.io())); + assertThat(dataFiles, not(empty())); + for (DataFile file : dataFiles) { + assertThat(file.location(), containsString(runId)); + } + + // A TableSetup built with the same runId (the assign/write stages' construction) adopts the + // stamped spec over the live evolved one. + t.updateSpec().addField(Expressions.bucket("id", 8)).commit(); + t.refresh(); + assertThat(t.spec().specId(), not(equalTo(stampedSpecId))); + TableSetup joining = + new TableSetup( + catalogConfig(), + CdcWriteConfig.builder().setSinkId("runid-sink").setSequenceNumberColumn("seq").build(), + DynamicDestinations.singleTable(id, DATA_SCHEMA), + runId); + assertThat(joining.get(id.toString(), DATA_SCHEMA).spec().specId(), equalTo(stampedSpecId)); + } +} diff --git a/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRowsTest.java b/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRowsTest.java new file mode 100644 index 000000000000..1e3a2a03d121 --- /dev/null +++ b/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRowsTest.java @@ -0,0 +1,1499 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.iceberg.cdc.sink; + +import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull; +import static org.apache.beam.sdk.values.ValueKind.DELETE; +import static org.apache.beam.sdk.values.ValueKind.INSERT; +import static org.apache.beam.sdk.values.ValueKind.UPDATE_AFTER; +import static org.apache.beam.sdk.values.ValueKind.UPDATE_BEFORE; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.lessThan; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; + +import java.util.ArrayList; +import java.util.List; +import org.apache.beam.sdk.Pipeline; +import org.apache.beam.sdk.PipelineResult; +import org.apache.beam.sdk.coders.CoderException; +import org.apache.beam.sdk.coders.KvCoder; +import org.apache.beam.sdk.coders.RowCoder; +import org.apache.beam.sdk.coders.SerializableCoder; +import org.apache.beam.sdk.io.iceberg.IcebergCatalogConfig; +import org.apache.beam.sdk.io.iceberg.IcebergIO; +import org.apache.beam.sdk.io.iceberg.IcebergWriteResult; +import org.apache.beam.sdk.metrics.MetricNameFilter; +import org.apache.beam.sdk.metrics.MetricResult; +import org.apache.beam.sdk.metrics.MetricsFilter; +import org.apache.beam.sdk.schemas.Schema; +import org.apache.beam.sdk.testing.ExpectedLogs; +import org.apache.beam.sdk.testing.PAssert; +import org.apache.beam.sdk.testing.TestPipeline; +import org.apache.beam.sdk.testing.TestStream; +import org.apache.beam.sdk.transforms.Create; +import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.transforms.MapElements; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.transforms.windowing.AfterPane; +import org.apache.beam.sdk.transforms.windowing.GlobalWindows; +import org.apache.beam.sdk.transforms.windowing.Repeatedly; +import org.apache.beam.sdk.transforms.windowing.Window; +import org.apache.beam.sdk.util.CoderUtils; +import org.apache.beam.sdk.values.KV; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.Row; +import org.apache.beam.sdk.values.TimestampedValue; +import org.apache.beam.sdk.values.TypeDescriptor; +import org.apache.beam.sdk.values.TypeDescriptors; +import org.apache.beam.sdk.values.ValueKind; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.Table; +import org.apache.iceberg.catalog.Catalog; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.data.IcebergGenerics; +import org.apache.iceberg.data.Record; +import org.apache.iceberg.hadoop.HadoopCatalog; +import org.apache.iceberg.types.Types; +import org.joda.time.Duration; +import org.joda.time.Instant; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Tests for {@link WriteCdcRows}, the top-level CDC sink {@code PTransform} exposed via {@link + * IcebergIO#writeCdcRows}. End-to-end tests drive a real {@link HadoopCatalog} V2 table on the + * DirectRunner; each test uses a unique table identifier so the process-wide TableCache never sees + * a repeat. + */ +@RunWith(JUnit4.class) +public class WriteCdcRowsTest { + + @Rule public transient TestPipeline p = TestPipeline.create(); + @Rule public transient TemporaryFolder tmp = new TemporaryFolder(); + @Rule public transient ExpectedLogs expectedLogs = ExpectedLogs.none(WriteCdcRows.class); + + /** Canonical test table schema. */ + private static final org.apache.iceberg.Schema ICEBERG_SCHEMA = + new org.apache.iceberg.Schema( + Types.NestedField.required(1, "id", Types.IntegerType.get()), + Types.NestedField.optional(2, "name", Types.StringType.get()), + Types.NestedField.optional(3, "data", Types.StringType.get())); + + private static final Schema DATA_SCHEMA = + Schema.builder() + .addInt32Field("id") + .addNullableField("name", Schema.FieldType.STRING) + .addNullableField("data", Schema.FieldType.STRING) + .build(); + + /** Input schema = data schema + a sequence-number column named {@code seq}. */ + private static final Schema INPUT_SCHEMA = + Schema.builder().addFields(DATA_SCHEMA.getFields()).addInt64Field("seq").build(); + + /** The default sequence-number column name (see {@code CdcWriteConfig}). */ + private static final String DEFAULT_SEQ_COL = "_commit_snapshot_sequence_number"; + + /** Replayed dead letters are re-stamped here so the replay lands in an open window. */ + private static final Instant REPLAY_EVENT_TIME = new Instant(0).plus(Duration.standardMinutes(2)); + + private Catalog catalog; + + private void setUpCatalog() { + catalog = CdcSinkTestUtils.hadoopCatalog(tmp.getRoot()); + } + + private IcebergCatalogConfig catalogConfig() { + return CdcSinkTestUtils.catalogConfig(tmp.getRoot()); + } + + /** A catalog config pointing at an unreachable warehouse (for no-catalog-at-expand tests). */ + private static IcebergCatalogConfig bogusCatalogConfig() { + return IcebergCatalogConfig.builder() + .setCatalogProperties( + ImmutableMap.of( + "type", "hadoop", "warehouse", "file:/nonexistent-cdc-" + System.nanoTime())) + .build(); + } + + /** Creates a fresh unpartitioned V2 table (PK = {@code id}) with a unique identifier. */ + private TableIdentifier v2Table() { + TableIdentifier id = TableIdentifier.of("db", "t" + System.nanoTime()); + createV2Table(id); + return id; + } + + /** Creates a fresh unpartitioned V2 table with the given identifier (PK = {@code id}). */ + private void createV2Table(TableIdentifier id) { + CdcSinkTestUtils.createTable( + catalog, id, ICEBERG_SCHEMA, ImmutableSet.of(1), 2, PartitionSpec.unpartitioned()); + } + + /** Builds an input data+seq {@link Row}. */ + private static Row row(int id, String name, String data, long seq) { + return Row.withSchema(INPUT_SCHEMA).addValues(id, name, data, seq).build(); + } + + /** Builds an input data-only {@link Row} (no {@code seq} column). */ + private static Row rowNoSeq(int id, String name, String data) { + return Row.withSchema(DATA_SCHEMA).addValues(id, name, data).build(); + } + + /** Reads all live rows of {@code table} as sorted {@code id:name:data} strings. */ + private static List readRows(Table table) { + table.refresh(); + return ImmutableList.copyOf(IcebergGenerics.read(table).build()).stream() + .map(r -> r.getField("id") + ":" + r.getField("name") + ":" + r.getField("data")) + .sorted() + .collect(ImmutableList.toImmutableList()); + } + + /** A bounded CDC input of the given kind-tagged rows over {@code schema}. */ + @SafeVarargs + private final PCollection boundedInput(Schema schema, KV... rows) { + return CdcSinkTestUtils.withKinds(p.apply(Create.of(ImmutableList.copyOf(rows)))) + .setRowSchema(schema); + } + + /** An unbounded single-INSERT CDC input over {@link #INPUT_SCHEMA}. */ + private PCollection unboundedInput() { + TestStream stream = + TestStream.create(RowCoder.of(INPUT_SCHEMA)) + .addElements(row(1, "a", "x", 1L)) + .advanceWatermarkToInfinity(); + PCollection> tagged = + p.apply(stream) + .setRowSchema(INPUT_SCHEMA) + .apply( + MapElements.into( + TypeDescriptors.kvs( + TypeDescriptor.of(ValueKind.class), TypeDescriptors.rows())) + .via(r -> KV.of(INSERT, r))) + .setCoder(KvCoder.of(SerializableCoder.of(ValueKind.class), RowCoder.of(INPUT_SCHEMA))); + return CdcSinkTestUtils.withKinds(tagged).setRowSchema(INPUT_SCHEMA); + } + + /** + * The {@code beam.cdc.committed-through-ms.} token of each of {@code table}'s snapshots, + * in snapshot order. + */ + private static List committedThroughTokens(Table table, String sinkId) { + table.refresh(); + List tokens = new ArrayList<>(); + for (Snapshot s : table.snapshots()) { + tokens.add(Long.parseLong(s.summary().get("beam.cdc.committed-through-ms." + sinkId))); + } + return tokens; + } + + /** Sums the named {@link CommitDeltas} counter committed by the pipeline. */ + private static long committerCounter(PipelineResult result, String name) { + long total = 0; + for (MetricResult c : + result + .metrics() + .queryMetrics( + MetricsFilter.builder() + .addNameFilter(MetricNameFilter.named(CommitDeltas.class, name)) + .build()) + .getCounters()) { + total += c.getCommitted(); + } + return total; + } + + // --------------------------------------------------------------------------------------------- + // 1. Construction-time validation matrix + // --------------------------------------------------------------------------------------------- + + /** Exactly one destination: none at all and both kinds at once are each rejected. */ + @Test + public void constructionRequiresExactlyOneDestination() { + p.enableAbandonedNodeEnforcement(false); + TableIdentifier id = TableIdentifier.of("db", "both_" + System.nanoTime()); + PCollection in = boundedInput(INPUT_SCHEMA, KV.of(INSERT, row(1, "a", "x", 1L))); + + // facet: no destination. + IllegalArgumentException none = + assertThrows( + IllegalArgumentException.class, + () -> + in.apply( + "NoDestination", + IcebergIO.writeCdcRows(bogusCatalogConfig()).withSequenceNumberColumn("seq"))); + assertThat(none.getMessage(), containsString("exactly one")); + + // facet: both destination kinds. + IllegalArgumentException both = + assertThrows( + IllegalArgumentException.class, + () -> + in.apply( + "BothDestinations", + IcebergIO.writeCdcRows(bogusCatalogConfig()) + .to(id) + .to( + CdcSinkTestUtils.templatedDestinations( + "db.{name}", INPUT_SCHEMA, "seq")) + .withSequenceNumberColumn("seq"))); + assertThat(both.getMessage(), containsString("exactly one")); + } + + /** An unbounded input requires a triggering frequency, and it must be positive. */ + @Test + public void unboundedInputRequiresPositiveTriggeringFrequency() { + p.enableAbandonedNodeEnforcement(false); + TableIdentifier id = TableIdentifier.of("db", "stream_tf_" + System.nanoTime()); + PCollection in = unboundedInput(); + + IllegalArgumentException missing = + assertThrows( + IllegalArgumentException.class, + () -> + in.apply( + "NoFrequency", + IcebergIO.writeCdcRows(bogusCatalogConfig()) + .to(id) + .withSequenceNumberColumn("seq"))); + assertThat(missing.getMessage(), containsString("withTriggeringFrequency")); + IllegalArgumentException zero = + assertThrows( + IllegalArgumentException.class, + () -> + in.apply( + "ZeroFrequency", + IcebergIO.writeCdcRows(bogusCatalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withTriggeringFrequency(Duration.ZERO))); + assertThat(zero.getMessage(), containsString("must be positive")); + IllegalArgumentException negative = + assertThrows( + IllegalArgumentException.class, + () -> + in.apply( + "NegativeFrequency", + IcebergIO.writeCdcRows(bogusCatalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withTriggeringFrequency(Duration.standardSeconds(-30)))); + assertThat(negative.getMessage(), containsString("must be positive")); + } + + /** + * Dead-letter metadata is nested under {@code record}, so data columns named like it ({@code + * _cdc_change_type}, {@code change_type}) are ordinary columns and write end-to-end. + */ + @Test + public void deadLetterMetadataNamedDataColumnsWriteFine() { + setUpCatalog(); + TableIdentifier id = TableIdentifier.of("db", "dlname_" + System.nanoTime()); + CdcSinkTestUtils.createTable( + catalog, + id, + new org.apache.iceberg.Schema( + Types.NestedField.required(1, "id", Types.IntegerType.get()), + Types.NestedField.optional(2, "_cdc_change_type", Types.StringType.get()), + Types.NestedField.optional(3, "change_type", Types.StringType.get())), + ImmutableSet.of(1), + 2, + PartitionSpec.unpartitioned()); + + Schema inputSchema = + Schema.builder() + .addInt32Field("id") + .addNullableField("_cdc_change_type", Schema.FieldType.STRING) + .addNullableField("change_type", Schema.FieldType.STRING) + .addInt64Field("seq") + .build(); + Row r = Row.withSchema(inputSchema).addValues(1, "c", "u", 1L).build(); + boundedInput(inputSchema, KV.of(INSERT, r)) + .apply(IcebergIO.writeCdcRows(catalogConfig()).to(id).withSequenceNumberColumn("seq")); + p.run().waitUntilFinish(); + + Table t = catalog.loadTable(id); + Record written = + Iterables.getOnlyElement(ImmutableList.copyOf(IcebergGenerics.read(t).build())); + assertThat(written.getField("id"), equalTo(1)); + assertThat(written.getField("_cdc_change_type"), equalTo("c")); + assertThat(written.getField("change_type"), equalTo("u")); + } + + /** The sequence-number column must exist and be a non-nullable INT64. */ + @Test + public void constructionRejectsInvalidSequenceColumnConfigs() { + p.enableAbandonedNodeEnforcement(false); + TableIdentifier id = TableIdentifier.of("db", "badseq_" + System.nanoTime()); + + // facet: missing (the default name is absent from the input schema). + PCollection noSeq = boundedInput(DATA_SCHEMA, KV.of(INSERT, rowNoSeq(1, "a", "x"))); + IllegalArgumentException missing = + assertThrows( + IllegalArgumentException.class, + () -> noSeq.apply("MissingSeq", IcebergIO.writeCdcRows(bogusCatalogConfig()).to(id))); + assertThat(missing.getMessage(), containsString(DEFAULT_SEQ_COL)); + assertThat(missing.getMessage(), containsString("sequence-number column")); + + // facet: present but STRING. + Schema stringSeq = + Schema.builder().addFields(DATA_SCHEMA.getFields()).addStringField("seq").build(); + PCollection badSeq = + boundedInput( + stringSeq, + KV.of( + INSERT, Row.withSchema(stringSeq).addValues(1, "a", "x", "not-a-number").build())); + IllegalArgumentException nonInt64 = + assertThrows( + IllegalArgumentException.class, + () -> + badSeq.apply( + "StringSeq", + IcebergIO.writeCdcRows(bogusCatalogConfig()) + .to(id) + .withSequenceNumberColumn("seq"))); + assertThat(nonInt64.getMessage(), containsString("sequence-number column 'seq'")); + assertThat(nonInt64.getMessage(), containsString("must be INT64")); + assertThat(nonInt64.getMessage(), containsString("STRING")); + + // facet: INT64 but declared nullable. + Schema nullableSeq = + Schema.builder() + .addFields(DATA_SCHEMA.getFields()) + .addNullableField("seq", Schema.FieldType.INT64) + .build(); + PCollection nullableIn = + boundedInput( + nullableSeq, + KV.of(INSERT, Row.withSchema(nullableSeq).addValues(1, "a", "x", 1L).build())); + IllegalArgumentException nullable = + assertThrows( + IllegalArgumentException.class, + () -> + nullableIn.apply( + "NullableSeq", + IcebergIO.writeCdcRows(bogusCatalogConfig()) + .to(id) + .withSequenceNumberColumn("seq"))); + assertThat(nullable.getMessage(), containsString("sequence-number column 'seq'")); + assertThat(nullable.getMessage(), containsString("must be non-nullable")); + } + + /** + * The change-type column must exist (it is the cross-language default path, so a typo otherwise + * fails every record at runtime), be a non-nullable STRING, and be distinct from the + * sequence-number column. + */ + @Test + public void constructionRejectsInvalidChangeTypeColumnConfigs() { + p.enableAbandonedNodeEnforcement(false); + TableIdentifier id = TableIdentifier.of("db", "badct_" + System.nanoTime()); + PCollection in = boundedInput(INPUT_SCHEMA, KV.of(INSERT, row(1, "a", "x", 1L))); + + // facet: names a column the input does not have. + IllegalArgumentException missing = + assertThrows( + IllegalArgumentException.class, + () -> + in.apply( + "MissingChangeType", + IcebergIO.writeCdcRows(bogusCatalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withChangeTypeColumn("op_typo"))); + assertThat(missing.getMessage(), containsString("change-type column 'op_typo'")); + assertThat(missing.getMessage(), containsString("change_type_column")); + + // facet: present but INT32. + Schema intOp = Schema.builder().addFields(INPUT_SCHEMA.getFields()).addInt32Field("op").build(); + PCollection badOp = + boundedInput( + intOp, KV.of(INSERT, Row.withSchema(intOp).addValues(1, "a", "x", 1L, 7).build())); + IllegalArgumentException nonString = + assertThrows( + IllegalArgumentException.class, + () -> + badOp.apply( + "IntChangeType", + IcebergIO.writeCdcRows(bogusCatalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withChangeTypeColumn("op"))); + assertThat(nonString.getMessage(), containsString("change-type column 'op'")); + assertThat(nonString.getMessage(), containsString("must be STRING")); + assertThat(nonString.getMessage(), containsString("INT32")); + + // facet: STRING but declared nullable. + Schema nullableOp = + Schema.builder() + .addFields(INPUT_SCHEMA.getFields()) + .addNullableField("op", Schema.FieldType.STRING) + .build(); + PCollection nullableIn = + boundedInput( + nullableOp, + KV.of(INSERT, Row.withSchema(nullableOp).addValues(1, "a", "x", 1L, "I").build())); + IllegalArgumentException nullable = + assertThrows( + IllegalArgumentException.class, + () -> + nullableIn.apply( + "NullableChangeType", + IcebergIO.writeCdcRows(bogusCatalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withChangeTypeColumn("op"))); + assertThat(nullable.getMessage(), containsString("change-type column 'op'")); + assertThat(nullable.getMessage(), containsString("must be non-nullable")); + + // facet: collides with the sequence-number column. + IllegalArgumentException collides = + assertThrows( + IllegalArgumentException.class, + () -> + in.apply( + "ChangeTypeIsSeq", + IcebergIO.writeCdcRows(bogusCatalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withChangeTypeColumn("seq"))); + assertThat(collides.getMessage(), containsString("must be distinct")); + } + + /** + * The shard-config bounds: {@code num_shards >= 1}, {@code 1 <= shards_per_partition <= + * num_shards}, with the boundary {@code shards_per_partition == num_shards} (the uncapped + * default) accepted. + */ + @Test + public void constructionValidatesShardConfigBounds() { + p.enableAbandonedNodeEnforcement(false); + TableIdentifier id = TableIdentifier.of("db", "shards_" + System.nanoTime()); + PCollection in = boundedInput(INPUT_SCHEMA, KV.of(INSERT, row(1, "a", "x", 1L))); + + // facet: num_shards = 0. + IllegalArgumentException zeroShards = + assertThrows( + IllegalArgumentException.class, + () -> + in.apply( + "ZeroShards", + IcebergIO.writeCdcRows(bogusCatalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withNumShards(0))); + assertThat(zeroShards.getMessage(), containsString("num_shards must be >= 1")); + + // facet: shards_per_partition = 0, which names the option and both bounds (num_shards + // defaults to 16 here). + IllegalArgumentException sppZero = + assertThrows( + IllegalArgumentException.class, + () -> + in.apply( + "ZeroShardsPerPartition", + IcebergIO.writeCdcRows(bogusCatalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withShardsPerPartition(0))); + assertThat( + sppZero.getMessage(), + containsString("shards_per_partition must be between 1 and num_shards")); + assertThat(sppZero.getMessage(), containsString("(16)")); + assertThat(sppZero.getMessage(), containsString("got 0")); + + // facet: shards_per_partition above num_shards. + IllegalArgumentException sppAbove = + assertThrows( + IllegalArgumentException.class, + () -> + in.apply( + "ShardsPerPartitionAboveNumShards", + IcebergIO.writeCdcRows(bogusCatalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withNumShards(16) + .withShardsPerPartition(32))); + assertThat(sppAbove.getMessage(), containsString("shards_per_partition")); + assertThat(sppAbove.getMessage(), containsString("32")); + assertThat(sppAbove.getMessage(), containsString("16")); + + // facet: shards_per_partition == num_shards must build (never run). + in.apply( + "ShardsPerPartitionEqualNumShards", + IcebergIO.writeCdcRows(bogusCatalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withNumShards(16) + .withShardsPerPartition(16)); + } + + /** + * A reserved {@code beam.cdc.} snapshot-property key and an explicitly empty equality-column list + * are each rejected at construction. + */ + @Test + public void constructionRejectsReservedSnapshotPropertiesAndEmptyEqualityColumns() { + p.enableAbandonedNodeEnforcement(false); + TableIdentifier id = TableIdentifier.of("db", "props_" + System.nanoTime()); + PCollection in = boundedInput(INPUT_SCHEMA, KV.of(INSERT, row(1, "a", "x", 1L))); + + IllegalArgumentException reserved = + assertThrows( + IllegalArgumentException.class, + () -> + in.apply( + "ReservedProperty", + IcebergIO.writeCdcRows(bogusCatalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withSnapshotProperties(ImmutableMap.of("beam.cdc.foo", "x")))); + assertThat(reserved.getMessage(), containsString("beam.cdc.")); + + IllegalArgumentException emptyEquality = + assertThrows( + IllegalArgumentException.class, + () -> + in.apply( + "EmptyEqualityColumns", + IcebergIO.writeCdcRows(bogusCatalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withEqualityColumns(ImmutableList.of()))); + assertThat(emptyEquality.getMessage(), containsString("non-empty or unset")); + } + + /** {@code withSorterMemoryMB} accepts a positive value and rejects a non-positive one eagerly. */ + @Test + public void sorterMemoryMbValidatedAtConstruction() { + TableIdentifier id = TableIdentifier.of("db", "sorter_" + System.nanoTime()); + IcebergIO.writeCdcRows(bogusCatalogConfig()).to(id).withSorterMemoryMB(50); + IllegalArgumentException ex = + assertThrows( + IllegalArgumentException.class, + () -> IcebergIO.writeCdcRows(bogusCatalogConfig()).to(id).withSorterMemoryMB(0)); + assertThat(ex.getMessage(), containsString("sorter_memory_mb")); + } + + // --------------------------------------------------------------------------------------------- + // 2. Launcher touches no catalog + // --------------------------------------------------------------------------------------------- + + /** + * Neither the single-table nor the dynamic-destination branch may touch the catalog at {@code + * expand()}: building against an unreachable warehouse (and a nonexistent table) must succeed; + * schema resolution is deferred to the workers. The pipeline is never run. + */ + @Test + public void expandTouchesNoCatalog() { + p.enableAbandonedNodeEnforcement(false); + TableIdentifier id = TableIdentifier.of("db", "no_such_" + System.nanoTime()); + PCollection in = boundedInput(INPUT_SCHEMA, KV.of(INSERT, row(1, "a", "x", 1L))); + in.apply( + "SingleTable", + IcebergIO.writeCdcRows(bogusCatalogConfig()).to(id).withSequenceNumberColumn("seq")); + in.apply( + "DynamicDestinations", + IcebergIO.writeCdcRows(bogusCatalogConfig()) + .to(CdcSinkTestUtils.templatedDestinations("db.{name}", INPUT_SCHEMA, "seq")) + .withSequenceNumberColumn("seq")); + } + + // --------------------------------------------------------------------------------------------- + // 3. Batch end-to-end + // --------------------------------------------------------------------------------------------- + + @Test + public void batchCdcAppliesInsertsUpdatesDeletes() { + setUpCatalog(); + TableIdentifier id = v2Table(); + Table t = catalog.loadTable(id); + + PCollection in = + boundedInput( + INPUT_SCHEMA, + KV.of(INSERT, row(1, "a", "x", 1L)), + KV.of(INSERT, row(2, "b", "y", 1L)), + KV.of(UPDATE_BEFORE, row(1, "a", "x", 2L)), + KV.of(UPDATE_AFTER, row(1, "a", "z", 2L)), + KV.of(DELETE, row(2, "b", "y", 3L))); + IcebergWriteResult result = + in.apply(IcebergIO.writeCdcRows(catalogConfig()).to(id).withSequenceNumberColumn("seq")); + + PAssert.that(result.getSnapshots()) + .satisfies( + snapshots -> { + assertFalse(Iterables.isEmpty(snapshots)); + return null; + }); + p.run().waitUntilFinish(); + + // id=1 updated to (a,z); id=2 deleted. + assertThat(readRows(t), contains("1:a:z")); + } + + /** + * A bounded input carrying a NON-default upstream trigger must still commit every row exactly + * once: {@code CommitWindows}' bounded branch has to pin its trigger explicitly, or the inherited + * trigger fires several panes per key and every pane past the first is skipped as a same-end + * twin: rows in neither the table nor the dead letters. {@code withNumShards(1)} funnels all rows + * through one key so a pane split cannot hide in the sharding. + */ + @Test + public void boundedInputWithUpstreamTriggerCommitsAllRowsExactlyOnce() { + setUpCatalog(); + TableIdentifier id = v2Table(); + Table t = catalog.loadTable(id); + + PCollection in = + boundedInput( + INPUT_SCHEMA, + KV.of(INSERT, row(1, "a", "x1", 1L)), + KV.of(INSERT, row(2, "b", "x2", 1L)), + KV.of(INSERT, row(3, "c", "x3", 1L)), + KV.of(INSERT, row(4, "d", "x4", 1L)), + KV.of(INSERT, row(5, "e", "x5", 1L)), + KV.of(INSERT, row(6, "f", "x6", 1L)), + KV.of(INSERT, row(7, "g", "x7", 1L)), + KV.of(INSERT, row(8, "h", "x8", 1L))) + .apply( + "NonDefaultUpstreamTrigger", + Window.into(new GlobalWindows()) + .triggering(Repeatedly.forever(AfterPane.elementCountAtLeast(1))) + .discardingFiredPanes()); + + in.apply( + IcebergIO.writeCdcRows(catalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withNumShards(1)); + PipelineResult result = p.run(); + result.waitUntilFinish(); + + // readRows lists ALL live rows: a lost pane or a double-commit both fail this. + assertThat( + readRows(t), + contains("1:a:x1", "2:b:x2", "3:c:x3", "4:d:x4", "5:e:x5", "6:f:x6", "7:g:x7", "8:h:x8")); + assertThat(committerCounter(result, "alreadyCommittedWindowsSkipped"), equalTo(0L)); + } + + // --------------------------------------------------------------------------------------------- + // 4. Streaming two-window end-to-end + // --------------------------------------------------------------------------------------------- + + /** Two event-time windows commit as two snapshots with ascending tokens; final state correct. */ + @Test + public void streamingTwoWindowsCommitsPerWindow() { + setUpCatalog(); + TableIdentifier id = v2Table(); + Table t = catalog.loadTable(id); + String sinkId = "sink-" + System.nanoTime(); + Duration window = Duration.standardSeconds(60); + Instant base = new Instant(0); + + TestStream> stream = + TestStream.create( + KvCoder.of(SerializableCoder.of(ValueKind.class), RowCoder.of(INPUT_SCHEMA))) + .advanceWatermarkTo(base) + .addElements( + TimestampedValue.of( + KV.of(INSERT, row(1, "a", "x", 1L)), base.plus(Duration.standardSeconds(1))), + TimestampedValue.of( + KV.of(INSERT, row(2, "b", "y", 1L)), base.plus(Duration.standardSeconds(2)))) + .advanceWatermarkTo(base.plus(Duration.standardSeconds(70))) // window 0 commits + .addElements( + TimestampedValue.of( + KV.of(UPDATE_BEFORE, row(1, "a", "x", 2L)), + base.plus(Duration.standardSeconds(61))), + TimestampedValue.of( + KV.of(UPDATE_AFTER, row(1, "a2", "z", 2L)), + base.plus(Duration.standardSeconds(61))), + TimestampedValue.of( + KV.of(DELETE, row(2, "b", "y", 2L)), base.plus(Duration.standardSeconds(62)))) + .advanceWatermarkTo(base.plus(Duration.standardSeconds(130))) // window 1 commits + .advanceWatermarkToInfinity(); + + PCollection in = CdcSinkTestUtils.withKinds(p.apply(stream)).setRowSchema(INPUT_SCHEMA); + in.apply( + IcebergIO.writeCdcRows(catalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withTriggeringFrequency(window) + .withSinkId(sinkId)); + p.run().waitUntilFinish(); + + List tokens = committedThroughTokens(t, sinkId); + assertThat(tokens, hasSize(2)); + assertThat(tokens.get(0), lessThan(tokens.get(1))); + assertThat(readRows(t), contains("1:a2:z")); + } + + // --------------------------------------------------------------------------------------------- + // 5. Dynamic destinations (template) + control-column strip + // --------------------------------------------------------------------------------------------- + + /** + * A {@code db.{dest}} template routes rows to two tables; the default sequence-number column is + * stripped from the written rows, so neither table needs (or gains) it. + */ + @Test + public void dynamicDestinationsRouteAndStripSequenceColumn() { + setUpCatalog(); + long suffix = System.nanoTime(); + String tableA = "tmpl_a" + suffix; + String tableB = "tmpl_b" + suffix; + // Tables whose columns are (id, dest): the routing column is also a data column. + CdcSinkTestUtils.createDestTables(catalog, tableA, tableB); + + Schema inputSchema = + Schema.builder() + .addInt32Field("id") + .addNullableField("dest", Schema.FieldType.STRING) + .addInt64Field(DEFAULT_SEQ_COL) + .build(); + Row rowA = Row.withSchema(inputSchema).addValues(1, tableA, 1L).build(); + Row rowB = Row.withSchema(inputSchema).addValues(2, tableB, 1L).build(); + + PCollection in = boundedInput(inputSchema, KV.of(INSERT, rowA), KV.of(INSERT, rowB)); + in.apply( + IcebergIO.writeCdcRows(catalogConfig()) + .to(CdcSinkTestUtils.templatedDestinations("db.{dest}", inputSchema, DEFAULT_SEQ_COL))); + p.run().waitUntilFinish(); + + Table a = catalog.loadTable(TableIdentifier.of("db", tableA)); + Table b = catalog.loadTable(TableIdentifier.of("db", tableB)); + assertThat( + ImmutableList.copyOf(IcebergGenerics.read(a).build()).stream() + .map(r -> r.getField("id") + ":" + r.getField("dest")) + .collect(ImmutableList.toImmutableList()), + contains("1:" + tableA)); + assertThat( + ImmutableList.copyOf(IcebergGenerics.read(b).build()).stream() + .map(r -> r.getField("id") + ":" + r.getField("dest")) + .collect(ImmutableList.toImmutableList()), + contains("2:" + tableB)); + a.refresh(); + b.refresh(); + assertNull(a.schema().findField(DEFAULT_SEQ_COL)); + assertNull(b.schema().findField(DEFAULT_SEQ_COL)); + } + + /** + * Streaming + dynamic destinations, the combination the suite otherwise never runs. A receives + * data in windows 0 and 2, B in 1 and 2, so each destination's committer state and commit timer + * must advance on its own schedule; state shared across destinations would skip a window as + * "already committed" for the OTHER destination's end and silently lose its rows. + */ + @Test + public void streamingDynamicDestinationsCommitPerDestinationWindows() { + setUpCatalog(); + long suffix = System.nanoTime(); + String tableA = "stream_a" + suffix; + String tableB = "stream_b" + suffix; + CdcSinkTestUtils.createDestTables(catalog, tableA, tableB); + + Schema inputSchema = + Schema.builder() + .addInt32Field("id") + .addNullableField("dest", Schema.FieldType.STRING) + .addInt64Field(DEFAULT_SEQ_COL) + .build(); + String sinkId = "sink-" + suffix; + Duration window = Duration.standardSeconds(60); + Instant base = new Instant(0); + + TestStream> stream = + TestStream.create( + KvCoder.of(SerializableCoder.of(ValueKind.class), RowCoder.of(inputSchema))) + .advanceWatermarkTo(base) + // Window 0: destination A only. + .addElements( + TimestampedValue.of( + KV.of(INSERT, destRow(inputSchema, 1, tableA, 1L)), + base.plus(Duration.standardSeconds(1))), + TimestampedValue.of( + KV.of(INSERT, destRow(inputSchema, 5, tableA, 1L)), + base.plus(Duration.standardSeconds(2)))) + .advanceWatermarkTo(base.plus(Duration.standardSeconds(70))) + // Window 1: destination B only. + .addElements( + TimestampedValue.of( + KV.of(INSERT, destRow(inputSchema, 2, tableB, 2L)), + base.plus(Duration.standardSeconds(61)))) + .advanceWatermarkTo(base.plus(Duration.standardSeconds(130))) + // Window 2: both destinations. A's DELETE removes a key committed in window 0, so it + // also proves A's cross-window equality delete landed in the right order. + .addElements( + TimestampedValue.of( + KV.of(DELETE, destRow(inputSchema, 1, tableA, 3L)), + base.plus(Duration.standardSeconds(121))), + TimestampedValue.of( + KV.of(INSERT, destRow(inputSchema, 3, tableB, 3L)), + base.plus(Duration.standardSeconds(122)))) + .advanceWatermarkTo(base.plus(Duration.standardSeconds(190))) + .advanceWatermarkToInfinity(); + + PCollection in = CdcSinkTestUtils.withKinds(p.apply(stream)).setRowSchema(inputSchema); + IcebergWriteResult result = + in.apply( + IcebergIO.writeCdcRows(catalogConfig()) + .to( + CdcSinkTestUtils.templatedDestinations( + "db.{dest}", inputSchema, DEFAULT_SEQ_COL)) + .withTriggeringFrequency(window) + .withSinkId(sinkId)); + PAssert.that(result.getDeadLetterRows()).empty(); + p.run().waitUntilFinish(); + + Table a = catalog.loadTable(TableIdentifier.of("db", tableA)); + Table b = catalog.loadTable(TableIdentifier.of("db", tableB)); + + // Each destination committed exactly its OWN windows, ascending: A's first is window 0, B's + // is window 1 (not in lockstep), and both saw window 2. + List tokensA = committedThroughTokens(a, sinkId); + List tokensB = committedThroughTokens(b, sinkId); + assertThat(tokensA, hasSize(2)); + assertThat(tokensB, hasSize(2)); + assertThat(tokensA.get(0), lessThan(tokensA.get(1))); + assertThat(tokensB.get(0), lessThan(tokensB.get(1))); + assertThat(tokensA.get(0), lessThan(tokensB.get(0))); + assertThat(tokensA.get(1), equalTo(tokensB.get(1))); + + // A: window 0's id=5 survives, window 2's DELETE removed window 0's id=1. + assertThat(destRows(a), contains("5:" + tableA)); + assertThat(destRows(b), contains("2:" + tableB, "3:" + tableB)); + } + + /** A routing input row for the {@code (id, dest, seq)} dynamic-destination schema. */ + private static Row destRow(Schema schema, int id, String dest, long seq) { + return Row.withSchema(schema).addValues(id, dest, seq).build(); + } + + /** Reads all live rows of a dynamic-destination table as sorted {@code id:dest} strings. */ + private static List destRows(Table table) { + table.refresh(); + return ImmutableList.copyOf(IcebergGenerics.read(table).build()).stream() + .map(r -> r.getField("id") + ":" + r.getField("dest")) + .sorted() + .collect(ImmutableList.toImmutableList()); + } + + // --------------------------------------------------------------------------------------------- + // 6. Upsert end-to-end + // --------------------------------------------------------------------------------------------- + + @Test + public void upsertModeAppliesAfterImageOnly() { + setUpCatalog(); + TableIdentifier id = v2Table(); + Table t = catalog.loadTable(id); + + // Upsert mode: only INSERT / UPDATE_AFTER, no before-images at all. + PCollection in = + boundedInput( + INPUT_SCHEMA, + KV.of(INSERT, row(1, "a", "x", 1L)), + KV.of(INSERT, row(2, "b", "y", 1L)), + KV.of(UPDATE_AFTER, row(1, "a", "z", 2L))); + in.apply( + IcebergIO.writeCdcRows(catalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withUpsert(true)); + p.run().waitUntilFinish(); + + // id=1's UPDATE_AFTER (seq 2) replaces its INSERT (seq 1); id=2 unaffected. + assertThat(readRows(t), contains("1:a:z", "2:b:y")); + } + + /** + * Upsert mode still applies DELETEs: widening the upsert drop predicate from "UPDATE_BEFORE" to + * "anything not an after-image" would swallow every DELETE, and nothing else in the suite fails. + */ + @Test + public void upsertModeStillAppliesDeletes() { + setUpCatalog(); + TableIdentifier id = v2Table(); + Table t = catalog.loadTable(id); + + PCollection in = + boundedInput( + INPUT_SCHEMA, + KV.of(INSERT, row(1, "a", "x", 1L)), + KV.of(INSERT, row(2, "b", "y", 1L)), + KV.of(UPDATE_AFTER, row(1, "a", "z", 2L)), + KV.of(DELETE, row(2, "b", "y", 3L))); + in.apply( + IcebergIO.writeCdcRows(catalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withUpsert(true)); + p.run().waitUntilFinish(); + + // id=2 is genuinely gone; id=1 carries its after-image. + assertThat(readRows(t), contains("1:a:z")); + } + + // --------------------------------------------------------------------------------------------- + // 8. Dead letters: exposure, first-late diversion, replayability + // --------------------------------------------------------------------------------------------- + + /** + * A FIRST-late pane (the shard-window's only record arrives after the watermark passed) is + * dead-lettered, not committed: pane timing is per shard-window, the committer's skip is per + * destination-window, so the pane cannot prove its window uncommitted. + */ + @Test + public void firstLatePaneIsDeadLetteredNotCommitted() { + setUpCatalog(); + TableIdentifier id = v2Table(); + Table t = catalog.loadTable(id); + + Duration window = Duration.standardSeconds(60); + Instant base = new Instant(0); + + TestStream> stream = + TestStream.create( + KvCoder.of(SerializableCoder.of(ValueKind.class), RowCoder.of(INPUT_SCHEMA))) + .advanceWatermarkTo(base) + // Advance PAST window [0, 60s) with no data, then deliver its first (late) element. + .advanceWatermarkTo(base.plus(Duration.standardSeconds(120))) + .addElements( + TimestampedValue.of( + KV.of(INSERT, row(1, "a", "x", 1L)), base.plus(Duration.standardSeconds(1)))) + .advanceWatermarkToInfinity(); + + PCollection in = CdcSinkTestUtils.withKinds(p.apply(stream)).setRowSchema(INPUT_SCHEMA); + IcebergWriteResult result = + in.apply( + IcebergIO.writeCdcRows(catalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withTriggeringFrequency(window) + .withAllowedLateness(Duration.standardDays(7))); + + // Captured by the PAssert lambda: TableIdentifier itself is not Serializable. + String destString = id.toString(); + PAssert.that(result.getDeadLetterRows()) + .satisfies( + rows -> { + Row dl = Iterables.getOnlyElement(rows); + assertThat(dl.getRow("record"), equalTo(rowNoSeq(1, "a", "x"))); + assertThat(dl.getString("change_type"), equalTo("INSERT")); + assertThat(dl.getInt64("sequence_number"), equalTo(1L)); + assertThat(dl.getString("destination"), equalTo(destString)); + return null; + }); + p.run().waitUntilFinish(); + + // The first-late record was dead-lettered (replayable), not committed. + assertThat(readRows(t), hasSize(0)); + } + + /** + * A late record whose key hashes to a previously-untouched shard of a window that already + * committed opens a fresh key+window whose first pane is late: the case a {@code !pane.isFirst()} + * test lets through, after which the committer discards it and the row reaches no output. It must + * be dead-lettered. + */ + @Test + public void firstLatePaneOnAnUntouchedShardIsDeadLettered() { + setUpCatalog(); + TableIdentifier id = v2Table(); + Table t = catalog.loadTable(id); + + int onTimeId = 1; + int lateId = idOnADifferentShardThan(onTimeId, CdcWriteConfig.DEFAULT_NUM_SHARDS); + + Duration window = Duration.standardSeconds(60); + Instant base = new Instant(0); + + TestStream> stream = + TestStream.create( + KvCoder.of(SerializableCoder.of(ValueKind.class), RowCoder.of(INPUT_SCHEMA))) + .advanceWatermarkTo(base) + // On-time data commits window [0, 60s); then a late record for the SAME window on a + // shard that saw nothing. + .addElements( + TimestampedValue.of( + KV.of(INSERT, row(onTimeId, "a", "x", 1L)), + base.plus(Duration.standardSeconds(1)))) + .advanceWatermarkTo(base.plus(Duration.standardSeconds(70))) + .addElements( + TimestampedValue.of( + KV.of(INSERT, row(lateId, "late", "z", 2L)), + base.plus(Duration.standardSeconds(2)))) + .advanceWatermarkToInfinity(); + + PCollection in = CdcSinkTestUtils.withKinds(p.apply(stream)).setRowSchema(INPUT_SCHEMA); + IcebergWriteResult result = + in.apply( + IcebergIO.writeCdcRows(catalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withTriggeringFrequency(window) + .withAllowedLateness(Duration.standardDays(7))); + + // Captured by the PAssert lambda: TableIdentifier itself is not Serializable. + String destString = id.toString(); + PAssert.that(result.getDeadLetterRows()) + .satisfies( + rows -> { + Row dl = Iterables.getOnlyElement(rows); + assertThat(dl.getRow("record"), equalTo(rowNoSeq(lateId, "late", "z"))); + assertThat(dl.getString("change_type"), equalTo("INSERT")); + assertThat(dl.getInt64("sequence_number"), equalTo(2L)); + assertThat(dl.getString("destination"), equalTo(destString)); + return null; + }); + p.run().waitUntilFinish(); + + // NOT silently lost: it is absent from the table (above) and present in the dead letters. + assertThat(readRows(t), contains(onTimeId + ":a:x")); + } + + /** + * The lowest id above {@code id} mapping to a different shard: computed, not hard-coded, so the + * test survives hash or pk-encoding changes. + */ + private static int idOnADifferentShardThan(int id, int numShards) { + int target = shardForId(id, numShards); + for (int candidate = id + 1; candidate < id + 10_000; candidate++) { + if (shardForId(candidate, numShards) != target) { + return candidate; + } + } + throw new AssertionError("no id maps to a shard other than " + target); + } + + /** The shard {@code AssignCdcKeys} assigns to a row with primary key {@code id}. */ + private static int shardForId(int id, int numShards) { + Schema pkSchema = Schema.builder().addInt32Field("id").build(); + try { + byte[] pkBytes = + CoderUtils.encodeToByteArray( + RowCoder.of(pkSchema), Row.withSchema(pkSchema).addValue(id).build()); + return TableSetup.shardFor(pkBytes, numShards); + } catch (CoderException e) { + throw new AssertionError(e); + } + } + + /** + * Pins that {@link WriteCdcRows#withAllowedLateness} is honored: a record late but within the + * bound is dead-lettered, one past the bound is dropped ENTIRELY by the runner before {@code + * SplitLateData} sees it: no dead letter, no counter, no row. That documented asymmetry is + * exactly what a wrong or ignored lateness value would change. + */ + @Test + public void recordsBeyondAllowedLatenessAreDroppedNotDeadLettered() { + setUpCatalog(); + TableIdentifier id = v2Table(); + Table t = catalog.loadTable(id); + + Duration window = Duration.standardSeconds(60); + Duration allowedLateness = Duration.standardMinutes(10); + Instant base = new Instant(0); + + TestStream> stream = + TestStream.create( + KvCoder.of(SerializableCoder.of(ValueKind.class), RowCoder.of(INPUT_SCHEMA))) + .advanceWatermarkTo(base) + // Window [0, 60s) closes empty; ~5 min late is inside the 10-min bound. + .advanceWatermarkTo(base.plus(Duration.standardMinutes(5))) + .addElements( + TimestampedValue.of( + KV.of(INSERT, row(1, "within", "x", 1L)), + base.plus(Duration.standardSeconds(1)))) + // Past window end + the whole allowed lateness: dropped outright. + .advanceWatermarkTo(base.plus(Duration.standardMinutes(30))) + .addElements( + TimestampedValue.of( + KV.of(INSERT, row(2, "beyond", "y", 2L)), + base.plus(Duration.standardSeconds(2)))) + .advanceWatermarkToInfinity(); + + PCollection in = CdcSinkTestUtils.withKinds(p.apply(stream)).setRowSchema(INPUT_SCHEMA); + IcebergWriteResult result = + in.apply( + IcebergIO.writeCdcRows(catalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withTriggeringFrequency(window) + .withAllowedLateness(allowedLateness)); + + PAssert.that(result.getDeadLetterRows()) + .satisfies( + rows -> { + // Exactly the within-bound record; the beyond-bound one never reaches the sink. + Row dl = Iterables.getOnlyElement(rows); + assertThat(dl.getRow("record"), equalTo(rowNoSeq(1, "within", "x"))); + return null; + }); + p.run().waitUntilFinish(); + + assertThat(readRows(t), hasSize(0)); + } + + /** + * A NON-first late pane (its shard-window already fired on time) is diverted to the replayable + * dead-letter output with the three metadata columns, and is NOT committed. + */ + @Test + public void deadLetterRowsAreExposed() { + setUpCatalog(); + TableIdentifier id = v2Table(); + Table t = catalog.loadTable(id); + + Duration window = Duration.standardSeconds(60); + Instant base = new Instant(0); + + TestStream> stream = + TestStream.create( + KvCoder.of(SerializableCoder.of(ValueKind.class), RowCoder.of(INPUT_SCHEMA))) + .advanceWatermarkTo(base) + .addElements( + TimestampedValue.of( + KV.of(INSERT, row(1, "a", "x", 1L)), base.plus(Duration.standardSeconds(1)))) + .advanceWatermarkTo(base.plus(Duration.standardSeconds(70))) // close window [0, 60s) + // The same key again, timestamped INSIDE the closed window: a NON-first late pane. + .addElements( + TimestampedValue.of( + KV.of(UPDATE_AFTER, row(1, "late", "z", 2L)), + base.plus(Duration.standardSeconds(2)))) + .advanceWatermarkToInfinity(); + + PCollection in = CdcSinkTestUtils.withKinds(p.apply(stream)).setRowSchema(INPUT_SCHEMA); + IcebergWriteResult result = + in.apply( + IcebergIO.writeCdcRows(catalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withTriggeringFrequency(window) + .withAllowedLateness(Duration.standardDays(7))); + + // Captured by the PAssert lambda: TableIdentifier itself is not Serializable. + String destString = id.toString(); + PAssert.that(result.getDeadLetterRows()) + .satisfies( + rows -> { + Row dl = Iterables.getOnlyElement(rows); + assertThat(dl.getRow("record"), equalTo(rowNoSeq(1, "late", "z"))); + assertThat(dl.getString("change_type"), equalTo("UPDATE_AFTER")); + assertThat(dl.getInt64("sequence_number"), equalTo(2L)); + assertThat(dl.getString("destination"), equalTo(destString)); + return null; + }); + p.run().waitUntilFinish(); + + // Invariant: dead-lettered => NOT committed. The table has only the on-time INSERT (1:a:x). + assertThat(readRows(t), contains("1:a:x")); + } + + /** + * The dead letters are REPLAYABLE through the real API: unnest {@code record}, map {@code + * change_type}/{@code sequence_number} as the replay sink's control columns, drop {@code + * destination}, and the row lands. The late record uses a different key than the on-time one so + * the final state is independent of commit interleaving. + */ + @Test + public void deadLettersAreReplayable() { + setUpCatalog(); + TableIdentifier id = v2Table(); + Table t = catalog.loadTable(id); + + Duration window = Duration.standardSeconds(60); + Instant base = new Instant(0); + + TestStream> stream = + TestStream.create( + KvCoder.of(SerializableCoder.of(ValueKind.class), RowCoder.of(INPUT_SCHEMA))) + .advanceWatermarkTo(base) + .addElements( + TimestampedValue.of( + KV.of(INSERT, row(1, "a", "x", 1L)), base.plus(Duration.standardSeconds(1)))) + .advanceWatermarkTo(base.plus(Duration.standardSeconds(70))) // window [0,60s) fires + .addElements( + TimestampedValue.of( + KV.of(UPDATE_AFTER, row(7, "late", "z", 2L)), + base.plus(Duration.standardSeconds(2)))) + .advanceWatermarkToInfinity(); + + PCollection in = CdcSinkTestUtils.withKinds(p.apply(stream)).setRowSchema(INPUT_SCHEMA); + IcebergWriteResult result = + in.apply( + "OriginalSink", + IcebergIO.writeCdcRows(catalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withTriggeringFrequency(window) + .withAllowedLateness(Duration.standardDays(7))); + + PCollection deadLetters = result.getDeadLetterRows(); + assertNotNull(deadLetters); + // Captured by the PAssert lambda: TableIdentifier itself is not Serializable. + String destString = id.toString(); + PAssert.that(deadLetters) + .satisfies( + rows -> { + Row dl = Iterables.getOnlyElement(rows); + assertThat(dl.getRow("record"), equalTo(rowNoSeq(7, "late", "z"))); + assertThat(dl.getString("change_type"), equalTo("UPDATE_AFTER")); + assertThat(dl.getInt64("sequence_number"), equalTo(2L)); + assertThat(dl.getString("destination"), equalTo(destString)); + return null; + }); + + // Re-feed the ACTUAL dead letters. + Schema replaySchema = + Schema.builder() + .addFields(DATA_SCHEMA.getFields()) + .addStringField("change_type") + .addInt64Field("sequence_number") + .build(); + PCollection replayInput = + deadLetters + .apply( + "UnnestRecord", + MapElements.into(TypeDescriptors.rows()) + .via( + dl -> + Row.withSchema(replaySchema) + .addValues(checkStateNotNull(dl.getRow("record")).getValues()) + .addValue(dl.getString("change_type")) + .addValue(dl.getInt64("sequence_number")) + .build())) + .setRowSchema(replaySchema) + // A dead letter's own event time is behind the watermark by definition; re-fed as-is + // it would just be late again, so a replay re-stamps it as fresh input. + .apply("RestampForReplay", ParDo.of(new RestampTo(REPLAY_EVENT_TIME))) + .setRowSchema(replaySchema); + replayInput.apply( + "ReplaySink", + IcebergIO.writeCdcRows(catalogConfig()) + .to(id) + .withChangeTypeColumn("change_type") + .withSequenceNumberColumn("sequence_number") + .withTriggeringFrequency(window) + .withAllowedLateness(Duration.standardDays(7))); + + p.run().waitUntilFinish(); + + assertThat(readRows(t), containsInAnyOrder("1:a:x", "7:late:z")); + } + + // --------------------------------------------------------------------------------------------- + // 9. Error handling (failed rows) + // --------------------------------------------------------------------------------------------- + + /** + * With {@code withErrorHandling()}, a poison record (an unknown change-type value) is diverted to + * {@link IcebergWriteResult#getFailedRows()}; the other records commit normally. + */ + @Test + public void errorHandlingDivertsPoisonRecords() { + setUpCatalog(); + TableIdentifier id = v2Table(); + Table t = catalog.loadTable(id); + + Schema inputWithOp = + Schema.builder() + .addFields(DATA_SCHEMA.getFields()) + .addInt64Field("seq") + .addStringField("op") + .build(); + Row r1 = Row.withSchema(inputWithOp).addValues(1, "a", "x", 1L, "INSERT").build(); + Row r2 = Row.withSchema(inputWithOp).addValues(2, "b", "y", 1L, "BOGUS").build(); // poison + Row r3 = Row.withSchema(inputWithOp).addValues(3, "c", "z", 1L, "INSERT").build(); + + PCollection in = + boundedInput(inputWithOp, KV.of(INSERT, r1), KV.of(INSERT, r2), KV.of(INSERT, r3)); + IcebergWriteResult result = + in.apply( + IcebergIO.writeCdcRows(catalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withChangeTypeColumn("op") + .withErrorHandling()); + + PCollection failedRows = result.getFailedRows(); + assertNotNull(failedRows); + PAssert.that(failedRows) + .satisfies( + rows -> { + Row err = Iterables.getOnlyElement(rows); + Row failedRow = err.getRow("failed_row"); + assertThat(failedRow.getInt32("id"), equalTo(2)); + assertThat(failedRow.getString("op"), equalTo("BOGUS")); + // Operators must see WHICH change-type value was bad, not just that something failed. + assertThat(err.getString("error_message"), containsString("BOGUS")); + return null; + }); + p.run().waitUntilFinish(); + + assertThat(readRows(t), contains("1:a:x", "3:c:z")); + } + + /** Without {@code withErrorHandling()}, the failed-rows accessor is {@code null}. */ + @Test + public void failedRowsNullWithoutErrorHandling() { + p.enableAbandonedNodeEnforcement(false); + TableIdentifier id = TableIdentifier.of("db", "nofail_" + System.nanoTime()); + PCollection in = boundedInput(INPUT_SCHEMA, KV.of(INSERT, row(1, "a", "x", 1L))); + IcebergWriteResult result = + in.apply( + IcebergIO.writeCdcRows(bogusCatalogConfig()).to(id).withSequenceNumberColumn("seq")); + assertNull(result.getFailedRows()); + assertNotNull(result.getDeadLetterRows()); + assertNotNull(result.getSnapshots()); + } + + // --------------------------------------------------------------------------------------------- + // 10. Dead-letter column names pinned + // --------------------------------------------------------------------------------------------- + + /** The dead-letter metadata column names and types are a public contract: pinned literally. */ + @Test + public void deadLetterColumnNamesPinned() { + p.enableAbandonedNodeEnforcement(false); + TableIdentifier id = TableIdentifier.of("db", "dlpin_" + System.nanoTime()); + PCollection in = boundedInput(INPUT_SCHEMA, KV.of(INSERT, row(1, "a", "x", 1L))); + IcebergWriteResult result = + in.apply( + IcebergIO.writeCdcRows(bogusCatalogConfig()).to(id).withSequenceNumberColumn("seq")); + + PCollection deadLetters = result.getDeadLetterRows(); + assertNotNull(deadLetters); + Schema dl = deadLetters.getSchema(); + assertThat( + dl.getFieldNames(), contains("record", "change_type", "sequence_number", "destination")); + assertThat(dl.getField("record").getType(), equalTo(Schema.FieldType.row(DATA_SCHEMA))); + assertThat(dl.getField("change_type").getType(), equalTo(Schema.FieldType.STRING)); + assertThat(dl.getField("sequence_number").getType(), equalTo(Schema.FieldType.INT64)); + assertThat(dl.getField("destination").getType(), equalTo(Schema.FieldType.STRING)); + } + + // --------------------------------------------------------------------------------------------- + // 12. Two-pipeline batch rerun with stable sink id + // --------------------------------------------------------------------------------------------- + + /** + * A true two-pipeline batch rerun (not a hand-seeded token): the second pipeline, same table and + * {@code sink_id} but a different row, must write nothing: the batch token is recovered from the + * real prior commit and the rerun's window skipped loudly. + */ + @Test + public void secondBatchRunWithStableSinkIdWritesNothing() { + setUpCatalog(); + TableIdentifier id = v2Table(); + Table t = catalog.loadTable(id); + String sinkId = "stable-rerun"; + + // P1: first batch run, one INSERT under the stable sink id. + boundedInput(INPUT_SCHEMA, KV.of(INSERT, row(1, "a", "x", 1L))) + .apply( + IcebergIO.writeCdcRows(catalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withSinkId(sinkId)); + p.run().waitUntilFinish(); + + t.refresh(); + assertThat(Lists.newArrayList(t.snapshots()), hasSize(1)); + assertThat(readRows(t), contains("1:a:x")); + + // P2: a fresh batch pipeline, SAME table + SAME sink id, DIFFERENT row. + Pipeline p2 = Pipeline.create(); + CdcSinkTestUtils.withKinds(p2.apply(Create.of(KV.of(INSERT, row(2, "b", "y", 2L))))) + .setRowSchema(INPUT_SCHEMA) + .apply( + IcebergIO.writeCdcRows(catalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withSinkId(sinkId)); + PipelineResult r2 = p2.run(); + r2.waitUntilFinish(); + + t.refresh(); + // Still one snapshot, still only the first row, and the skip was loud. + assertThat(Lists.newArrayList(t.snapshots()), hasSize(1)); + assertThat(readRows(t), contains("1:a:x")); + assertThat(committerCounter(r2, "alreadyCommittedWindowsSkipped"), greaterThanOrEqualTo(1L)); + } + + // --------------------------------------------------------------------------------------------- + // 13. Token heartbeat gated to streaming + // --------------------------------------------------------------------------------------------- + + /** + * A heartbeat configured on BOUNDED input must be dropped with a warn at {@code expand()}: a + * self-rescheduling processing-time timer can keep a runner from quiescing. The run terminates + * promptly with exactly one snapshot. + */ + @Test(timeout = 120_000) + public void tokenHeartbeatIgnoredForBoundedInput() { + setUpCatalog(); + TableIdentifier id = v2Table(); + Table t = catalog.loadTable(id); + + PCollection in = boundedInput(INPUT_SCHEMA, KV.of(INSERT, row(1, "a", "x", 1L))); + in.apply( + IcebergIO.writeCdcRows(catalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withTokenHeartbeat(Duration.millis(1))); + + // expand() runs during apply(); on bounded input it must warn that the heartbeat is dropped. + expectedLogs.verifyWarn("heartbeat is ignored for bounded input"); + + p.run().waitUntilFinish(); + + t.refresh(); + assertThat(Lists.newArrayList(t.snapshots()), hasSize(1)); + } + + /** Re-stamps every element to one fixed event time (a replay assigns fresh event times). */ + private static final class RestampTo extends DoFn { + private final Instant timestamp; + + RestampTo(Instant timestamp) { + this.timestamp = timestamp; + } + + @ProcessElement + public void process(@Element Row row, OutputReceiver out) { + out.outputWithTimestamp(row, timestamp); + } + } +} From 603af4c7ceb2b5f268675f9920b058199559b947 Mon Sep 17 00:00:00 2001 From: Ahmed Abualsaud Date: Tue, 22 Sep 2026 11:28:45 -0700 Subject: [PATCH 2/5] addreress comments --- .../sdk/io/iceberg/IcebergWriteResult.java | 18 +++++++------ .../sdk/io/iceberg/cdc/sink/WriteCdcRows.java | 27 +++++++++++++++++++ 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergWriteResult.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergWriteResult.java index 8db954428b3f..d08a0acb5ab4 100644 --- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergWriteResult.java +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergWriteResult.java @@ -17,6 +17,8 @@ */ package org.apache.beam.sdk.io.iceberg; +import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull; + import java.util.Map; import org.apache.beam.sdk.Pipeline; import org.apache.beam.sdk.annotations.Internal; @@ -75,11 +77,11 @@ public PCollection> getSnapshots() { * change for those keys has committed; a stale replay's equality delete removes the newer row. * * @return the dead-letter {@code PCollection} for results produced by {@code - * IcebergIO.writeCdcRows}/{@link #cdc}, or {@code null} for results produced by the - * append-only sink ({@code IcebergIO.writeRows}), which has no dead-letter output. + * IcebergIO.writeCdcRows}/{@link #cdc}. Throws when attempting to retrieve it for the + * append-only sink ({@code IcebergIO.writeRows}) as has no dead-letter output. */ - public @Nullable PCollection getDeadLetterRows() { - return deadLetterRows; + public PCollection getDeadLetterRows() { + return checkStateNotNull(deadLetterRows); } /** @@ -88,11 +90,11 @@ public PCollection> getSnapshots() { * {@link org.apache.beam.sdk.schemas.transforms.providers.ErrorHandling}). Distinct from {@link * #getDeadLetterRows()} (which carries late-but-valid records). * - * @return the failed-rows {@code PCollection}, or {@code null} when error handling was not - * enabled (or for the append-only sink). + * @return the failed-rows {@code PCollection} if error handling was enabled for the CDC + * sink. Will throw otherwise. */ - public @Nullable PCollection getFailedRows() { - return failedRows; + public PCollection getFailedRows() { + return checkStateNotNull(failedRows); } IcebergWriteResult(Pipeline pipeline, PCollection> snapshots) { diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRows.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRows.java index 20a44dc1c7cb..cc707076a35d 100644 --- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRows.java +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRows.java @@ -31,11 +31,13 @@ import org.apache.beam.sdk.io.iceberg.SnapshotInfo; import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.transforms.display.DisplayData; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionTuple; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.ValueKind; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Predicates; import org.apache.iceberg.catalog.TableIdentifier; @@ -380,6 +382,8 @@ private CdcWriteConfig cdcWriteConfig() { /** Construction-time validation. */ private void validate(PCollection input, CdcWriteConfig config) { + Preconditions.checkState( + input.hasSchema(), "CDC input must have a schema; set one with setRowSchema(...)."); Preconditions.checkArgument( 1 == Stream.of(getTableIdentifier(), getDynamicDestinations()) @@ -401,7 +405,15 @@ private void validate(PCollection input, CdcWriteConfig config) { triggeringFrequency.isLongerThan(Duration.ZERO), "triggering frequency must be positive for streaming CDC writes, got %s", triggeringFrequency); + } else if (triggeringFrequency != null) { + LOG.warn( + "Triggering frequency is ignored for bounded input. Batch loads commit once at the end."); } + @Nullable Duration allowedLateness = getAllowedLateness(); + Preconditions.checkArgument( + allowedLateness == null || !allowedLateness.isShorterThan(Duration.ZERO), + "allowed lateness must not be negative, got %s", + allowedLateness); @Nullable String changeTypeColumn = config.getChangeTypeColumn(); if (changeTypeColumn != null) { @@ -461,6 +473,21 @@ private void validate(PCollection input, CdcWriteConfig config) { } } + @Override + public void populateDisplayData(DisplayData.Builder builder) { + super.populateDisplayData(builder); + builder.add(DisplayData.item("sinkId", getSinkId())); + builder.add(DisplayData.item("upsert", getUpsert())); + builder.add(DisplayData.item("numShards", getNumShards())); + builder.addIfNotNull(DisplayData.item("shardsPerPartition", getShardsPerPartition())); + builder.add(DisplayData.item("sorterMemoryMB", getSorterMemoryMB())); + builder.addIfNotNull(DisplayData.item("triggeringFrequency", getTriggeringFrequency())); + builder.add( + DisplayData.item( + "allowedLateness", + MoreObjects.firstNonNull(getAllowedLateness(), DEFAULT_ALLOWED_LATENESS))); + } + @Override public IcebergWriteResult expand(PCollection input) { CdcWriteConfig config = cdcWriteConfig(); From 78d15de97b8130873727e55d38bcfaf199199406 Mon Sep 17 00:00:00 2001 From: Ahmed Abualsaud Date: Tue, 22 Sep 2026 11:31:57 -0700 Subject: [PATCH 3/5] address comments --- .../apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRows.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRows.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRows.java index cc707076a35d..33a11d554f96 100644 --- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRows.java +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRows.java @@ -20,6 +20,8 @@ import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull; import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import java.util.List; import java.util.Map; import java.util.UUID; @@ -214,7 +216,7 @@ public WriteCdcRows to(DynamicDestinations destinations) { * {@link #withShardsPerPartition} cap. */ public WriteCdcRows withEqualityColumns(List columns) { - return toBuilder().setEqualityColumns(columns).build(); + return toBuilder().setEqualityColumns(ImmutableList.copyOf(columns)).build(); } /** @@ -242,7 +244,7 @@ public WriteCdcRows withChangeTypeColumn(String column) { * #withChangeTypeColumn} to also be set. */ public WriteCdcRows withChangeTypeMap(Map changeTypeMap) { - return toBuilder().setChangeTypeMap(changeTypeMap).build(); + return toBuilder().setChangeTypeMap(ImmutableMap.copyOf(changeTypeMap)).build(); } /** @@ -328,7 +330,7 @@ public WriteCdcRows withErrorHandling() { * rejected at construction. */ public WriteCdcRows withSnapshotProperties(Map snapshotProperties) { - return toBuilder().setSnapshotProperties(snapshotProperties).build(); + return toBuilder().setSnapshotProperties(ImmutableMap.copyOf(snapshotProperties)).build(); } /** From c5aea7e626262fb491af87d20f5aa73cd6ae3d89 Mon Sep 17 00:00:00 2001 From: Ahmed Abualsaud Date: Tue, 22 Sep 2026 11:49:53 -0700 Subject: [PATCH 4/5] spotless --- .../org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRows.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRows.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRows.java index 33a11d554f96..9d240c0e7d76 100644 --- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRows.java +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRows.java @@ -20,8 +20,6 @@ import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull; import com.google.auto.value.AutoValue; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; import java.util.List; import java.util.Map; import java.util.UUID; @@ -42,6 +40,8 @@ import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Predicates; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.apache.iceberg.catalog.TableIdentifier; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; From cb3e637f883cff4f46c43bd9050e82b7fd914328 Mon Sep 17 00:00:00 2001 From: Ahmed Abualsaud Date: Tue, 22 Sep 2026 11:52:25 -0700 Subject: [PATCH 5/5] update test --- .../apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRowsTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRowsTest.java b/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRowsTest.java index 1e3a2a03d121..e093440225c9 100644 --- a/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRowsTest.java +++ b/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRowsTest.java @@ -1374,7 +1374,7 @@ public void failedRowsNullWithoutErrorHandling() { IcebergWriteResult result = in.apply( IcebergIO.writeCdcRows(bogusCatalogConfig()).to(id).withSequenceNumberColumn("seq")); - assertNull(result.getFailedRows()); + assertThrows(IllegalStateException.class, result::getFailedRows); assertNotNull(result.getDeadLetterRows()); assertNotNull(result.getSnapshots()); }