From 1dc2ac1c795a97dbe7b58465ef14b83596c78493 Mon Sep 17 00:00:00 2001 From: Ahmed Abualsaud Date: Thu, 17 Sep 2026 08:57:47 -0700 Subject: [PATCH 1/9] top api layer --- .../apache/beam/sdk/io/iceberg/IcebergIO.java | 18 + .../sdk/io/iceberg/IcebergWriteResult.java | 85 + .../sdk/io/iceberg/cdc/sink/WriteCdcRows.java | 521 ++++++ .../sdk/io/iceberg/cdc/sink/package-info.java | 233 ++- .../cdc/sink/WriteCdcRowsReadBackTest.java | 1526 +++++++++++++++++ .../io/iceberg/cdc/sink/WriteCdcRowsTest.java | 1499 ++++++++++++++++ 6 files changed, 3880 insertions(+), 2 deletions(-) create mode 100644 sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRows.java create mode 100644 sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRowsReadBackTest.java create mode 100644 sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRowsTest.java diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergIO.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergIO.java index 080c31ff9126..d2119a241e3e 100644 --- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergIO.java +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergIO.java @@ -26,6 +26,7 @@ import org.apache.beam.sdk.annotations.Internal; import org.apache.beam.sdk.io.Read; import org.apache.beam.sdk.io.iceberg.cdc.IncrementalChangelogSource; +import org.apache.beam.sdk.io.iceberg.cdc.sink.WriteCdcRows; import org.apache.beam.sdk.options.StreamingOptions; import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.transforms.PTransform; @@ -393,6 +394,23 @@ public static WriteRows writeRows(IcebergCatalogConfig catalog) { .build(); } + /** + * Returns a {@link WriteCdcRows} transform: the CDC sink, applying inserts/updates/deletes from a + * {@code PCollection} of change records (each carrying a {@link + * org.apache.beam.sdk.values.ValueKind}) to one or more Iceberg V2+ tables via equality deletes; + * superseded rows are never written. + * + * 
{@code
+   * input.apply(IcebergIO.writeCdcRows(catalogConfig)
+   *     .to(tableId)
+   *     .withSequenceNumberColumn("seq")
+   *     .withTriggeringFrequency(Duration.standardMinutes(1)));
+   * }
+ */ + public static WriteCdcRows writeCdcRows(IcebergCatalogConfig catalog) { + return WriteCdcRows.of(catalog); + } + @AutoValue public abstract static class WriteRows extends PTransform, IcebergWriteResult> { diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergWriteResult.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergWriteResult.java index 8e2549b5dadb..8db954428b3f 100644 --- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergWriteResult.java +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergWriteResult.java @@ -19,31 +19,110 @@ import java.util.Map; import org.apache.beam.sdk.Pipeline; +import org.apache.beam.sdk.annotations.Internal; import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PInput; import org.apache.beam.sdk.values.POutput; import org.apache.beam.sdk.values.PValue; +import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.checkerframework.checker.nullness.qual.Nullable; +/** + * The output of an {@code IcebergIO} write: the snapshots each destination table committed, plus + * the two diversion outputs the CDC sink can produce. + * + *

Only {@link #getSnapshots()} is always present. {@link #getDeadLetterRows()} (late-but-valid + * records) and {@link #getFailedRows()} (per-record poison rows) are non-null only for results + * built by {@link #cdc}, that is, by {@code IcebergIO.writeCdcRows}, and {@code getFailedRows()} + * additionally only when error handling was enabled. The append-only sink ({@code + * IcebergIO.writeRows}) leaves both null rather than exposing outputs that can never carry data. + */ public final class IcebergWriteResult implements POutput { private static final TupleTag> SNAPSHOTS_TAG = new TupleTag>() {}; + private static final TupleTag DEAD_LETTER_TAG = new TupleTag() {}; + + private static final TupleTag FAILED_ROWS_TAG = new TupleTag() {}; + private final Pipeline pipeline; private final PCollection> snapshots; + private final @Nullable PCollection deadLetterRows; + + private final @Nullable PCollection failedRows; + + /** + * The committed snapshots, keyed by destination. A window committed in a commit fire that later + * fails may not re-emit its {@link SnapshotInfo} on the retry (table state is unaffected). + */ public PCollection> getSnapshots() { return snapshots; } + /** + * The replayable dead-letter {@link Row}s from the CDC sink: records whose grouped pane fired + * late, i.e. after the watermark had passed their commit window's end. Schema is {@code record + * ROW + change_type STRING + sequence_number INT64 + destination STRING}. To replay, + * unnest {@code record}, map {@code change_type}/{@code sequence_number} as the sink's control + * columns, and route each row by {@code destination}. Replaying is only safe while no newer + * change for those keys has committed; a stale replay's equality delete removes the newer row. + * + * @return the dead-letter {@code PCollection} for results produced by {@code + * IcebergIO.writeCdcRows}/{@link #cdc}, or {@code null} for results produced by the + * append-only sink ({@code IcebergIO.writeRows}), which has no dead-letter output. + */ + public @Nullable PCollection getDeadLetterRows() { + return deadLetterRows; + } + + /** + * The per-record poison rows diverted by the CDC sink when error handling is enabled (see {@code + * WriteCdcRows.withErrorHandling}); schema is {@code failed_row ROW + error_message STRING} (see + * {@link org.apache.beam.sdk.schemas.transforms.providers.ErrorHandling}). Distinct from {@link + * #getDeadLetterRows()} (which carries late-but-valid records). + * + * @return the failed-rows {@code PCollection}, or {@code null} when error handling was not + * enabled (or for the append-only sink). + */ + public @Nullable PCollection getFailedRows() { + return failedRows; + } + IcebergWriteResult(Pipeline pipeline, PCollection> snapshots) { + this(pipeline, snapshots, null, null); + } + + private IcebergWriteResult( + Pipeline pipeline, + PCollection> snapshots, + @Nullable PCollection deadLetterRows, + @Nullable PCollection failedRows) { this.pipeline = pipeline; this.snapshots = snapshots; + this.deadLetterRows = deadLetterRows; + this.failedRows = failedRows; + } + + /** + * Returns an {@link IcebergWriteResult} for the CDC sink, exposing the committed-snapshot {@code + * snapshots}, the replayable {@code deadLetterRows} (see {@link #getDeadLetterRows()}), and the + * optional per-record {@code failedRows} (see {@link #getFailedRows()}, {@code null} when error + * handling is off). + */ + @Internal + public static IcebergWriteResult cdc( + Pipeline pipeline, + PCollection> snapshots, + PCollection deadLetterRows, + @Nullable PCollection failedRows) { + return new IcebergWriteResult(pipeline, snapshots, deadLetterRows, failedRows); } @Override @@ -55,6 +134,12 @@ public Pipeline getPipeline() { public Map, PValue> expand() { ImmutableMap.Builder, PValue> output = ImmutableMap.builder(); output.put(SNAPSHOTS_TAG, snapshots); + if (deadLetterRows != null) { + output.put(DEAD_LETTER_TAG, deadLetterRows); + } + if (failedRows != null) { + output.put(FAILED_ROWS_TAG, failedRows); + } return output.build(); } diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRows.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRows.java new file mode 100644 index 000000000000..20a44dc1c7cb --- /dev/null +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRows.java @@ -0,0 +1,521 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.iceberg.cdc.sink; + +import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull; + +import com.google.auto.value.AutoValue; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.stream.Stream; +import org.apache.beam.sdk.annotations.Internal; +import org.apache.beam.sdk.io.iceberg.DynamicDestinations; +import org.apache.beam.sdk.io.iceberg.IcebergCatalogConfig; +import org.apache.beam.sdk.io.iceberg.IcebergWriteResult; +import org.apache.beam.sdk.io.iceberg.SnapshotInfo; +import org.apache.beam.sdk.schemas.Schema; +import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.values.KV; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.PCollectionTuple; +import org.apache.beam.sdk.values.Row; +import org.apache.beam.sdk.values.ValueKind; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Predicates; +import org.apache.iceberg.catalog.TableIdentifier; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.joda.time.Duration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The top-level CDC sink transform: applies a collection of change records to one or more Iceberg + * V2+ tables: + * + * 

{@code
+ * changeRecords.apply(IcebergIO.writeCdcRows(catalogConfig)
+ *     .to(tableId)
+ *     .withSequenceNumberColumn("seq")
+ *     .withTriggeringFrequency(Duration.standardMinutes(1)));
+ * }
+ * + *

Input contract

+ * + *

Each input {@link Row} is one change record. Its change kind is its native {@link ValueKind}, + * but can be overridden with a string column value using {@link #withChangeTypeColumn}, optionally + * translated via {@link #withChangeTypeMap}. Each row must also carry a per-key monotonic + * (long) sequence number specified by {@link #withSequenceNumberColumn}, which orders a single + * primary key's changes; the column is required in the input schema as a non-nullable {@code + * INT64}; set defaults upstream. These control columns are stripped from the input rows before + * writing to the table. + * + *

Ordering requirement (not validated): for a given primary key, the input element's + * event-time must be non-decreasing with its (sequence number, kind rank): a higher-sequence change + * never carries an earlier event time, and equal-sequence records (an update's before and after + * images) carry equal event times, so neither half lands in an earlier commit window. A violation + * can corrupt final table state (a lower-sequence equality delete deleting a higher-sequence row + * committed in a later snapshot). + * + *

Semantics

+ * + *

The sink commits one new snapshot to the destination table per commit window, in ascending + * window order, with an idempotency token written to each snapshot's summary. A retried or + * restarted commit finds the token and skips already-committed windows, making the commit + * effectively-once on runners that honor {@code @RequiresStableInput}. Records whose grouped pane + * fires late (watermark is past their commit window's end) are diverted to the DLQ output, + * accessible with {@link IcebergWriteResult#getDeadLetterRows()}. + * + *

Each dead-lettered record nests the data row under {@code record}, beside {@code change_type}, + * {@code sequence_number}, and {@code destination}. + * + *

Sink id

+ * + *

The sink creates a fresh unique id by default. All commits in a single pipeline run share the + * same id. Commits get stamped with the sink id and a window-end millis token. Set {@link + * #withSinkId} explicitly (and keep it stable) for cross-relaunch idempotency. Don't reuse sink ids + * for batch runs though because all commits fall under a single global window, so a second load + * with the same sink id will recognize the same window-end millis token and skip the commit. A + * batch load's sink id must likewise not be carried into a streaming continuation. + */ +@Internal +@AutoValue +public abstract class WriteCdcRows extends PTransform, IcebergWriteResult> { + + private static final Logger LOG = LoggerFactory.getLogger(WriteCdcRows.class); + + public static final Duration DEFAULT_ALLOWED_LATENESS = Duration.standardHours(6); + + abstract IcebergCatalogConfig getCatalogConfig(); + + abstract @Nullable TableIdentifier getTableIdentifier(); + + abstract @Nullable DynamicDestinations getDynamicDestinations(); + + abstract @Nullable List getEqualityColumns(); + + abstract String getSequenceNumberColumn(); + + abstract @Nullable String getChangeTypeColumn(); + + abstract @Nullable Map getChangeTypeMap(); + + abstract int getNumShards(); + + abstract @Nullable Integer getShardsPerPartition(); + + abstract int getSorterMemoryMB(); + + abstract boolean getUpsert(); + + abstract @Nullable Long getTokenHeartbeatMillis(); + + abstract boolean getErrorHandlingEnabled(); + + abstract @Nullable Map getSnapshotProperties(); + + /** The sink id namespacing the {@code beam.cdc.} snapshot-summary tokens. */ + abstract String getSinkId(); + + abstract @Nullable Duration getTriggeringFrequency(); + + abstract @Nullable Duration getAllowedLateness(); + + abstract Builder toBuilder(); + + @AutoValue.Builder + abstract static class Builder { + abstract Builder setCatalogConfig(IcebergCatalogConfig catalogConfig); + + abstract Builder setTableIdentifier(TableIdentifier tableIdentifier); + + abstract Builder setDynamicDestinations(DynamicDestinations destinations); + + abstract Builder setEqualityColumns(List equalityColumns); + + abstract Builder setSequenceNumberColumn(String sequenceNumberColumn); + + abstract Builder setChangeTypeColumn(String changeTypeColumn); + + abstract Builder setChangeTypeMap(Map changeTypeMap); + + abstract Builder setNumShards(int numShards); + + abstract Builder setShardsPerPartition(Integer shardsPerPartition); + + abstract Builder setSorterMemoryMB(int sorterMemoryMB); + + abstract Builder setUpsert(boolean upsert); + + abstract Builder setTokenHeartbeatMillis(@Nullable Long tokenHeartbeatMillis); + + abstract Builder setErrorHandlingEnabled(boolean errorHandlingEnabled); + + abstract Builder setSnapshotProperties(Map snapshotProperties); + + abstract Builder setSinkId(String sinkId); + + abstract Builder setTriggeringFrequency(Duration triggeringFrequency); + + abstract Builder setAllowedLateness(Duration allowedLateness); + + abstract WriteCdcRows build(); + } + + public static WriteCdcRows of(IcebergCatalogConfig catalogConfig) { + return new AutoValue_WriteCdcRows.Builder() + .setCatalogConfig(catalogConfig) + .setSequenceNumberColumn(CdcWriteConfig.DEFAULT_SEQUENCE_NUMBER_COLUMN) + .setNumShards(CdcWriteConfig.DEFAULT_NUM_SHARDS) + .setSorterMemoryMB(CdcWriteConfig.DEFAULT_SORTER_MEMORY_MB) + .setUpsert(false) + .setErrorHandlingEnabled(false) + .setSinkId(UUID.randomUUID().toString()) + .build(); + } + + /** Writes to a single table. Mutually exclusive with {@link #to(DynamicDestinations)}. */ + public WriteCdcRows to(TableIdentifier tableIdentifier) { + return toBuilder().setTableIdentifier(tableIdentifier).build(); + } + + /** + * Writes to multiple tables. Mutually exclusive with {@link #to(TableIdentifier)}. + * + *

The sink reads the control columns from the raw element and writes {@link + * DynamicDestinations#getData}, whose schema must match the destination table's and must exclude + * control columns. + */ + public WriteCdcRows to(DynamicDestinations destinations) { + return toBuilder().setDynamicDestinations(destinations).build(); + } + + /** + * Columns that define a row's identity (the Iceberg equality-delete fields). Defaults to the + * destination table's identifier (primary-key) fields. Tables may be partitioned on non-key + * columns; partition source columns must be equality columns only under {@link #withUpsert} or a + * {@link #withShardsPerPartition} cap. + */ + public WriteCdcRows withEqualityColumns(List columns) { + return toBuilder().setEqualityColumns(columns).build(); + } + + /** + * The column holding the per-primary-key monotonic sequence number used to order a single key's + * changes. Must be declared as a non-nullable {@code INT64} in the input schema. The column is + * stripped from the written rows. Defaults to {@value + * CdcWriteConfig#DEFAULT_SEQUENCE_NUMBER_COLUMN}. + */ + public WriteCdcRows withSequenceNumberColumn(String column) { + return toBuilder().setSequenceNumberColumn(column).build(); + } + + /** + * When set, reads the change kind from this column instead of the element's native {@link + * ValueKind}. Must be declared as a non-nullable {@code STRING} in the input schema. The column + * is stripped from the written rows. + */ + public WriteCdcRows withChangeTypeColumn(String column) { + return toBuilder().setChangeTypeColumn(column).build(); + } + + /** + * Mapping from {@link #withChangeTypeColumn} values to {@link ValueKind} names (e.g. for + * Debezium: {@code {"c": "INSERT", "u": "UPDATE_AFTER", "d": "DELETE"}}). Requires {@link + * #withChangeTypeColumn} to also be set. + */ + public WriteCdcRows withChangeTypeMap(Map changeTypeMap) { + return toBuilder().setChangeTypeMap(changeTypeMap).build(); + } + + /** + * The number of deterministic primary-key-hash shards per destination. Controls the sink's + * write-parallelism knob. Defaults to {@value CdcWriteConfig#DEFAULT_NUM_SHARDS}. + * + *

Too low may cause a write bottleneck with a growing commit backlog, too high increases the + * sink's output file count ({@code num_shards x touched partitions} files per commit window). + * + *

On a partitioned table, a commit window writes up to this many files per touched partition. + * {@link #withShardsPerPartition} lowers that per-partition count while leaving this number + * unchanged, so the file count drops without lowering the ceiling on total write parallelism. + * Each individual partition is then written by at most that many shards. + */ + public WriteCdcRows withNumShards(int numShards) { + return toBuilder().setNumShards(numShards).build(); + } + + /** + * Caps how many shards one partition may occupy (default: {@code num_shards}, i.e. all shards may + * write to all partitions). A lower cap reduces write parallelism per partition but also + * concentrates the writes in fewer files. A cap of {@code 1} pins each partition to a single + * writer. Ignored for unpartitioned tables. A cap below {@code num_shards} requires every + * partition source column to be an equality column, making the partition (and with it the shard) + * a pure function of the primary key. + */ + public WriteCdcRows withShardsPerPartition(int shardsPerPartition) { + return toBuilder().setShardsPerPartition(shardsPerPartition).build(); + } + + /** + * The in-memory buffer size (MB) for the sorter that orders each shard's records by primary key, + * then sequence number, then change kind, before writing; groups larger than this spill to disk. + * Must be {@code >= 1}. Defaults to {@value CdcWriteConfig#DEFAULT_SORTER_MEMORY_MB}. + */ + public WriteCdcRows withSorterMemoryMB(int sorterMemoryMB) { + Preconditions.checkArgument( + sorterMemoryMB >= 1, "sorter_memory_mb must be >= 1, got %s", sorterMemoryMB); + return toBuilder().setSorterMemoryMB(sorterMemoryMB).build(); + } + + /** + * If {@code true}, only the after-image of each change ({@code INSERT}/{@code UPDATE_AFTER}) is + * required; {@code UPDATE_BEFORE} records are dropped and {@code INSERT}/{@code UPDATE_AFTER} are + * applied as upserts (equality-delete-then-insert on the primary key). Defaults to {@code false}. + * Requires every partition source column to be an equality column: with before-images dropped, a + * row that moved partitions could never be deleted from its old one. + */ + public WriteCdcRows withUpsert(boolean upsert) { + return toBuilder().setUpsert(upsert).build(); + } + + /** + * Enables a periodic empty token-refresh (heartbeat) commit for each destination with a + * committed-through token, whether committed in this run or recovered from the table: while idle, + * the committer re-writes the token into a fresh snapshot every {@code interval}, so the + * token-bearing snapshot stays recent and is less likely to be lost to {@code expire_snapshots} + * before the sink resumes. Disabled by default, and ignored for bounded (batch) input. + * + *

With heartbeat enabled prefer cancel-and-resubmit over drain: the self-re-arming + * processing-time timer can keep a drain from completing. + */ + public WriteCdcRows withTokenHeartbeat(Duration interval) { + Preconditions.checkArgument( + interval != null && interval.getMillis() > 0, "token heartbeat interval must be positive"); + return toBuilder().setTokenHeartbeatMillis(interval.getMillis()).build(); + } + + /** + * Enables per-record error handling. Poison records (unknown change type, missing/null sequence + * number, null equality value, an unresolvable destination) are diverted to {@link + * IcebergWriteResult#getFailedRows()} (schema {@code failed_row ROW + error_message STRING}). + * Defaults to off where the DoFn throws an error instead. Diversion covers poison detectable in + * the assignment stage; a failure later in the pipeline fails and retries its bundle as usual. + */ + public WriteCdcRows withErrorHandling() { + return toBuilder().setErrorHandlingEnabled(true).build(); + } + + /** + * Extra user properties to add to every commit's Iceberg snapshot summary. Keys prefixed with + * {@code beam.cdc.} are reserved for the sink's own idempotency/diagnostic tokens and are + * rejected at construction. + */ + public WriteCdcRows withSnapshotProperties(Map snapshotProperties) { + return toBuilder().setSnapshotProperties(snapshotProperties).build(); + } + + /** + * A stable identifier for this sink, used to namespace the idempotency tokens written to each + * commit's Iceberg snapshot summary ({@code beam.cdc.sink-id}, {@code + * beam.cdc.committed-through-ms.}). Mostly useful when re-using the same {@code sinkId} + * across subsequent streaming relaunches to maintain exactly-once commit idempotency from the + * same source. If unset, a fresh UUID is used per launch. + * + *

Also useful in batch to make runs idempotent for the same {@code sinkId}. A stable {@code + * sinkId} should only be used once though. In batch, the idempotency token is always + * global-window end, so re-using it will lead to no-op for subsequent loads (they will be skipped + * at commit time). Do not carry a batch job's {@code sinkId} into a streaming job, + * as it would skip every commit window forever. + */ + public WriteCdcRows withSinkId(String sinkId) { + return toBuilder().setSinkId(sinkId).build(); + } + + /** The size of each event-time commit window (streaming only). */ + public WriteCdcRows withTriggeringFrequency(Duration triggeringFrequency) { + return toBuilder().setTriggeringFrequency(triggeringFrequency).build(); + } + + /** + * How far behind the watermark an element's event-time may be before it is dropped entirely, + * otherwise routed to {@link IcebergWriteResult#getDeadLetterRows()} as a late firing. Defaults + * to {@link #DEFAULT_ALLOWED_LATENESS} (6 hours) if unset. + */ + public WriteCdcRows withAllowedLateness(Duration allowedLateness) { + return toBuilder().setAllowedLateness(allowedLateness).build(); + } + + private CdcWriteConfig cdcWriteConfig() { + @Nullable Integer shardsPerPartition = getShardsPerPartition(); + return CdcWriteConfig.builder() + .setEqualityColumns(getEqualityColumns()) + .setSequenceNumberColumn(getSequenceNumberColumn()) + .setChangeTypeColumn(getChangeTypeColumn()) + .setChangeTypeMap(getChangeTypeMap()) + .setNumShards(getNumShards()) + .setShardsPerPartition(shardsPerPartition == null ? getNumShards() : shardsPerPartition) + .setSorterMemoryMB(getSorterMemoryMB()) + .setUpsert(getUpsert()) + .setTokenHeartbeatMillis(getTokenHeartbeatMillis()) + .setSinkId(getSinkId()) + .setSnapshotProperties(getSnapshotProperties()) + .setErrorHandling(getErrorHandlingEnabled()) + .build(); + } + + /** Construction-time validation. */ + private void validate(PCollection input, CdcWriteConfig config) { + Preconditions.checkArgument( + 1 + == Stream.of(getTableIdentifier(), getDynamicDestinations()) + .filter(Predicates.notNull()) + .count(), + "Must set exactly one of to(TableIdentifier) or to(DynamicDestinations)."); + + // Structural config invariants (value bounds, mutually exclusive options, reserved names). + config.validate(); + + boolean unbounded = input.isBounded() == PCollection.IsBounded.UNBOUNDED; + @Nullable Duration triggeringFrequency = getTriggeringFrequency(); + if (unbounded) { + if (triggeringFrequency == null) { + throw new IllegalArgumentException( + "Streaming CDC writes require withTriggeringFrequency(...)"); + } + Preconditions.checkArgument( + triggeringFrequency.isLongerThan(Duration.ZERO), + "triggering frequency must be positive for streaming CDC writes, got %s", + triggeringFrequency); + } + + @Nullable String changeTypeColumn = config.getChangeTypeColumn(); + if (changeTypeColumn != null) { + if (!input.getSchema().hasField(changeTypeColumn)) { + throw new IllegalArgumentException( + "CDC input schema is missing the change-type column '" + + changeTypeColumn + + "' named by change_type_column. Check the spelling, or drop change_type_column " + + "to use native change kind. Input schema: " + + input.getSchema()); + } + Schema.FieldType changeTypeType = input.getSchema().getField(changeTypeColumn).getType(); + if (changeTypeType.getTypeName() != Schema.TypeName.STRING) { + throw new IllegalArgumentException( + "CDC change-type column '" + + changeTypeColumn + + "' must be STRING, but the input schema declares it " + + changeTypeType + + "."); + } + if (changeTypeType.getNullable()) { + throw new IllegalArgumentException( + "CDC change-type column '" + + changeTypeColumn + + "' must be non-nullable, but the input schema declares it nullable."); + } + } + + String seqCol = config.getSequenceNumberColumn(); + if (!input.getSchema().hasField(seqCol)) { + throw new IllegalArgumentException( + "CDC input schema is missing the sequence-number column '" + + seqCol + + "', required to order each primary key's changes. Input schema: " + + input.getSchema()); + } + Schema.FieldType seqType = input.getSchema().getField(seqCol).getType(); + if (seqType.getTypeName() != Schema.TypeName.INT64) { + throw new IllegalArgumentException( + "CDC sequence-number column '" + + seqCol + + "' must be INT64, but the input schema declares it " + + seqType + + "."); + } + if (seqType.getNullable()) { + throw new IllegalArgumentException( + "CDC sequence-number column '" + + seqCol + + "' must be non-nullable, but the input schema declares it nullable."); + } + + // The idle token-refresh heartbeat is a streaming-only concept. + if (!unbounded && config.getTokenHeartbeatMillis() != null) { + LOG.warn( + "Token heartbeat is ignored for bounded input. Batch loads will commit once and exit."); + } + } + + @Override + public IcebergWriteResult expand(PCollection input) { + CdcWriteConfig config = cdcWriteConfig(); + validate(input, config); + + @Nullable DynamicDestinations destinations = getDynamicDestinations(); + if (destinations == null) { + destinations = + SingleTableDestinations.of( + checkStateNotNull(getTableIdentifier()), input.getSchema(), config); + } + + boolean unbounded = input.isBounded() == PCollection.IsBounded.UNBOUNDED; + @Nullable Duration allowedLateness = getAllowedLateness(); + + String runId = UUID.randomUUID().toString(); + + // Stage 1: destination + shard/sort-key assignment, and handle error rows + PCollectionTuple assigned = + input.apply( + "AssignCdcKeys", new AssignCdcKeys(getCatalogConfig(), config, destinations, runId)); + PCollection>> keyed = + assigned.get(AssignCdcKeys.KEYED); + // getFailedRows() is null when error handling is disabled + @Nullable PCollection failedRows = + getErrorHandlingEnabled() ? assigned.get(AssignCdcKeys.FAILED) : null; + + // Stage 2: create commit windows, group by (destination, shard) and split late rows to the + // replayable dead-letter, and sort groups by (pk, seq, kind) + CommitWindows.Result windowed = + keyed.apply( + "CommitWindows", + new CommitWindows( + config, + getTriggeringFrequency(), + allowedLateness != null ? allowedLateness : DEFAULT_ALLOWED_LATENESS)); + + // Stage 3: write delta files and emit their serialized metadata, one element per shard. + PCollection written = + windowed + .getSortedGroups() + .apply("WriteDeltas", new WriteDeltas(getCatalogConfig(), config, destinations, runId)); + + // Commit stage: ordered, idempotent commit + PCollection> snapshots = + written.apply( + "CommitDeltas", + new CommitDeltas( + getCatalogConfig(), + getSinkId(), + config.getSnapshotProperties(), + unbounded ? config.getTokenHeartbeatMillis() : null, + runId)); + + return IcebergWriteResult.cdc( + input.getPipeline(), snapshots, windowed.getDeadLetterRows(), failedRows); + } +} diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/package-info.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/package-info.java index 2bcd8b5f6222..c25496648f98 100644 --- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/package-info.java +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/package-info.java @@ -17,7 +17,236 @@ */ /** - * A change data capture (CDC) write sink for Apache Iceberg: applies inserts, updates, and deletes - * to a table, identifying rows by equality columns. + * Iceberg CDC sink: applies a stream or batch of inserts, updates and deletes to Iceberg V2+ + * tables, exposed as {@code IcebergIO.writeCdcRows(catalogConfig)}. For the transform's API and + * semantics see {@code WriteCdcRows}. + * + *

This is a merge-on-read sink. Within one commit window the writer collapses each primary key's + * changes to their final state, so a superseded intermediate row is never written at all; a change + * that reaches back into an already-committed snapshot is written as a PK-only equality + * delete. No position deletes or deletion vectors are ever written. + * + *

Monitoring

+ * + *

All metrics below are queryable through {@code PipelineResult.metrics()}; the namespace is the + * owning class's fully-qualified name, so filter on {@code + * org.apache.beam.sdk.io.iceberg.cdc.sink.*}. + * + *

Committer volume and latency

+ * + *

The normal throughput picture. None of these indicate a problem by themselves; watch their + * shape over time. + * + *

    + *
  • snapshotsCreated: CDC snapshots committed. In streaming this should track one per + * commit window per destination. A flat line while data is flowing means commits have + * stalled. + *
  • committedDataFiles / committedDeleteFiles: files published. Divide by {@code + * snapshotsCreated} for files per commit. A large ratio points to the small-file problem + * described below. + *
  • committedRecords: rows committed. + *
  • committedEqualityDeleteRecords: equality-delete rows committed. + *
  • committedBytes: total bytes of committed files. + *
  • commitDurationMs (distribution): wall-clock time spent waiting on the Iceberg commit + * operation. Its tail is your catalog's health: a growing maximum usually means catalog + * contention, not a Beam problem. + *
  • commitFailures: fires whenever the commit path throws. Iceberg's own optimistic + * retry runs underneath, so a genuine commit collision is counted only once that is + * exhausted. The failure is rethrown, so the bundle fails and the runner retries it; + * occasional increments under heavy concurrent writing are survivable. A sustained rate means + * the table has too many concurrent writers, and the sink will not make progress past the + * failing window (commits are strictly ordered). + *
  • heartbeatCommits: empty token-refresh commits emitted while a destination is idle + * (only if {@code withTokenHeartbeat} is configured). Each one is a real snapshot. With + * heartbeat enabled prefer cancel-and-resubmit over drain: the self-re-arming processing-time + * timer can keep a drain from completing. + *
+ * + *

Safety tripwires

+ * + *

These should be zero. A nonzero value is not necessarily an outage, but each one means + * something specific happened that you want to know about. + * + *

    + *
  • alreadyCommittedWindowsSkipped: a commit window was skipped because the table + * already carried a committed-through token at or past its end. Expected and benign after a + * restart or a retried bundle (we skip to ensure idempotency). Alarming in two cases: (1) in + * batch, where the whole load commits under one window, a nonzero value on a fresh + * load means a stable {@code sink_id} matched a previous load and this run wrote + * nothing at all. Use a unique {@code sink_id} per load, or omit it. Never carry a batch + * load's {@code sink_id} into a subsequent streaming run (the batch token is the + * global-window end, above every streaming window; the streaming committer will refuse a + * recovered batch token); and (2) in streaming, a persistently nonzero rate means data keeps + * arriving for windows that already closed, which is a source-lateness problem. The token + * proves that a window with that end committed, not that the skipped rows are the rows that + * committed. A logged warning names up to 5 of the window's files plus a count: your + * handle on the skipped rows until {@code remove_orphan_files} reclaims them. + *
  • orphanFiles: files (data and delete) carried by a skipped window: potential + * orphans (a pure redelivery's files are the committed live ones). Wasted storage until + * {@code remove_orphan_files} runs. + *
  • tokenParseFailures: an unparseable {@code beam.cdc.*} value was found while scanning + * snapshot ancestry. The committed-through scan continues to older ancestors rather than + * crash-looping; a bad max-seq or run-spec stamp just reads as absent. Nonzero means + * something wrote a malformed value into a snapshot summary; investigate before trusting the + * recovered position. + *
  • suspectedTokenExpiry: the sink's own {@code beam.cdc.sink-id} marker was found in + * the ancestry but no committed-through token was. Strongly implies that {@code + * expire_snapshots} removed the token-bearing snapshots while the pipeline was down. Recovery + * then falls back to the beginning and may re-apply retained windows. If you see this, either + * lengthen snapshot retention or enable {@code withTokenHeartbeat} so the token stays young. + *
  • crossWindowSequenceInversions: a committed window's minimum source sequence number + * was below an earlier window's committed maximum. This is the detector for a violated + * ordering contract (see below) and the only tripwire here that can mean silently wrong + * table contents. It has benign false positives when the two windows touch entirely + * disjoint primary keys, so treat it as "go and check the source's ordering", not as proof of + * corruption. + *
  • specMismatchedWindows: signals the partition spec was evolved mid-run. Incremented + * when a committed window carries equality deletes and uses partition-spec ids different from + * the run's pinned spec. Those deletes may not reach rows written under the other spec. + *
+ * + *

Input health

+ * + *

These describe what is arriving, not what the sink did with it. + * + *

    + *
  • deadLetterRecords: records whose grouped pane fired late (the watermark had already + * passed their commit window's end), diverted to the replayable dead-letter output ({@code + * IcebergWriteResult.getDeadLetterRows()}) instead of being applied out of order. + * Every late pane is diverted, including a window's first ({@code SplitLateData}'s + * javadoc explains why). Nonzero means your source is lagging past the commit window. The + * records are not lost, but nothing consumes them unless you wire that output somewhere. + * Records later than {@code withAllowedLateness} are a different case: see below. + *
  • failedRecords: poison records diverted to {@code IcebergWriteResult.getFailedRows()} + * when {@code withErrorHandling()} is on: unknown change type, missing sequence number, null + * equality value, unresolvable destination. Without error handling these fail the pipeline + * instead. Like the dead letters above, the records are not lost, but you need to wire the + * output somewhere to avoid dropping them. + *
  • upsertUpdateBeforeDropped: {@code UPDATE_BEFORE} records discarded because the sink + * is in upsert mode, which needs only after-images. Tells you how much of your input was + * redundant. + *
+ * + *

Records more than {@code withAllowedLateness} behind the watermark are dropped by the runner + * at the {@code GroupByShardKey} step. Raising {@code withAllowedLateness} is the only way to + * capture them (at the cost of more live window state per destination). + * + *

Files and maintenance

+ * + *

File count per commit is governed by {@code num_shards} × touched partitions, and + * total file count by that times the number of commit windows. A table with many partitions and a + * short {@code triggering_frequency_seconds} produces a lot of small files very quickly, regardless + * of the data rate. {@code num_shards} (default=16) is the sink's write-parallelism knob. Too low + * may lead to a write bottleneck and commit backlog. Too high is fine for the sink but may lead to + * slower downstream reads due to small files. + * + *

There are three levers that you can use: + * + *

    + *
  • {@code triggering_frequency_seconds}: increasing it leads to fewer, larger commits, + * at the cost of end-to-end latency. + *
  • {@code num_shards}: trades throughput for file count. Lowering it reduces file + * count, but also the write-parallelism ceiling. + *
  • {@code shards_per_partition} (default={@code num_shards}, i.e. no cap): on a + * partitioned table, cap how many shards one partition's rows may occupy. A touched + * partition then writes about {@code min(shards_per_partition, distinct keys)} files per file + * kind per commit, with its write parallelism capped to match; {@code 1} is the pure + * partition-affine endpoint (one writer, about one data file, per partition). + *
+ * + *

The trade-off a lowered {@code shards_per_partition} makes, stated plainly: effective + * write parallelism per destination becomes at most the cap times the number of distinct partitions + * receiving data in a window: at {@code 1}, one writer per partition. A cap of {@code 1} is right + * for a table with many partitions and wrong for a table with three, which it would hold at + * three concurrent writers however many workers the pipeline has; such a table wants an + * intermediate cap, sized so cap × touched partitions still covers the pipeline's write + * parallelism. The cap is moot for an unpartitioned table, where it is simply ignored (with a WARN) + * rather than funnelling everything through a single shard. At {@code 1} this is the same + * limitation Flink's {@code hash} write-distribution mode carries; the dial exists because the + * alternative (leaving {@code num_shards} as the only lever) forces the same trade with none of the + * benefit. It is safe with {@code upsert}: both options require every partition source column to be + * an equality column (see the partitioning section below), which makes a row's partition a pure + * function of its primary key. + * + *

Snapshot expiry bounds the sink's own cost, not just read performance

+ * + *

This sink creates one snapshot per commit window per table. The following sink costs + * grow as snapshot count grows: + * + *

    + *
  • Commit latency: Every commit fire loads the destination's metadata once, then walks its + * ancestry to find the latest committed-through token. Each commit within that fire publishes + * a snapshot, which requires refreshing and re-parsing the metadata. Both scale with the + * number of snapshots in that file. + *
  • Worker heap: Each cached table pins a {@code TableMetadata} holding every retained + * snapshot, per worker, multiplied by the number of dynamic destinations that worker touches. + *
+ * + *

Orphan files

+ * + *

The writer stage writes data files and delete files and passes their metadata to the committer + * stage in the pipeline. Every orphan is an ordinary data or delete file that no snapshot ended up + * referencing. Orphan files can be produced in three ways: + * + *

    + *
  • A retried or failed writer bundle. Each attempt names its files with a fresh UUID, + * so whichever attempt's output element loses the retry race leaves its files unreferenced. + *
  • A window skipped because the committed-through token already covers its end. This + * happens when a relaunch reuses a stable {@code sink_id}. The incoming element is targeting + * a window that has already been committed with different files. If the incoming element's + * files contain different content than the committed ones, they may hold rows the table never + * received. Those files are left in place so you can recover them. Counted by {@code + * orphanFiles}; the skip WARN names up to 5 of them plus a count. + *
  • A failed commit attempt. Iceberg normally deletes the manifests it wrote when a + * commit fails, but it cannot when the outcome is unknown (i.e. {@code + * CommitStateUnknownException}). The commit may in fact have succeeded, and deleting those + * manifests would corrupt a live snapshot. Only the manifests are left orphaned. The window's + * own data and delete files are untouched, because the pending bag survives the failure and + * the retry commits those same files. + *
+ * + *

All of these are reclaimed by Iceberg's {@code remove_orphan_files}. The sink writes + * everything through the table's own write path. Give the procedure an age threshold much longer + * than your longest in-flight bundle to avoid deleting files a live pipeline is about to reference. + * + *

Partition-spec evolution mid-run

+ * + *

Iceberg matches an equality delete to its data files by {@code (spec id, partition)}, so if a + * table's spec changes during streaming writes, the sink will choose to keep writing with the old + * spec. Otherwise, it could leave new deletes that never reach old-spec rows, which is data + * corruption. If a window happens to still mix specs while carrying equality deletes, it commits + * anyway, WARNs, and increments the {@code specMismatchedWindows} counter. + * + *

If you update a table's spec, run {@code rewrite_data_files} first so old-spec rows are + * rewritten under the new spec. A relaunched pipeline or in-place update will adopt the new spec. + * + *

Partitioning

+ * + *

Tables may be partitioned on any columns, key or not, with any transforms. Two options still + * require every partition source column to be an equality column: {@code upsert} (before-images are + * dropped, so a row that moved partitions could never be deleted from its old one) and a {@code + * shards_per_partition} cap below {@code num_shards} (the shard is derived from the partition + * tuple, which must therefore be a pure function of the primary key). + * + *

Non-key partitioning sharpens the input contract: every update must carry its {@code + * UPDATE_BEFORE}, because a moved row whose before-image never arrives leaves a permanent duplicate + * in the old partition that no later delete reaches; and {@code UPDATE_BEFORE}/{@code DELETE} rows + * must carry the row's actual old values in the partition source columns, because a nulled non-key + * column routes the equality delete to the null partition. + * + *

The ordering contract

+ * + *

The sink orders each primary key's changes by the sequence-number column, and commits one + * snapshot per commit window in ascending window order. That is only sound if the two orderings + * agree at the source: for a given key, an element's event time must be non-decreasing with its + * (sequence number, kind rank), so equal-sequence records (an update's before and after images) + * carry equal event times. If a higher-sequence change lands in an earlier commit window + * than a lower-sequence one, a stale equality delete can be committed after the row it should not + * have touched, and the table's final contents are wrong. + * + *

This is a source contract. Violations are detected at commit time and visible with the {@code + * crossWindowSequenceInversions} counter and warning logs, but the commit still proceeds. The fix + * is in how event times are assigned upstream. Replaying dead letters is bound by the same + * contract: see the replay caveat at {@code IcebergWriteResult#getDeadLetterRows}. */ package org.apache.beam.sdk.io.iceberg.cdc.sink; diff --git a/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRowsReadBackTest.java b/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRowsReadBackTest.java new file mode 100644 index 000000000000..6e18e903e7e7 --- /dev/null +++ b/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRowsReadBackTest.java @@ -0,0 +1,1526 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.iceberg.cdc.sink; + +import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull; +import static org.apache.beam.sdk.values.ValueKind.DELETE; +import static org.apache.beam.sdk.values.ValueKind.INSERT; +import static org.apache.beam.sdk.values.ValueKind.UPDATE_AFTER; +import static org.apache.beam.sdk.values.ValueKind.UPDATE_BEFORE; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.notNullValue; + +import java.io.IOException; +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.time.Instant; +import java.time.LocalDate; +import java.time.OffsetDateTime; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.stream.Collectors; +import org.apache.beam.sdk.PipelineResult; +import org.apache.beam.sdk.coders.CoderException; +import org.apache.beam.sdk.coders.KvCoder; +import org.apache.beam.sdk.coders.RowCoder; +import org.apache.beam.sdk.coders.SerializableCoder; +import org.apache.beam.sdk.io.iceberg.DynamicDestinations; +import org.apache.beam.sdk.io.iceberg.IcebergCatalogConfig; +import org.apache.beam.sdk.io.iceberg.IcebergIO; +import org.apache.beam.sdk.io.iceberg.IcebergUtils; +import org.apache.beam.sdk.metrics.MetricNameFilter; +import org.apache.beam.sdk.metrics.MetricResult; +import org.apache.beam.sdk.metrics.MetricsFilter; +import org.apache.beam.sdk.schemas.Schema; +import org.apache.beam.sdk.testing.TestPipeline; +import org.apache.beam.sdk.testing.TestStream; +import org.apache.beam.sdk.transforms.Create; +import org.apache.beam.sdk.transforms.windowing.FixedWindows; +import org.apache.beam.sdk.transforms.windowing.GlobalWindow; +import org.apache.beam.sdk.transforms.windowing.Window; +import org.apache.beam.sdk.util.CoderUtils; +import org.apache.beam.sdk.values.KV; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.Row; +import org.apache.beam.sdk.values.TimestampedValue; +import org.apache.beam.sdk.values.ValueKind; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileContent; +import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.SortOrder; +import org.apache.iceberg.StructLike; +import org.apache.iceberg.Table; +import org.apache.iceberg.catalog.Catalog; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.data.IcebergGenerics; +import org.apache.iceberg.expressions.Expressions; +import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.DateTimeUtil; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.joda.time.Duration; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Read-back end-to-end tests for the assembled CDC sink: each case drives the full public transform + * against a real HadoopCatalog table and asserts the table's final committed contents via {@link + * IcebergGenerics#read}: the sink's ground truth, where a composition bug the per-stage suites + * cannot see shows up as wrong rows. Every case asserts the COMPLETE expected row set, all columns, + * never a count. The partition-type cases pin ground the path-rendered predecessor banned or + * crashed on; the typed-JSON transport carries partition tuples with no rendered path to parse. + * Every test creates a uniquely named table (TableCache is process-wide). + */ +@RunWith(JUnit4.class) +public class WriteCdcRowsReadBackTest { + + @Rule public transient TestPipeline p = TestPipeline.create(); + @Rule public transient TemporaryFolder tmp = new TemporaryFolder(); + + /** Canonical test table schema, shared with the other {@code cdc/sink} suites. */ + private static final org.apache.iceberg.Schema ICEBERG_SCHEMA = + new org.apache.iceberg.Schema( + Types.NestedField.required(1, "id", Types.IntegerType.get()), + Types.NestedField.optional(2, "name", Types.StringType.get()), + Types.NestedField.optional(3, "data", Types.StringType.get())); + + private static final Schema DATA_SCHEMA = + Schema.builder() + .addInt32Field("id") + .addNullableField("name", Schema.FieldType.STRING) + .addNullableField("data", Schema.FieldType.STRING) + .build(); + + /** Input schema = the canonical data schema + a sequence-number column named {@code seq}. */ + private static final Schema INPUT_SCHEMA = + Schema.builder().addFields(DATA_SCHEMA.getFields()).addInt64Field("seq").build(); + + /** Event-time origin for the streaming (TestStream) cases. */ + private static final org.joda.time.Instant BASE = new org.joda.time.Instant(0); + + private static final Duration WINDOW = Duration.standardSeconds(60); + + private Catalog catalog; + + @Before + public void setUp() { + catalog = CdcSinkTestUtils.hadoopCatalog(tmp.getRoot()); + } + + private IcebergCatalogConfig catalogConfig() { + return CdcSinkTestUtils.catalogConfig(tmp.getRoot()); + } + + // ----------------------------------------------------------------------------------------------- + // Fixtures + // ----------------------------------------------------------------------------------------------- + + /** Creates a uniquely named table and returns its identifier. */ + private TableIdentifier createTable( + String prefix, + org.apache.iceberg.Schema schema, + Set identifierFieldIds, + int formatVersion, + PartitionSpec spec) { + TableIdentifier id = TableIdentifier.of("db", prefix + "_" + System.nanoTime()); + CdcSinkTestUtils.createTable(catalog, id, schema, identifierFieldIds, formatVersion, spec); + return id; + } + + /** A uniquely named unpartitioned table over {@link #ICEBERG_SCHEMA} (PK = {@code id}). */ + private TableIdentifier createCanonicalTable(String prefix, int formatVersion) { + return createTable( + prefix, ICEBERG_SCHEMA, ImmutableSet.of(1), formatVersion, PartitionSpec.unpartitioned()); + } + + /** The sink's input schema for {@code table}: its Beam schema plus the {@code seq} column. */ + private static Schema inputSchemaFor(Table table) { + return Schema.builder() + .addFields(IcebergUtils.icebergSchemaToBeamSchema(table.schema()).getFields()) + .addInt64Field("seq") + .build(); + } + + /** Builds an input row over the canonical {@link #INPUT_SCHEMA}. */ + private static Row row(int id, String name, String data, long seq) { + return Row.withSchema(INPUT_SCHEMA).addValues(id, name, data, seq).build(); + } + + /** Builds an input row over an arbitrary schema (the per-test partitioned/timestamp tables). */ + private static Row row(Schema schema, Object... values) { + return Row.withSchema(schema).addValues(values).build(); + } + + /** A bounded CDC input of the given kind-tagged rows. */ + @SafeVarargs + private final PCollection boundedInput(Schema schema, KV... rows) { + return boundedInput(schema, ImmutableList.copyOf(rows)); + } + + /** A bounded CDC input of the given kind-tagged rows. */ + private PCollection boundedInput(Schema schema, List> rows) { + return CdcSinkTestUtils.withKinds(p.apply(Create.of(rows).withCoder(taggedRowCoder(schema)))) + .setRowSchema(schema); + } + + /** An unbounded CDC input driven by {@code stream}. */ + private PCollection streamingInput(Schema schema, TestStream> stream) { + return CdcSinkTestUtils.withKinds(p.apply(stream)).setRowSchema(schema); + } + + private static KvCoder taggedRowCoder(Schema schema) { + return KvCoder.of(SerializableCoder.of(ValueKind.class), RowCoder.of(schema)); + } + + private static TestStream.Builder> testStream(Schema schema) { + return TestStream.create(taggedRowCoder(schema)); + } + + private static TimestampedValue> at(ValueKind kind, Row row, int atSeconds) { + return TimestampedValue.of(KV.of(kind, row), BASE.plus(Duration.standardSeconds(atSeconds))); + } + + // ----------------------------------------------------------------------------------------------- + // Ground truth: read the committed table back + // ----------------------------------------------------------------------------------------------- + + /** + * Every live row, the named columns joined by {@code ':'}, sorted; {@code timestamptz} rendered + * as micros-since-epoch so a truncated or shifted timestamp cannot hide behind formatting. + */ + private static List readColumns(Table table, String... columns) { + table.refresh(); + return ImmutableList.copyOf(IcebergGenerics.read(table).build()).stream() + .map( + record -> + Arrays.stream(columns) + .map(column -> render(record.getField(column))) + .collect(Collectors.joining(":"))) + .sorted() + .collect(ImmutableList.toImmutableList()); + } + + /** {@link #readColumns} over the canonical {@code (id, name, data)} schema. */ + private static List readRows(Table table) { + return readColumns(table, "id", "name", "data"); + } + + private static String render(@Nullable Object value) { + if (value instanceof OffsetDateTime) { + return String.valueOf(DateTimeUtil.microsFromTimestamptz((OffsetDateTime) value)); + } + return String.valueOf(value); + } + + /** {@code instant} as micros since epoch, matching how {@link #readColumns} renders it. */ + private static String micros(Instant instant) { + return String.valueOf(ChronoUnit.MICROS.between(Instant.EPOCH, instant)); + } + + /** + * The distinct partition tuples of the live data files: proves WHERE rows physically landed, + * which the row contents alone cannot show. Binary values render as hex so a byte-lossy + * round-trip is visible. + */ + private static List scannedPartitions(Table table, int numFields) throws IOException { + table.refresh(); + Set partitions = new TreeSet<>(); + try (CloseableIterable tasks = table.newScan().planFiles()) { + for (FileScanTask task : tasks) { + StructLike partition = task.file().partition(); + List values = new ArrayList<>(); + for (int i = 0; i < numFields; i++) { + Object value = partition.get(i, Object.class); + values.add( + value instanceof ByteBuffer || value instanceof byte[] + ? hex(value) + : String.valueOf(value)); + } + partitions.add(String.join(":", values)); + } + } + return ImmutableList.copyOf(partitions); + } + + private static int snapshotCount(Table table) { + table.refresh(); + return Lists.newArrayList(table.snapshots()).size(); + } + + private static List addedDeleteFiles(Table table) { + table.refresh(); + return Lists.newArrayList( + checkStateNotNull(table.currentSnapshot()).addedDeleteFiles(table.io())); + } + + // ----------------------------------------------------------------------------------------------- + // 1. Same-commit insert + delete (V2) + // ----------------------------------------------------------------------------------------------- + + /** + * A key inserted then deleted inside one commit window leaves no trace, and the same-commit dedup + * touches nothing else. + */ + @Test + public void v2SameCommitInsertThenDeleteLeavesNoRowForTheKey() { + TableIdentifier id = createCanonicalTable("v2dedup", 2); + Table t = catalog.loadTable(id); + + boundedInput( + INPUT_SCHEMA, + KV.of(INSERT, row(1, "a", "x", 1L)), + KV.of(DELETE, row(1, "a", "x", 2L)), + KV.of(INSERT, row(2, "b", "y", 1L)), + KV.of(INSERT, row(3, "c", "z", 1L))) + .apply(IcebergIO.writeCdcRows(catalogConfig()).to(id).withSequenceNumberColumn("seq")); + p.run().waitUntilFinish(); + + assertThat(readRows(t), containsInAnyOrder("2:b:y", "3:c:z")); + } + + // ----------------------------------------------------------------------------------------------- + // 2. Same-commit dedup on a format-version-3 table + // ----------------------------------------------------------------------------------------------- + + /** + * Same-commit churn on V3 collapses inside the writer: the commit carries only each key's final + * row and no delete files at all. Structurally case 1 on a V3 table, kept because V3 is the + * format version the sink used to special-case, and it must keep working end to end. + */ + @Test + public void v3SameCommitChangesCollapseWithoutDeleteFiles() { + TableIdentifier id = createCanonicalTable("v3dedup", 3); + Table t = catalog.loadTable(id); + + boundedInput( + INPUT_SCHEMA, + KV.of(INSERT, row(1, "a", "x", 1L)), + KV.of(UPDATE_BEFORE, row(1, "a", "x", 2L)), + KV.of(UPDATE_AFTER, row(1, "a2", "z", 2L)), + KV.of(INSERT, row(2, "b", "y", 1L)), + KV.of(DELETE, row(2, "b", "y", 2L)), + KV.of(INSERT, row(3, "c", "w", 1L))) + .apply(IcebergIO.writeCdcRows(catalogConfig()).to(id).withSequenceNumberColumn("seq")); + p.run().waitUntilFinish(); + + assertThat(readRows(t), containsInAnyOrder("1:a2:z", "3:c:w")); + + // Every change here is same-window churn, so the collapse leaves nothing to delete. + assertThat(addedDeleteFiles(t), empty()); + } + + // ----------------------------------------------------------------------------------------------- + // 3. Cross-commit delete + // ----------------------------------------------------------------------------------------------- + + /** + * A later window's delete removes a row committed by an earlier snapshot: the cross-commit + * equality delete reaches back, and only the deleted key disappears. + */ + @Test + public void crossCommitDeleteRemovesRowCommittedByEarlierWindow() { + TableIdentifier id = createCanonicalTable("xcommit", 2); + Table t = catalog.loadTable(id); + + TestStream> stream = + testStream(INPUT_SCHEMA) + .advanceWatermarkTo(BASE) + .addElements( + at(INSERT, row(1, "a", "x", 1L), 1), + at(INSERT, row(2, "b", "y", 1L), 2), + at(INSERT, row(3, "c", "z", 1L), 3)) + .advanceWatermarkTo(BASE.plus(Duration.standardSeconds(70))) // window 0 commits + .addElements(at(DELETE, row(1, "a", "x", 2L), 61)) + .advanceWatermarkTo(BASE.plus(Duration.standardSeconds(130))) // window 1 commits + .advanceWatermarkToInfinity(); + + streamingInput(INPUT_SCHEMA, stream) + .apply( + IcebergIO.writeCdcRows(catalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withTriggeringFrequency(WINDOW)); + p.run().waitUntilFinish(); + + // Two windows really did commit separately; otherwise this would be same-commit dedup again. + assertThat(snapshotCount(t), equalTo(2)); + assertThat(readRows(t), containsInAnyOrder("2:b:y", "3:c:z")); + } + + // ----------------------------------------------------------------------------------------------- + // 4. Upsert across commit windows + // ----------------------------------------------------------------------------------------------- + + /** + * In upsert mode, a later window's image of a key replaces the one an earlier window committed: + * one row survives, carrying the later values. + */ + @Test + public void upsertCrossWindowReplacesEarlierCommittedImage() { + TableIdentifier id = createCanonicalTable("upsertxwin", 2); + Table t = catalog.loadTable(id); + + TestStream> stream = + testStream(INPUT_SCHEMA) + .advanceWatermarkTo(BASE) + .addElements( + at(INSERT, row(1, "a", "v1", 1L), 1), at(INSERT, row(2, "b", "keep", 1L), 2)) + .advanceWatermarkTo(BASE.plus(Duration.standardSeconds(70))) // window 0 commits + .addElements(at(UPDATE_AFTER, row(1, "a2", "v2", 2L), 61)) + .advanceWatermarkTo(BASE.plus(Duration.standardSeconds(130))) // window 1 commits + .advanceWatermarkToInfinity(); + + streamingInput(INPUT_SCHEMA, stream) + .apply( + IcebergIO.writeCdcRows(catalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withUpsert(true) + .withTriggeringFrequency(WINDOW)); + p.run().waitUntilFinish(); + + assertThat(snapshotCount(t), equalTo(2)); + assertThat(readRows(t), containsInAnyOrder("1:a2:v2", "2:b:keep")); + } + + // ----------------------------------------------------------------------------------------------- + // 5. Adversarial element order within one window + // ----------------------------------------------------------------------------------------------- + + /** + * Element arrival order inside a commit window is irrelevant: a DELETE fed before its own INSERT + * and out-of-order after-images still produce the sequence-order final state. + */ + @Test + public void streamingShuffledWithinWindowAppliesSequenceOrder() { + TableIdentifier id = createCanonicalTable("shuffled", 2); + Table t = catalog.loadTable(id); + + // All eight changes land in window [0s, 60s), arrival order deliberately adversarial. Event + // times are equal, so the per-key event-time/sequence source contract still holds. + TestStream> stream = + testStream(INPUT_SCHEMA) + .advanceWatermarkTo(BASE) + .addElements( + at(DELETE, row(1, "a2", "x2", 3L), 5), + at(UPDATE_AFTER, row(2, "v3", "c", 3L), 5), + at(UPDATE_AFTER, row(1, "a2", "x2", 2L), 5), + at(INSERT, row(3, "c", "w", 1L), 5), + at(INSERT, row(2, "v1", "a", 1L), 5), + at(UPDATE_BEFORE, row(1, "a", "x", 2L), 5), + at(UPDATE_AFTER, row(2, "v2", "b", 2L), 5), + at(INSERT, row(1, "a", "x", 1L), 5)) + .advanceWatermarkTo( + BASE.plus(Duration.standardSeconds(70))) // the single window commits + .advanceWatermarkToInfinity(); + + streamingInput(INPUT_SCHEMA, stream) + .apply( + IcebergIO.writeCdcRows(catalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withTriggeringFrequency(WINDOW)); + p.run().waitUntilFinish(); + + // One window => one snapshot, so this really is intra-window ordering, not commit ordering. + assertThat(snapshotCount(t), equalTo(1)); + // In sequence order id=1 ends deleted, id=2's seq-3 after-image wins, id=3 is untouched. + assertThat(readRows(t), containsInAnyOrder("2:v3:c", "3:c:w")); + } + + // ----------------------------------------------------------------------------------------------- + // 7. day(timestamptz) partitioning, through the full sink + // ----------------------------------------------------------------------------------------------- + + /** + * A {@code day(timestamptz)}-partitioned table takes inserts, updates and deletes through the + * whole sink, and every row lands in the day partition its timestamp implies, with no rendered + * partition path anywhere (the predecessor crashed here until DATE got a renderer special-case). + */ + @Test + public void dayPartitionedTableAppliesChangesAndLandsInExpectedPartitions() throws IOException { + org.apache.iceberg.Schema icebergSchema = + new org.apache.iceberg.Schema( + Types.NestedField.required(1, "id", Types.IntegerType.get()), + Types.NestedField.required(2, "ts", Types.TimestampType.withZone()), + Types.NestedField.optional(3, "name", Types.StringType.get())); + // ts is also an equality column: the key-derived-partition shape (case 16 covers non-key). + TableIdentifier id = + createTable( + "daypart", + icebergSchema, + ImmutableSet.of(1, 2), + 2, + PartitionSpec.builderFor(icebergSchema).day("ts").build()); + Table t = catalog.loadTable(id); + Schema inputSchema = inputSchemaFor(t); + + Instant march = Instant.parse("2024-03-15T10:30:00.123456Z"); + Instant june = Instant.parse("2024-06-01T00:00:00Z"); + Instant marchLate = Instant.parse("2024-03-15T23:59:59.999999Z"); + + boundedInput( + inputSchema, + KV.of(INSERT, row(inputSchema, 1, march, "a", 1L)), + KV.of(INSERT, row(inputSchema, 2, june, "b", 1L)), + KV.of(INSERT, row(inputSchema, 3, marchLate, "c", 1L)), + // Update inside the March partition (ts unchanged, so the key is unchanged). + KV.of(UPDATE_BEFORE, row(inputSchema, 1, march, "a", 2L)), + KV.of(UPDATE_AFTER, row(inputSchema, 1, march, "a2", 2L)), + // Delete the other March row. + KV.of(DELETE, row(inputSchema, 3, marchLate, "c", 2L))) + .apply(IcebergIO.writeCdcRows(catalogConfig()).to(id).withSequenceNumberColumn("seq")); + p.run().waitUntilFinish(); + + assertThat( + readColumns(t, "id", "ts", "name"), + containsInAnyOrder("1:" + micros(march) + ":a2", "2:" + micros(june) + ":b")); + + // Rows landed in the day partitions their timestamps imply, not in one lump. + assertThat( + scannedPartitions(t, 1), + containsInAnyOrder( + String.valueOf((int) LocalDate.of(2024, 3, 15).toEpochDay()), + String.valueOf((int) LocalDate.of(2024, 6, 1).toEpochDay()))); + } + + // ----------------------------------------------------------------------------------------------- + // 8. Multi-transform partition spec, through the full sink + // ----------------------------------------------------------------------------------------------- + + /** + * A spec combining {@code hour(ts)} and {@code identity(DATE)} applies end to end: contents + * right, each row in its {@code (hour, date)} partition pair. + */ + @Test + public void multiTransformPartitionSpecAppliesChangesEndToEnd() throws IOException { + org.apache.iceberg.Schema icebergSchema = + new org.apache.iceberg.Schema( + Types.NestedField.required(1, "id", Types.IntegerType.get()), + Types.NestedField.required(2, "ts", Types.TimestampType.withZone()), + Types.NestedField.required(3, "d", Types.DateType.get()), + Types.NestedField.optional(4, "name", Types.StringType.get())); + TableIdentifier id = + createTable( + "multipart", + icebergSchema, + ImmutableSet.of(1, 2, 3), + 2, + PartitionSpec.builderFor(icebergSchema).hour("ts").identity("d").build()); + Table t = catalog.loadTable(id); + Schema inputSchema = inputSchemaFor(t); + + Instant hour10 = Instant.parse("2024-03-15T10:30:00Z"); + Instant hour10b = Instant.parse("2024-03-15T10:45:00Z"); + Instant hour11 = Instant.parse("2024-03-15T11:05:00Z"); + LocalDate day15 = LocalDate.of(2024, 3, 15); + LocalDate day16 = LocalDate.of(2024, 3, 16); + + boundedInput( + inputSchema, + KV.of(INSERT, row(inputSchema, 1, hour10, day15, "a", 1L)), + KV.of(INSERT, row(inputSchema, 2, hour11, day16, "b", 1L)), + KV.of(INSERT, row(inputSchema, 3, hour10b, day15, "c", 1L)), + KV.of(UPDATE_BEFORE, row(inputSchema, 1, hour10, day15, "a", 2L)), + KV.of(UPDATE_AFTER, row(inputSchema, 1, hour10, day15, "a2", 2L)), + KV.of(DELETE, row(inputSchema, 3, hour10b, day15, "c", 2L))) + .apply(IcebergIO.writeCdcRows(catalogConfig()).to(id).withSequenceNumberColumn("seq")); + p.run().waitUntilFinish(); + + assertThat( + readColumns(t, "id", "ts", "d", "name"), + containsInAnyOrder( + "1:" + micros(hour10) + ":" + day15 + ":a2", + "2:" + micros(hour11) + ":" + day16 + ":b")); + + assertThat( + scannedPartitions(t, 2), + containsInAnyOrder( + hoursFromEpoch(hour10) + ":" + (int) day15.toEpochDay(), + hoursFromEpoch(hour11) + ":" + (int) day16.toEpochDay())); + } + + private static long hoursFromEpoch(Instant instant) { + return ChronoUnit.HOURS.between(Instant.EPOCH, instant); + } + + // ----------------------------------------------------------------------------------------------- + // 9. timestamptz round-trip at microsecond precision + // ----------------------------------------------------------------------------------------------- + + /** + * {@code timestamptz} survives the whole sink at microsecond precision, including pre-epoch + * instants (negative seconds, non-negative sub-second part). + */ + @Test + public void timestamptzRoundTripsAtMicrosecondPrecision() { + org.apache.iceberg.Schema icebergSchema = + new org.apache.iceberg.Schema( + Types.NestedField.required(1, "id", Types.IntegerType.get()), + Types.NestedField.required(2, "ts", Types.TimestampType.withZone()), + Types.NestedField.optional(3, "name", Types.StringType.get())); + TableIdentifier id = + createTable("tstz", icebergSchema, ImmutableSet.of(1), 2, PartitionSpec.unpartitioned()); + Table t = catalog.loadTable(id); + Schema inputSchema = inputSchemaFor(t); + + Instant afterEpoch = Instant.parse("2024-03-15T10:30:00.123456Z"); + Instant beforeEpoch = Instant.parse("1969-07-20T20:17:40.000001Z"); + Instant longBeforeEpoch = Instant.parse("1901-12-13T20:45:52.654321Z"); + Instant updated = Instant.parse("1955-11-05T06:15:00.000123Z"); + Instant deleted = Instant.parse("1962-10-16T12:00:00.999999Z"); + + boundedInput( + inputSchema, + KV.of(INSERT, row(inputSchema, 1, afterEpoch, "after", 1L)), + KV.of(INSERT, row(inputSchema, 2, beforeEpoch, "before", 1L)), + KV.of(INSERT, row(inputSchema, 3, longBeforeEpoch, "long-before", 1L)), + KV.of(INSERT, row(inputSchema, 4, deleted, "deleted", 1L)), + // The timestamp itself is updated (ts is not part of the key here). + KV.of(UPDATE_BEFORE, row(inputSchema, 2, beforeEpoch, "before", 2L)), + KV.of(UPDATE_AFTER, row(inputSchema, 2, updated, "before-updated", 2L)), + // The DELETE is what fails a sink that ignores change kinds: same-commit dedup would + // reproduce id=2's expected row regardless, but a DELETE has no such cover. + KV.of(DELETE, row(inputSchema, 4, deleted, "deleted", 2L))) + .apply(IcebergIO.writeCdcRows(catalogConfig()).to(id).withSequenceNumberColumn("seq")); + p.run().waitUntilFinish(); + + assertThat( + readColumns(t, "id", "ts", "name"), + containsInAnyOrder( + "1:" + micros(afterEpoch) + ":after", + "2:" + micros(updated) + ":before-updated", + "3:" + micros(longBeforeEpoch) + ":long-before")); + } + + // ----------------------------------------------------------------------------------------------- + // 10. '/' inside a string partition value + // ----------------------------------------------------------------------------------------------- + + /** + * A {@code '/'} inside an identity-partitioned string value round-trips: the tuple travels as + * typed JSON, so a path-separator lookalike is just a value. The predecessor banned this outright + * ({@code SerializableDataFileTest} still pins the NaN path-fallback's encoded form). + */ + @Test + public void slashInStringPartitionValueAppliesChangesEndToEnd() throws IOException { + org.apache.iceberg.Schema icebergSchema = + new org.apache.iceberg.Schema( + Types.NestedField.required(1, "id", Types.IntegerType.get()), + Types.NestedField.required(2, "name", Types.StringType.get()), + Types.NestedField.optional(3, "data", Types.StringType.get())); + TableIdentifier id = + createTable( + "slashpart", + icebergSchema, + ImmutableSet.of(1, 2), + 2, + PartitionSpec.builderFor(icebergSchema).identity("name").build()); + Table t = catalog.loadTable(id); + Schema inputSchema = inputSchemaFor(t); + + boundedInput( + inputSchema, + KV.of(INSERT, row(inputSchema, 1, "a/b", "x", 1L)), + KV.of(INSERT, row(inputSchema, 2, "c/d/e", "y", 1L)), + KV.of(INSERT, row(inputSchema, 3, "plain", "z", 1L)), + KV.of(INSERT, row(inputSchema, 4, "a/b", "w", 1L)), + // Update and delete inside the slash-bearing partition. + KV.of(UPDATE_BEFORE, row(inputSchema, 1, "a/b", "x", 2L)), + KV.of(UPDATE_AFTER, row(inputSchema, 1, "a/b", "x2", 2L)), + KV.of(DELETE, row(inputSchema, 4, "a/b", "w", 2L))) + .apply(IcebergIO.writeCdcRows(catalogConfig()).to(id).withSequenceNumberColumn("seq")); + p.run().waitUntilFinish(); + + assertThat(readRows(t), containsInAnyOrder("1:a/b:x2", "2:c/d/e:y", "3:plain:z")); + assertThat(scannedPartitions(t, 1), containsInAnyOrder("a/b", "c/d/e", "plain")); + } + + // ----------------------------------------------------------------------------------------------- + // 11. shards_per_partition + // ----------------------------------------------------------------------------------------------- + + /** {@code (id INT, ts TIMESTAMPTZ)} both required and both identifier fields, plus a name. */ + private static final org.apache.iceberg.Schema DAY_PARTITIONED_SCHEMA = + new org.apache.iceberg.Schema( + Types.NestedField.required(1, "id", Types.IntegerType.get()), + Types.NestedField.required(2, "ts", Types.TimestampType.withZone()), + Types.NestedField.optional(3, "name", Types.StringType.get())); + + /** A uniquely named {@code day(ts)}-partitioned table over {@link #DAY_PARTITIONED_SCHEMA}. */ + private TableIdentifier createDayPartitionedTable(String prefix) { + return createTable( + prefix, + DAY_PARTITIONED_SCHEMA, + ImmutableSet.of(1, 2), + 2, + PartitionSpec.builderFor(DAY_PARTITIONED_SCHEMA).day("ts").build()); + } + + /** The number of live data files in {@code table}: the quantity the option exists to reduce. */ + private static int dataFileCount(Table table) throws IOException { + table.refresh(); + try (CloseableIterable tasks = table.newScan().planFiles()) { + return Lists.newArrayList(tasks).size(); + } + } + + /** + * The payoff, measured: 48 rows over 8 day partitions written twice at {@code num_shards = 8}. + * {@code shards_per_partition = 1} yields exactly one data file per day, the uncapped default + * several per day. Same rows, same partitions; only the file layout differs. + */ + @Test + public void shardsPerPartitionOneCollapsesFilesToThePartitionCount() throws IOException { + int numDays = 8; + int rowsPerDay = 6; + int numShards = 8; + + TableIdentifier affineId = createDayPartitionedTable("pas_files_affine"); + TableIdentifier pkId = createDayPartitionedTable("pas_files_pk"); + Table affine = catalog.loadTable(affineId); + Table pk = catalog.loadTable(pkId); + Schema inputSchema = inputSchemaFor(affine); + + List> rows = new ArrayList<>(); + int id = 0; + for (int day = 0; day < numDays; day++) { + Instant ts = Instant.parse("2024-03-01T12:00:00Z").plus(day, ChronoUnit.DAYS); + for (int i = 0; i < rowsPerDay; i++) { + id++; + rows.add(KV.of(INSERT, row(inputSchema, id, ts, "n" + id, 1L))); + } + } + + // One input, two sinks: the only difference between them is the sharding mode. + PCollection in = boundedInput(inputSchema, rows); + in.apply( + "WriteAffine", + IcebergIO.writeCdcRows(catalogConfig()) + .to(affineId) + .withSequenceNumberColumn("seq") + .withNumShards(numShards) + .withShardsPerPartition(1)); + in.apply( + "WritePk", + IcebergIO.writeCdcRows(catalogConfig()) + .to(pkId) + .withSequenceNumberColumn("seq") + .withNumShards(numShards)); + p.run().waitUntilFinish(); + + // Same rows, same partitions: the tables are indistinguishable to a reader. + assertThat(readColumns(affine, "id", "ts", "name"), hasSize(numDays * rowsPerDay)); + assertThat( + readColumns(affine, "id", "ts", "name"), equalTo(readColumns(pk, "id", "ts", "name"))); + assertThat(scannedPartitions(affine, 1), hasSize(numDays)); + assertThat(scannedPartitions(affine, 1), equalTo(scannedPartitions(pk, 1))); + + // Exactly one data file per partition, whatever num_shards is. + assertThat(dataFileCount(affine), equalTo(numDays)); + // Uncapped, several shards write each partition (36 files here, deterministically); the bound + // is deliberately loose so this asserts the scaling claim, not a hash fixture. + assertThat(dataFileCount(pk), greaterThanOrEqualTo(2 * numDays)); + } + + /** The {@code (id, ts)} primary-key schema exactly as {@code TableSetup} derives it. */ + private static Schema dayPartitionedPkSchema(Table table) { + Schema dataSchema = IcebergUtils.icebergSchemaToBeamSchema(table.schema()); + return Schema.builder() + .addField(dataSchema.getField("id")) + .addField(dataSchema.getField("ts")) + .build(); + } + + /** {@code pkBytes} for one {@code (id, ts)} key, byte-identical to the sink's own encoding. */ + private static byte[] encodePk(Schema pkSchema, int id, Instant ts) throws CoderException { + return CoderUtils.encodeToByteArray( + RowCoder.of(pkSchema), Row.withSchema(pkSchema).addValues(id, ts).build()); + } + + /** + * Picks 16 ids whose pk hashes cover all 12 {@code mod 12} residues plus all 4 {@code mod 4} + * residues once more: exact shard coverage at both {@code shards_per_partition = 4} and the + * uncapped default, no probabilistic spread. + */ + private static List idsCoveringAllShardResidues(Schema pkSchema, Instant ts, int firstId) + throws CoderException { + Map idByShardResidue = new TreeMap<>(); + Map extraIdByOffsetResidue = new TreeMap<>(); + for (int candidate = firstId; + idByShardResidue.size() < 12 || extraIdByOffsetResidue.size() < 4; + candidate++) { + if (candidate - firstId >= 10_000) { + throw new AssertionError("residue search did not converge; the PK encoding drifted"); + } + int hash = TableSetup.pkHash(encodePk(pkSchema, candidate, ts)); + if (idByShardResidue.putIfAbsent(Math.floorMod(hash, 12), candidate) != null) { + extraIdByOffsetResidue.putIfAbsent(Math.floorMod(hash, 4), candidate); + } + } + List ids = new ArrayList<>(idByShardResidue.values()); + ids.addAll(extraIdByOffsetResidue.values()); + return ids; + } + + /** + * The dial between the endpoints, measured: three day partitions at {@code num_shards = 12}, + * written at {@code shards_per_partition = 4} and uncapped. {@link #idsCoveringAllShardResidues} + * makes both counts exact (3 x 4 = 12 files dialed, 3 x 12 = 36 uncapped) and both tables must + * hold identical contents: the dial moves files, never rows. + */ + @Test + public void shardsPerPartitionDialsFileCountBetweenTheEndpoints() throws IOException { + int numShards = 12; + int shardsPerPartition = 4; + int numDays = 3; + + TableIdentifier dialId = createDayPartitionedTable("spp_dial"); + TableIdentifier uncappedId = createDayPartitionedTable("spp_dial_uncapped"); + Table dial = catalog.loadTable(dialId); + Table uncapped = catalog.loadTable(uncappedId); + Schema inputSchema = inputSchemaFor(dial); + Schema pkSchema = dayPartitionedPkSchema(dial); + + List> rows = new ArrayList<>(); + List expected = new ArrayList<>(); + for (int day = 0; day < numDays; day++) { + Instant ts = Instant.parse("2024-06-01T12:00:00Z").plus(day, ChronoUnit.DAYS); + for (int id : idsCoveringAllShardResidues(pkSchema, ts, 1000 * (day + 1))) { + rows.add(KV.of(INSERT, row(inputSchema, id, ts, "n" + id, 1L))); + expected.add(id + ":" + micros(ts) + ":n" + id); + } + } + + PCollection in = boundedInput(inputSchema, rows); + in.apply( + "WriteDial", + IcebergIO.writeCdcRows(catalogConfig()) + .to(dialId) + .withSequenceNumberColumn("seq") + .withNumShards(numShards) + .withShardsPerPartition(shardsPerPartition)); + in.apply( + "WriteUncapped", + IcebergIO.writeCdcRows(catalogConfig()) + .to(uncappedId) + .withSequenceNumberColumn("seq") + .withNumShards(numShards)); + p.run().waitUntilFinish(); + + // Identical, complete contents in identical partitions under both settings. + List expectedSorted = + expected.stream().sorted().collect(ImmutableList.toImmutableList()); + assertThat(readColumns(dial, "id", "ts", "name"), equalTo(expectedSorted)); + assertThat(readColumns(uncapped, "id", "ts", "name"), equalTo(expectedSorted)); + assertThat(scannedPartitions(dial, 1), hasSize(numDays)); + assertThat(scannedPartitions(dial, 1), equalTo(scannedPartitions(uncapped, 1))); + + // The measured dial: exactly spp shards per partition, against numShards shards uncapped. + assertThat(dataFileCount(dial), equalTo(numDays * shardsPerPartition)); + assertThat(dataFileCount(uncapped), equalTo(numDays * numShards)); + } + + /** + * End-to-end correctness at the tightest cap ({@code shards_per_partition = 1}) over cross-window + * updates and deletes: the final contents must be identical to the uncapped write of the same + * stream. + */ + @Test + public void shardsPerPartitionOneMatchesPrimaryKeyShardingEndToEnd() throws IOException { + TableIdentifier affineId = createDayPartitionedTable("pas_e2e_affine"); + TableIdentifier pkId = createDayPartitionedTable("pas_e2e_pk"); + Table affine = catalog.loadTable(affineId); + Table pk = catalog.loadTable(pkId); + Schema inputSchema = inputSchemaFor(affine); + + Instant march = Instant.parse("2024-03-15T10:30:00Z"); + Instant april = Instant.parse("2024-04-02T08:00:00Z"); + Instant may = Instant.parse("2024-05-20T23:00:00Z"); + + PCollection in = + streamingInput( + inputSchema, + testStream(inputSchema) + .advanceWatermarkTo(BASE) + // Window 1: four inserts across three day partitions. + .addElements( + at(INSERT, row(inputSchema, 1, march, "a", 1L), 1), + at(INSERT, row(inputSchema, 2, march, "b", 1L), 1), + at(INSERT, row(inputSchema, 3, april, "c", 1L), 2), + at(INSERT, row(inputSchema, 4, may, "d", 2L), 2)) + .advanceWatermarkTo(BASE.plus(WINDOW)) + // Window 2: update one March row, delete the other; delete the May row. + .addElements( + at(UPDATE_BEFORE, row(inputSchema, 1, march, "a", 3L), 61), + at(UPDATE_AFTER, row(inputSchema, 1, march, "a2", 3L), 61), + at(DELETE, row(inputSchema, 2, march, "b", 3L), 62), + at(DELETE, row(inputSchema, 4, may, "d", 4L), 62)) + .advanceWatermarkTo(BASE.plus(WINDOW).plus(WINDOW)) + // Window 3: update the April row and re-insert into the emptied May partition. + .addElements( + at(UPDATE_BEFORE, row(inputSchema, 3, april, "c", 5L), 121), + at(UPDATE_AFTER, row(inputSchema, 3, april, "c2", 5L), 121), + at(INSERT, row(inputSchema, 5, may, "e", 5L), 122)) + .advanceWatermarkToInfinity()); + + in.apply( + "WriteAffine", + IcebergIO.writeCdcRows(catalogConfig()) + .to(affineId) + .withSequenceNumberColumn("seq") + .withTriggeringFrequency(WINDOW) + .withShardsPerPartition(1)); + in.apply( + "WritePk", + IcebergIO.writeCdcRows(catalogConfig()) + .to(pkId) + .withSequenceNumberColumn("seq") + .withTriggeringFrequency(WINDOW)); + p.run().waitUntilFinish(); + + List expected = + ImmutableList.of( + "1:" + micros(march) + ":a2", "3:" + micros(april) + ":c2", "5:" + micros(may) + ":e"); + assertThat(readColumns(affine, "id", "ts", "name"), equalTo(expected)); + assertThat(readColumns(pk, "id", "ts", "name"), equalTo(expected)); + + // Rows physically landed in their day partitions under both modes. + assertThat(scannedPartitions(affine, 1), equalTo(scannedPartitions(pk, 1))); + assertThat( + scannedPartitions(affine, 1), + containsInAnyOrder( + String.valueOf((int) LocalDate.of(2024, 3, 15).toEpochDay()), + String.valueOf((int) LocalDate.of(2024, 4, 2).toEpochDay()), + String.valueOf((int) LocalDate.of(2024, 5, 20).toEpochDay()))); + } + + // ----------------------------------------------------------------------------------------------- + // 12. DECIMAL and BINARY partition values, through the full sink + // ----------------------------------------------------------------------------------------------- + + /** + * {@code identity(DECIMAL)} + {@code identity(BINARY)} partitions apply changes end to end: the + * types the path-rendered predecessor banned outright (base64 out, raw UTF-8 back in). The binary + * values are deliberately invalid UTF-8 with {@code 0x00} and {@code '/'}, so a byte-lossy + * round-trip anywhere changes the partition tuple and shows in {@link #scannedPartitions}. + */ + @Test + public void decimalAndBinaryPartitionValuesApplyChangesEndToEnd() throws IOException { + org.apache.iceberg.Schema icebergSchema = + new org.apache.iceberg.Schema( + Types.NestedField.required(1, "id", Types.IntegerType.get()), + Types.NestedField.required(2, "amount", Types.DecimalType.of(9, 3)), + Types.NestedField.required(3, "bin", Types.BinaryType.get()), + Types.NestedField.optional(4, "name", Types.StringType.get())); + TableIdentifier id = + createTable( + "decbinpart", + icebergSchema, + ImmutableSet.of(1, 2, 3), + 2, + PartitionSpec.builderFor(icebergSchema).identity("amount").identity("bin").build()); + Table t = catalog.loadTable(id); + Schema inputSchema = inputSchemaFor(t); + + BigDecimal negative = new BigDecimal("-12345.678"); + BigDecimal positive = new BigDecimal("0.001"); + byte[] gnarly = new byte[] {0x00, (byte) 0xFF, 0x2F, (byte) 0x80}; + byte[] plain = new byte[] {0x01, 0x02}; + + boundedInput( + inputSchema, + KV.of(INSERT, row(inputSchema, 1, negative, gnarly, "a", 1L)), + KV.of(INSERT, row(inputSchema, 2, positive, plain, "b", 1L)), + KV.of(INSERT, row(inputSchema, 3, negative, gnarly, "c", 1L)), + // Update and delete inside the gnarly-binary partition. + KV.of(UPDATE_BEFORE, row(inputSchema, 1, negative, gnarly, "a", 2L)), + KV.of(UPDATE_AFTER, row(inputSchema, 1, negative, gnarly, "a2", 2L)), + KV.of(DELETE, row(inputSchema, 3, negative, gnarly, "c", 2L))) + .apply(IcebergIO.writeCdcRows(catalogConfig()).to(id).withSequenceNumberColumn("seq")); + p.run().waitUntilFinish(); + + assertThat( + readColumns(t, "id", "amount", "name"), + containsInAnyOrder("1:" + negative + ":a2", "2:" + positive + ":b")); + // Both partition tuples survived byte-for-byte: the decimal keeps its scale and sign, and the + // binary keeps its 0x00 / 0xFF / '/' bytes. + assertThat( + scannedPartitions(t, 2), + containsInAnyOrder(negative + ":" + hex(gnarly), positive + ":" + hex(plain))); + } + + /** Renders a partition value that may be a {@link ByteBuffer} or {@code byte[]} as hex. */ + private static String hex(Object value) { + byte[] bytes; + if (value instanceof ByteBuffer) { + ByteBuffer view = ((ByteBuffer) value).duplicate(); + bytes = new byte[view.remaining()]; + view.get(bytes); + } else { + bytes = (byte[]) value; + } + StringBuilder sb = new StringBuilder(); + for (byte b : bytes) { + sb.append(String.format("%02x", b)); + } + return sb.toString(); + } + + // ----------------------------------------------------------------------------------------------- + // 13. Table CREATED with a sort order + // ----------------------------------------------------------------------------------------------- + + /** + * A cross-window DELETE commits against a table CREATED with a sort order. The sink's equality + * deletes carry sort-order id 0 (unsorted), and a table born sorted stores no id 0 at all; + * resolving the id straight out of {@code table.sortOrders()} threw here, and the committer's + * rethrow-before-state-write re-fired the identical window forever. Structurally case 3 on a + * sorted-on-create table; the order is on a NON-key column on purpose. + */ + @Test + public void crossWindowDeleteCommitsAgainstTableCreatedWithASortOrder() { + TableIdentifier tableId = TableIdentifier.of("db", "sortedcreate_" + System.nanoTime()); + CdcSinkTestUtils.createSortedTable( + catalog, + tableId, + ICEBERG_SCHEMA, + ImmutableSet.of(1), + 2, + PartitionSpec.unpartitioned(), + SortOrder.builderFor(ICEBERG_SCHEMA).asc("name").desc("data").build()); + Table t = catalog.loadTable(tableId); + + // The premise: this table has a sort order, and it does NOT have id 0. + assertThat(t.sortOrder().isSorted(), is(true)); + assertThat(t.sortOrders().keySet(), not(hasItem(SortOrder.unsorted().orderId()))); + + TestStream> stream = + testStream(INPUT_SCHEMA) + .advanceWatermarkTo(BASE) + .addElements( + at(INSERT, row(1, "a", "x", 1L), 1), + at(INSERT, row(2, "b", "y", 1L), 2), + at(INSERT, row(3, "c", "z", 1L), 3)) + .advanceWatermarkTo(BASE.plus(Duration.standardSeconds(70))) // window 0 commits + .addElements(at(DELETE, row(1, "a", "x", 2L), 61)) + .advanceWatermarkTo(BASE.plus(Duration.standardSeconds(130))) // window 1 commits + .advanceWatermarkToInfinity(); + + streamingInput(INPUT_SCHEMA, stream) + .apply( + IcebergIO.writeCdcRows(catalogConfig()) + .to(tableId) + .withSequenceNumberColumn("seq") + .withTriggeringFrequency(WINDOW)); + p.run().waitUntilFinish(); + + // The delete window really committed; a stalled destination leaves one snapshot, three rows. + assertThat(snapshotCount(t), equalTo(2)); + assertThat(readRows(t), containsInAnyOrder("2:b:y", "3:c:z")); + + // The equality delete landed under the unsorted order, not silently relabelled as the table's. + List deletes = addedDeleteFiles(t); + assertThat(deletes, not(empty())); + for (DeleteFile delete : deletes) { + assertThat(delete.content(), equalTo(FileContent.EQUALITY_DELETES)); + assertThat(delete.sortOrderId(), equalTo(SortOrder.unsorted().orderId())); + } + } + + // ----------------------------------------------------------------------------------------------- + // 14. Mid-run partition-spec evolution: the pinned worker keeps writing, correctly. + // ----------------------------------------------------------------------------------------------- + + /** The {@code id}-only primary-key schema of the {@link #DATA_SCHEMA} fixture tables. */ + private static final Schema ID_PK_SCHEMA = + Schema.builder().addField(DATA_SCHEMA.getField("id")).build(); + + /** + * A sorted-group element for driving {@link WriteDeltas.WriteDeltasFn} directly. The collapse + * writer blocks on the sort key's pk prefix, so it carries the row's encoded {@code id}, the + * fixture tables' primary key. + */ + private static KV cdc(Row data, long seq, ValueKind kind) + throws CoderException { + Row pk = Row.withSchema(ID_PK_SCHEMA).addValues(data.getInt32("id")).build(); + byte[] pkBytes = CoderUtils.encodeToByteArray(RowCoder.of(ID_PK_SCHEMA), pk); + return KV.of(CdcSortKey.encode(pkBytes, seq, kind), CdcRecord.of(data, kind, seq)); + } + + /** Sums the named {@link CommitDeltas} counter committed by the pipeline. */ + private static long committerCounter(PipelineResult result, String name) { + long total = 0; + for (MetricResult c : + result + .metrics() + .queryMetrics( + MetricsFilter.builder() + .addNameFilter(MetricNameFilter.named(CommitDeltas.class, name)) + .build()) + .getCounters()) { + total += c.getCommitted(); + } + return total; + } + + /** + * A worker that resolved before a mid-run spec evolution keeps writing under the pinned spec, + * equality deletes included, so they reach the earlier window's data. The write stage runs at the + * {@link WriteDeltas.WriteDeltasFn} level so both windows provably share one worker instance (the + * pin is per worker); the commit stage and read-back are real. + */ + @Test + public void evolvedSpecKeepsWritingUnderPinnedSpec() throws IOException { + TableIdentifier id = + createTable( + "pinnedspec", + ICEBERG_SCHEMA, + ImmutableSet.of(1), + 2, + PartitionSpec.builderFor(ICEBERG_SCHEMA).bucket("id", 4).build()); + Table t = catalog.loadTable(id); + int pinnedSpecId = t.spec().specId(); + + CdcWriteConfig config = CdcWriteConfig.builder().setSinkId("pin-sink").build(); + TableSetup setup = + new TableSetup( + catalogConfig(), config, DynamicDestinations.singleTable(id, DATA_SCHEMA), "px"); + WriteDeltas.WriteDeltasFn fn = new WriteDeltas.WriteDeltasFn(setup, config, "px", DATA_SCHEMA); + + // Window 1: three inserts, resolved and written under the original spec. + List w1 = new ArrayList<>(); + fn.process( + KV.of( + DestinationShard.of(id.toString(), 0), + ImmutableList.of( + cdc(row(DATA_SCHEMA, 1, "a", "x"), 1L, INSERT), + cdc(row(DATA_SCHEMA, 2, "b", "y"), 1L, INSERT), + cdc(row(DATA_SCHEMA, 3, "c", "z"), 1L, INSERT))), + GlobalWindow.INSTANCE, + CdcSinkTestUtils.collectInto(w1)); + + // The operator evolves the partition spec; the worker's shared Table instance sees it. + TableSetup.Dest dest = setup.get(id.toString(), DATA_SCHEMA); + dest.table().updateSpec().addField(Expressions.bucket("id", 8)).commit(); + dest.table().refresh(); + assertThat(dest.table().spec().specId(), not(equalTo(pinnedSpecId))); + + // Window 2 through the SAME worker instance: update id=1, delete id=2. + List w2 = new ArrayList<>(); + fn.process( + KV.of( + DestinationShard.of(id.toString(), 0), + ImmutableList.of( + cdc(row(DATA_SCHEMA, 1, "a", "x"), 2L, UPDATE_BEFORE), + cdc(row(DATA_SCHEMA, 1, "a2", "x2"), 2L, UPDATE_AFTER), + cdc(row(DATA_SCHEMA, 2, "b", "y"), 3L, DELETE))), + GlobalWindow.INSTANCE, + CdcSinkTestUtils.collectInto(w2)); + assertThat(w1, hasSize(1)); + assertThat(w2, hasSize(1)); + + // Commit both windows, in order, through the real committer. + TestStream.Builder stream = + TestStream.create(ShardDeltaFiles.coder()).advanceWatermarkTo(BASE); + stream = + stream.addElements(TimestampedValue.of(w1.get(0), BASE.plus(Duration.standardSeconds(1)))); + stream = + stream.addElements(TimestampedValue.of(w2.get(0), BASE.plus(Duration.standardSeconds(61)))); + PipelineResult result = + p.apply(stream.advanceWatermarkToInfinity()) + .apply(Window.into(FixedWindows.of(WINDOW))) + .apply(new CommitDeltas(catalogConfig(), "pin-sink")) + .getPipeline() + .run(); + result.waitUntilFinish(); + + // Two windows committed; every committed file carries the pinned original spec id. + assertThat(snapshotCount(t), equalTo(2)); + for (Snapshot snap : t.snapshots()) { + for (DataFile file : snap.addedDataFiles(t.io())) { + assertThat(file.specId(), equalTo(pinnedSpecId)); + } + for (DeleteFile file : snap.addedDeleteFiles(t.io())) { + assertThat(file.specId(), equalTo(pinnedSpecId)); + } + } + // The pinned-spec equality deletes applied to the pinned-spec data: exact final contents. + assertThat(readRows(t), containsInAnyOrder("1:a2:x2", "3:c:z")); + assertThat(committerCounter(result, "specMismatchedWindows"), equalTo(0L)); + } + + // ----------------------------------------------------------------------------------------------- + // 15. A worker joining mid-run after a spec evolution adopts the run's stamped spec. + // ----------------------------------------------------------------------------------------------- + + /** + * A worker that first resolves AFTER a mid-run spec evolution (a fresh {@link TableSetup} under + * the same runId) adopts the spec the first commit stamped, so its files still carry the original + * spec id and its deletes reach the first pass's data. Two pipelines because the stamp must be in + * the table before the joining worker resolves. + */ + @Test + public void joiningWorkerScenarioStaysConsistent() throws IOException { + TableIdentifier id = + createTable( + "joiningworker", + ICEBERG_SCHEMA, + ImmutableSet.of(1), + 2, + PartitionSpec.builderFor(ICEBERG_SCHEMA).bucket("id", 4).build()); + Table t = catalog.loadTable(id); + int stampedSpecId = t.spec().specId(); + String runId = "join-runId"; + CdcWriteConfig config = CdcWriteConfig.builder().setSinkId("join-sink").build(); + + // Pass 1: an original worker writes window 1 under the initial spec ... + TableSetup setup1 = + new TableSetup( + catalogConfig(), config, DynamicDestinations.singleTable(id, DATA_SCHEMA), runId); + WriteDeltas.WriteDeltasFn fn1 = + new WriteDeltas.WriteDeltasFn(setup1, config, runId, DATA_SCHEMA); + List w1 = new ArrayList<>(); + fn1.process( + KV.of( + DestinationShard.of(id.toString(), 0), + ImmutableList.of( + cdc(row(DATA_SCHEMA, 1, "a", "x"), 1L, INSERT), + cdc(row(DATA_SCHEMA, 2, "b", "y"), 1L, INSERT), + cdc(row(DATA_SCHEMA, 3, "c", "z"), 1L, INSERT))), + GlobalWindow.INSTANCE, + CdcSinkTestUtils.collectInto(w1)); + + // ... and the committer commits it, stamping the run's pinned spec into the summary. + PipelineResult first = + p.apply( + TestStream.create(ShardDeltaFiles.coder()) + .advanceWatermarkTo(BASE) + .addElements( + TimestampedValue.of(w1.get(0), BASE.plus(Duration.standardSeconds(1)))) + .advanceWatermarkToInfinity()) + .apply(Window.into(FixedWindows.of(WINDOW))) + .apply(new CommitDeltas(catalogConfig(), "join-sink", null, null, runId)) + .getPipeline() + .run(); + first.waitUntilFinish(); + + // The spec evolves after the stamp landed; the process-shared Table sees it. + Table shared = setup1.get(id.toString(), DATA_SCHEMA).table(); + shared.updateSpec().addField(Expressions.bucket("id", 8)).commit(); + shared.refresh(); + assertThat(shared.spec().specId(), not(equalTo(stampedSpecId))); + + // Pass 2: the joining worker resolves the STAMPED spec and writes window 2 under it: + // an update and a delete against window 1's rows. + TableSetup setup2 = + new TableSetup( + catalogConfig(), config, DynamicDestinations.singleTable(id, DATA_SCHEMA), runId); + WriteDeltas.WriteDeltasFn fn2 = + new WriteDeltas.WriteDeltasFn(setup2, config, runId, DATA_SCHEMA); + List w2 = new ArrayList<>(); + fn2.process( + KV.of( + DestinationShard.of(id.toString(), 0), + ImmutableList.of( + cdc(row(DATA_SCHEMA, 1, "a", "x"), 2L, UPDATE_BEFORE), + cdc(row(DATA_SCHEMA, 1, "a2", "x2"), 2L, UPDATE_AFTER), + cdc(row(DATA_SCHEMA, 2, "b", "y"), 3L, DELETE))), + GlobalWindow.INSTANCE, + CdcSinkTestUtils.collectInto(w2)); + assertThat(setup2.get(id.toString(), DATA_SCHEMA).spec().specId(), equalTo(stampedSpecId)); + + TestPipeline second = TestPipeline.create(); + second.enableAbandonedNodeEnforcement(false); + PipelineResult secondResult = + second + .apply( + TestStream.create(ShardDeltaFiles.coder()) + .advanceWatermarkTo(BASE) + .addElements( + TimestampedValue.of(w2.get(0), BASE.plus(Duration.standardSeconds(61)))) + .advanceWatermarkToInfinity()) + .apply(Window.into(FixedWindows.of(WINDOW))) + .apply(new CommitDeltas(catalogConfig(), "join-sink", null, null, runId)) + .getPipeline() + .run(); + secondResult.waitUntilFinish(); + + // Every committed file of both passes carries the stamped spec; no mismatch was flagged. + t.refresh(); + assertThat(snapshotCount(t), equalTo(2)); + for (Snapshot snap : t.snapshots()) { + for (DataFile file : snap.addedDataFiles(t.io())) { + assertThat(file.specId(), equalTo(stampedSpecId)); + } + for (DeleteFile file : snap.addedDeleteFiles(t.io())) { + assertThat(file.specId(), equalTo(stampedSpecId)); + } + } + assertThat(readRows(t), containsInAnyOrder("1:a2:x2", "3:c:z")); + assertThat( + committerCounter(first, "specMismatchedWindows") + + committerCounter(secondResult, "specMismatchedWindows"), + equalTo(0L)); + } + + // ----------------------------------------------------------------------------------------------- + // 16. Non-key partitioning: day(ts) with only id as the key + // ----------------------------------------------------------------------------------------------- + + /** {@link #DAY_PARTITIONED_SCHEMA} with {@code day(ts)} partitioning but ONLY {@code id} key. */ + private TableIdentifier createNonKeyDayPartitionedTable(String prefix) { + return createTable( + prefix, + DAY_PARTITIONED_SCHEMA, + ImmutableSet.of(1), + 2, + PartitionSpec.builderFor(DAY_PARTITIONED_SCHEMA).day("ts").build()); + } + + private static final Instant MARCH = Instant.parse("2024-03-15T10:30:00Z"); + private static final Instant JUNE = Instant.parse("2024-06-01T09:00:00Z"); + private static final int MARCH_DAY = (int) LocalDate.of(2024, 3, 15).toEpochDay(); + private static final int JUNE_DAY = (int) LocalDate.of(2024, 6, 1).toEpochDay(); + + /** + * On a {@code day(ts)}-partitioned table whose key is {@code id} alone, an update that moves a + * row across partitions within one window deletes it from the partition it actually occupied: a + * delete routed by the after-image would land in the new partition and leave the old row alive + * beside the new one. + */ + @Test + public void nonKeyPartitionedMoveDeletesTheRowFromItsOldPartition() throws IOException { + TableIdentifier id = createNonKeyDayPartitionedTable("nonkeymove"); + Table t = catalog.loadTable(id); + Schema inputSchema = inputSchemaFor(t); + + TestStream> stream = + testStream(inputSchema) + .advanceWatermarkTo(BASE) + .addElements( + at(INSERT, row(inputSchema, 1, MARCH, "a", 1L), 1), + at(INSERT, row(inputSchema, 2, MARCH, "b", 1L), 2)) + .advanceWatermarkTo(BASE.plus(WINDOW)) // window 1 commits both March rows + .addElements( + at(UPDATE_BEFORE, row(inputSchema, 1, MARCH, "a", 2L), 61), + at(UPDATE_AFTER, row(inputSchema, 1, JUNE, "a2", 2L), 61)) + .advanceWatermarkToInfinity(); + + streamingInput(inputSchema, stream) + .apply( + IcebergIO.writeCdcRows(catalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withTriggeringFrequency(WINDOW)); + p.run().waitUntilFinish(); + + assertThat(snapshotCount(t), equalTo(2)); + assertThat( + readColumns(t, "id", "ts", "name"), + containsInAnyOrder("1:" + micros(JUNE) + ":a2", "2:" + micros(MARCH) + ":b")); + + // Physical corroboration: March still holds id=2's live file, June holds the moved row, and + // window 2's only delete file sits in MARCH, where the old row lives. + assertThat( + scannedPartitions(t, 1), + containsInAnyOrder(String.valueOf(MARCH_DAY), String.valueOf(JUNE_DAY))); + List deletes = addedDeleteFiles(t); + assertThat(deletes, hasSize(1)); + assertThat(deletes.get(0).content(), equalTo(FileContent.EQUALITY_DELETES)); + assertThat(deletes.get(0).partition().get(0, Integer.class), equalTo(MARCH_DAY)); + } + + /** + * A move whose {@code UPDATE_BEFORE} and {@code UPDATE_AFTER} land in DIFFERENT commit windows + * still converges: the bare-UB window commits the equality delete in the old partition, the + * bare-UA window commits the row in the new one. This is why a bare-UA block must write without + * being rejected: its delete can already sit in an earlier commit. + */ + @Test + public void nonKeyPartitionedMoveSplitAcrossWindowsConverges() { + TableIdentifier id = createNonKeyDayPartitionedTable("nonkeysplit"); + Table t = catalog.loadTable(id); + Schema inputSchema = inputSchemaFor(t); + + TestStream> stream = + testStream(inputSchema) + .advanceWatermarkTo(BASE) + .addElements(at(INSERT, row(inputSchema, 1, MARCH, "a", 1L), 1)) + .advanceWatermarkTo(BASE.plus(WINDOW)) // window 1 commits the March row + .addElements(at(UPDATE_BEFORE, row(inputSchema, 1, MARCH, "a", 2L), 61)) + .advanceWatermarkTo(BASE.plus(WINDOW).plus(WINDOW)) // window 2: the bare UB + .addElements(at(UPDATE_AFTER, row(inputSchema, 1, JUNE, "a2", 2L), 121)) + .advanceWatermarkToInfinity(); // window 3: the bare UA + + streamingInput(inputSchema, stream) + .apply( + IcebergIO.writeCdcRows(catalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withTriggeringFrequency(WINDOW)); + p.run().waitUntilFinish(); + + t.refresh(); + assertThat(readColumns(t, "id", "ts", "name"), contains("1:" + micros(JUNE) + ":a2")); + + // The per-commit file trail: data at March; the bare UB's delete at March; data at June. + List snapshots = + Lists.newArrayList(t.snapshots()).stream() + .sorted(Comparator.comparingLong(Snapshot::sequenceNumber)) + .collect(Collectors.toList()); + assertThat(snapshots, hasSize(3)); + + List w1Data = Lists.newArrayList(snapshots.get(0).addedDataFiles(t.io())); + assertThat(w1Data, hasSize(1)); + assertThat(w1Data.get(0).partition().get(0, Integer.class), equalTo(MARCH_DAY)); + + assertThat(Lists.newArrayList(snapshots.get(1).addedDataFiles(t.io())), empty()); + List w2Deletes = Lists.newArrayList(snapshots.get(1).addedDeleteFiles(t.io())); + assertThat(w2Deletes, hasSize(1)); + assertThat(w2Deletes.get(0).content(), equalTo(FileContent.EQUALITY_DELETES)); + assertThat(w2Deletes.get(0).partition().get(0, Integer.class), equalTo(MARCH_DAY)); + + assertThat(Lists.newArrayList(snapshots.get(2).addedDeleteFiles(t.io())), empty()); + List w3Data = Lists.newArrayList(snapshots.get(2).addedDataFiles(t.io())); + assertThat(w3Data, hasSize(1)); + assertThat(w3Data.get(0).partition().get(0, Integer.class), equalTo(JUNE_DAY)); + } + + /** + * The bare-UA outcome on a non-key-partitioned table, pinned as intended behavior: without its + * {@code UPDATE_BEFORE} (the input contract for such tables, see the package-info partitioning + * section), a moved row cannot be deleted from the partition it occupied, so the old image stays + * live beside the new one and no delete file is written. + */ + @Test + public void nonKeyPartitionedBareUpdateAfterLeavesTheOldRowLive() throws IOException { + TableIdentifier id = createNonKeyDayPartitionedTable("nonkeybareua"); + Table t = catalog.loadTable(id); + Schema inputSchema = inputSchemaFor(t); + + TestStream> stream = + testStream(inputSchema) + .advanceWatermarkTo(BASE) + .addElements(at(INSERT, row(inputSchema, 1, MARCH, "a", 1L), 1)) + .advanceWatermarkTo(BASE.plus(WINDOW)) // window 1 commits the March row + .addElements(at(UPDATE_AFTER, row(inputSchema, 1, JUNE, "a2", 2L), 61)) + .advanceWatermarkToInfinity(); // window 2: the bare UA + + streamingInput(inputSchema, stream) + .apply( + IcebergIO.writeCdcRows(catalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withTriggeringFrequency(WINDOW)); + p.run().waitUntilFinish(); + + assertThat( + readColumns(t, "id", "ts", "name"), + containsInAnyOrder("1:" + micros(MARCH) + ":a", "1:" + micros(JUNE) + ":a2")); + assertThat( + scannedPartitions(t, 1), + containsInAnyOrder(String.valueOf(MARCH_DAY), String.valueOf(JUNE_DAY))); + assertThat(addedDeleteFiles(t), empty()); + } + + // ----------------------------------------------------------------------------------------------- + // 17. One runId end to end + // ----------------------------------------------------------------------------------------------- + + /** + * The committed run-spec stamp, the writer's file names, and spec adoption agree on one runId: + * the stamp's runId prefix is the runId embedded in the committed data file names, and a {@link + * TableSetup} built with it adopts the stamped spec over a live evolution. Fails if {@code + * WriteCdcRows.expand()} hands its stages different runIds. + */ + @Test + public void runIdThreadsIdenticallyThroughTheAssembledTransform() { + TableIdentifier id = + createTable( + "runid", + ICEBERG_SCHEMA, + ImmutableSet.of(1), + 2, + PartitionSpec.builderFor(ICEBERG_SCHEMA).bucket("id", 4).build()); + Table t = catalog.loadTable(id); + + boundedInput(INPUT_SCHEMA, KV.of(INSERT, row(1, "a", "x", 1L))) + .apply( + IcebergIO.writeCdcRows(catalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withSinkId("runid-sink")); + p.run().waitUntilFinish(); + + t.refresh(); + Snapshot committed = checkStateNotNull(t.currentSnapshot()); + String stamp = checkStateNotNull(committed.summary()).get("beam.cdc.run-spec.runid-sink"); + assertThat(stamp, notNullValue()); + int cut = stamp.lastIndexOf(':'); + String runId = stamp.substring(0, cut); + int stampedSpecId = Integer.parseInt(stamp.substring(cut + 1)); + + // The commit stage's stamped runId is the writer stage's runId, part of every file name. + List dataFiles = Lists.newArrayList(committed.addedDataFiles(t.io())); + assertThat(dataFiles, not(empty())); + for (DataFile file : dataFiles) { + assertThat(file.location(), containsString(runId)); + } + + // A TableSetup built with the same runId (the assign/write stages' construction) adopts the + // stamped spec over the live evolved one. + t.updateSpec().addField(Expressions.bucket("id", 8)).commit(); + t.refresh(); + assertThat(t.spec().specId(), not(equalTo(stampedSpecId))); + TableSetup joining = + new TableSetup( + catalogConfig(), + CdcWriteConfig.builder().setSinkId("runid-sink").setSequenceNumberColumn("seq").build(), + DynamicDestinations.singleTable(id, DATA_SCHEMA), + runId); + assertThat(joining.get(id.toString(), DATA_SCHEMA).spec().specId(), equalTo(stampedSpecId)); + } +} diff --git a/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRowsTest.java b/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRowsTest.java new file mode 100644 index 000000000000..1e3a2a03d121 --- /dev/null +++ b/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRowsTest.java @@ -0,0 +1,1499 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.iceberg.cdc.sink; + +import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull; +import static org.apache.beam.sdk.values.ValueKind.DELETE; +import static org.apache.beam.sdk.values.ValueKind.INSERT; +import static org.apache.beam.sdk.values.ValueKind.UPDATE_AFTER; +import static org.apache.beam.sdk.values.ValueKind.UPDATE_BEFORE; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.lessThan; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; + +import java.util.ArrayList; +import java.util.List; +import org.apache.beam.sdk.Pipeline; +import org.apache.beam.sdk.PipelineResult; +import org.apache.beam.sdk.coders.CoderException; +import org.apache.beam.sdk.coders.KvCoder; +import org.apache.beam.sdk.coders.RowCoder; +import org.apache.beam.sdk.coders.SerializableCoder; +import org.apache.beam.sdk.io.iceberg.IcebergCatalogConfig; +import org.apache.beam.sdk.io.iceberg.IcebergIO; +import org.apache.beam.sdk.io.iceberg.IcebergWriteResult; +import org.apache.beam.sdk.metrics.MetricNameFilter; +import org.apache.beam.sdk.metrics.MetricResult; +import org.apache.beam.sdk.metrics.MetricsFilter; +import org.apache.beam.sdk.schemas.Schema; +import org.apache.beam.sdk.testing.ExpectedLogs; +import org.apache.beam.sdk.testing.PAssert; +import org.apache.beam.sdk.testing.TestPipeline; +import org.apache.beam.sdk.testing.TestStream; +import org.apache.beam.sdk.transforms.Create; +import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.transforms.MapElements; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.transforms.windowing.AfterPane; +import org.apache.beam.sdk.transforms.windowing.GlobalWindows; +import org.apache.beam.sdk.transforms.windowing.Repeatedly; +import org.apache.beam.sdk.transforms.windowing.Window; +import org.apache.beam.sdk.util.CoderUtils; +import org.apache.beam.sdk.values.KV; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.Row; +import org.apache.beam.sdk.values.TimestampedValue; +import org.apache.beam.sdk.values.TypeDescriptor; +import org.apache.beam.sdk.values.TypeDescriptors; +import org.apache.beam.sdk.values.ValueKind; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.Table; +import org.apache.iceberg.catalog.Catalog; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.data.IcebergGenerics; +import org.apache.iceberg.data.Record; +import org.apache.iceberg.hadoop.HadoopCatalog; +import org.apache.iceberg.types.Types; +import org.joda.time.Duration; +import org.joda.time.Instant; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Tests for {@link WriteCdcRows}, the top-level CDC sink {@code PTransform} exposed via {@link + * IcebergIO#writeCdcRows}. End-to-end tests drive a real {@link HadoopCatalog} V2 table on the + * DirectRunner; each test uses a unique table identifier so the process-wide TableCache never sees + * a repeat. + */ +@RunWith(JUnit4.class) +public class WriteCdcRowsTest { + + @Rule public transient TestPipeline p = TestPipeline.create(); + @Rule public transient TemporaryFolder tmp = new TemporaryFolder(); + @Rule public transient ExpectedLogs expectedLogs = ExpectedLogs.none(WriteCdcRows.class); + + /** Canonical test table schema. */ + private static final org.apache.iceberg.Schema ICEBERG_SCHEMA = + new org.apache.iceberg.Schema( + Types.NestedField.required(1, "id", Types.IntegerType.get()), + Types.NestedField.optional(2, "name", Types.StringType.get()), + Types.NestedField.optional(3, "data", Types.StringType.get())); + + private static final Schema DATA_SCHEMA = + Schema.builder() + .addInt32Field("id") + .addNullableField("name", Schema.FieldType.STRING) + .addNullableField("data", Schema.FieldType.STRING) + .build(); + + /** Input schema = data schema + a sequence-number column named {@code seq}. */ + private static final Schema INPUT_SCHEMA = + Schema.builder().addFields(DATA_SCHEMA.getFields()).addInt64Field("seq").build(); + + /** The default sequence-number column name (see {@code CdcWriteConfig}). */ + private static final String DEFAULT_SEQ_COL = "_commit_snapshot_sequence_number"; + + /** Replayed dead letters are re-stamped here so the replay lands in an open window. */ + private static final Instant REPLAY_EVENT_TIME = new Instant(0).plus(Duration.standardMinutes(2)); + + private Catalog catalog; + + private void setUpCatalog() { + catalog = CdcSinkTestUtils.hadoopCatalog(tmp.getRoot()); + } + + private IcebergCatalogConfig catalogConfig() { + return CdcSinkTestUtils.catalogConfig(tmp.getRoot()); + } + + /** A catalog config pointing at an unreachable warehouse (for no-catalog-at-expand tests). */ + private static IcebergCatalogConfig bogusCatalogConfig() { + return IcebergCatalogConfig.builder() + .setCatalogProperties( + ImmutableMap.of( + "type", "hadoop", "warehouse", "file:/nonexistent-cdc-" + System.nanoTime())) + .build(); + } + + /** Creates a fresh unpartitioned V2 table (PK = {@code id}) with a unique identifier. */ + private TableIdentifier v2Table() { + TableIdentifier id = TableIdentifier.of("db", "t" + System.nanoTime()); + createV2Table(id); + return id; + } + + /** Creates a fresh unpartitioned V2 table with the given identifier (PK = {@code id}). */ + private void createV2Table(TableIdentifier id) { + CdcSinkTestUtils.createTable( + catalog, id, ICEBERG_SCHEMA, ImmutableSet.of(1), 2, PartitionSpec.unpartitioned()); + } + + /** Builds an input data+seq {@link Row}. */ + private static Row row(int id, String name, String data, long seq) { + return Row.withSchema(INPUT_SCHEMA).addValues(id, name, data, seq).build(); + } + + /** Builds an input data-only {@link Row} (no {@code seq} column). */ + private static Row rowNoSeq(int id, String name, String data) { + return Row.withSchema(DATA_SCHEMA).addValues(id, name, data).build(); + } + + /** Reads all live rows of {@code table} as sorted {@code id:name:data} strings. */ + private static List readRows(Table table) { + table.refresh(); + return ImmutableList.copyOf(IcebergGenerics.read(table).build()).stream() + .map(r -> r.getField("id") + ":" + r.getField("name") + ":" + r.getField("data")) + .sorted() + .collect(ImmutableList.toImmutableList()); + } + + /** A bounded CDC input of the given kind-tagged rows over {@code schema}. */ + @SafeVarargs + private final PCollection boundedInput(Schema schema, KV... rows) { + return CdcSinkTestUtils.withKinds(p.apply(Create.of(ImmutableList.copyOf(rows)))) + .setRowSchema(schema); + } + + /** An unbounded single-INSERT CDC input over {@link #INPUT_SCHEMA}. */ + private PCollection unboundedInput() { + TestStream stream = + TestStream.create(RowCoder.of(INPUT_SCHEMA)) + .addElements(row(1, "a", "x", 1L)) + .advanceWatermarkToInfinity(); + PCollection> tagged = + p.apply(stream) + .setRowSchema(INPUT_SCHEMA) + .apply( + MapElements.into( + TypeDescriptors.kvs( + TypeDescriptor.of(ValueKind.class), TypeDescriptors.rows())) + .via(r -> KV.of(INSERT, r))) + .setCoder(KvCoder.of(SerializableCoder.of(ValueKind.class), RowCoder.of(INPUT_SCHEMA))); + return CdcSinkTestUtils.withKinds(tagged).setRowSchema(INPUT_SCHEMA); + } + + /** + * The {@code beam.cdc.committed-through-ms.} token of each of {@code table}'s snapshots, + * in snapshot order. + */ + private static List committedThroughTokens(Table table, String sinkId) { + table.refresh(); + List tokens = new ArrayList<>(); + for (Snapshot s : table.snapshots()) { + tokens.add(Long.parseLong(s.summary().get("beam.cdc.committed-through-ms." + sinkId))); + } + return tokens; + } + + /** Sums the named {@link CommitDeltas} counter committed by the pipeline. */ + private static long committerCounter(PipelineResult result, String name) { + long total = 0; + for (MetricResult c : + result + .metrics() + .queryMetrics( + MetricsFilter.builder() + .addNameFilter(MetricNameFilter.named(CommitDeltas.class, name)) + .build()) + .getCounters()) { + total += c.getCommitted(); + } + return total; + } + + // --------------------------------------------------------------------------------------------- + // 1. Construction-time validation matrix + // --------------------------------------------------------------------------------------------- + + /** Exactly one destination: none at all and both kinds at once are each rejected. */ + @Test + public void constructionRequiresExactlyOneDestination() { + p.enableAbandonedNodeEnforcement(false); + TableIdentifier id = TableIdentifier.of("db", "both_" + System.nanoTime()); + PCollection in = boundedInput(INPUT_SCHEMA, KV.of(INSERT, row(1, "a", "x", 1L))); + + // facet: no destination. + IllegalArgumentException none = + assertThrows( + IllegalArgumentException.class, + () -> + in.apply( + "NoDestination", + IcebergIO.writeCdcRows(bogusCatalogConfig()).withSequenceNumberColumn("seq"))); + assertThat(none.getMessage(), containsString("exactly one")); + + // facet: both destination kinds. + IllegalArgumentException both = + assertThrows( + IllegalArgumentException.class, + () -> + in.apply( + "BothDestinations", + IcebergIO.writeCdcRows(bogusCatalogConfig()) + .to(id) + .to( + CdcSinkTestUtils.templatedDestinations( + "db.{name}", INPUT_SCHEMA, "seq")) + .withSequenceNumberColumn("seq"))); + assertThat(both.getMessage(), containsString("exactly one")); + } + + /** An unbounded input requires a triggering frequency, and it must be positive. */ + @Test + public void unboundedInputRequiresPositiveTriggeringFrequency() { + p.enableAbandonedNodeEnforcement(false); + TableIdentifier id = TableIdentifier.of("db", "stream_tf_" + System.nanoTime()); + PCollection in = unboundedInput(); + + IllegalArgumentException missing = + assertThrows( + IllegalArgumentException.class, + () -> + in.apply( + "NoFrequency", + IcebergIO.writeCdcRows(bogusCatalogConfig()) + .to(id) + .withSequenceNumberColumn("seq"))); + assertThat(missing.getMessage(), containsString("withTriggeringFrequency")); + IllegalArgumentException zero = + assertThrows( + IllegalArgumentException.class, + () -> + in.apply( + "ZeroFrequency", + IcebergIO.writeCdcRows(bogusCatalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withTriggeringFrequency(Duration.ZERO))); + assertThat(zero.getMessage(), containsString("must be positive")); + IllegalArgumentException negative = + assertThrows( + IllegalArgumentException.class, + () -> + in.apply( + "NegativeFrequency", + IcebergIO.writeCdcRows(bogusCatalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withTriggeringFrequency(Duration.standardSeconds(-30)))); + assertThat(negative.getMessage(), containsString("must be positive")); + } + + /** + * Dead-letter metadata is nested under {@code record}, so data columns named like it ({@code + * _cdc_change_type}, {@code change_type}) are ordinary columns and write end-to-end. + */ + @Test + public void deadLetterMetadataNamedDataColumnsWriteFine() { + setUpCatalog(); + TableIdentifier id = TableIdentifier.of("db", "dlname_" + System.nanoTime()); + CdcSinkTestUtils.createTable( + catalog, + id, + new org.apache.iceberg.Schema( + Types.NestedField.required(1, "id", Types.IntegerType.get()), + Types.NestedField.optional(2, "_cdc_change_type", Types.StringType.get()), + Types.NestedField.optional(3, "change_type", Types.StringType.get())), + ImmutableSet.of(1), + 2, + PartitionSpec.unpartitioned()); + + Schema inputSchema = + Schema.builder() + .addInt32Field("id") + .addNullableField("_cdc_change_type", Schema.FieldType.STRING) + .addNullableField("change_type", Schema.FieldType.STRING) + .addInt64Field("seq") + .build(); + Row r = Row.withSchema(inputSchema).addValues(1, "c", "u", 1L).build(); + boundedInput(inputSchema, KV.of(INSERT, r)) + .apply(IcebergIO.writeCdcRows(catalogConfig()).to(id).withSequenceNumberColumn("seq")); + p.run().waitUntilFinish(); + + Table t = catalog.loadTable(id); + Record written = + Iterables.getOnlyElement(ImmutableList.copyOf(IcebergGenerics.read(t).build())); + assertThat(written.getField("id"), equalTo(1)); + assertThat(written.getField("_cdc_change_type"), equalTo("c")); + assertThat(written.getField("change_type"), equalTo("u")); + } + + /** The sequence-number column must exist and be a non-nullable INT64. */ + @Test + public void constructionRejectsInvalidSequenceColumnConfigs() { + p.enableAbandonedNodeEnforcement(false); + TableIdentifier id = TableIdentifier.of("db", "badseq_" + System.nanoTime()); + + // facet: missing (the default name is absent from the input schema). + PCollection noSeq = boundedInput(DATA_SCHEMA, KV.of(INSERT, rowNoSeq(1, "a", "x"))); + IllegalArgumentException missing = + assertThrows( + IllegalArgumentException.class, + () -> noSeq.apply("MissingSeq", IcebergIO.writeCdcRows(bogusCatalogConfig()).to(id))); + assertThat(missing.getMessage(), containsString(DEFAULT_SEQ_COL)); + assertThat(missing.getMessage(), containsString("sequence-number column")); + + // facet: present but STRING. + Schema stringSeq = + Schema.builder().addFields(DATA_SCHEMA.getFields()).addStringField("seq").build(); + PCollection badSeq = + boundedInput( + stringSeq, + KV.of( + INSERT, Row.withSchema(stringSeq).addValues(1, "a", "x", "not-a-number").build())); + IllegalArgumentException nonInt64 = + assertThrows( + IllegalArgumentException.class, + () -> + badSeq.apply( + "StringSeq", + IcebergIO.writeCdcRows(bogusCatalogConfig()) + .to(id) + .withSequenceNumberColumn("seq"))); + assertThat(nonInt64.getMessage(), containsString("sequence-number column 'seq'")); + assertThat(nonInt64.getMessage(), containsString("must be INT64")); + assertThat(nonInt64.getMessage(), containsString("STRING")); + + // facet: INT64 but declared nullable. + Schema nullableSeq = + Schema.builder() + .addFields(DATA_SCHEMA.getFields()) + .addNullableField("seq", Schema.FieldType.INT64) + .build(); + PCollection nullableIn = + boundedInput( + nullableSeq, + KV.of(INSERT, Row.withSchema(nullableSeq).addValues(1, "a", "x", 1L).build())); + IllegalArgumentException nullable = + assertThrows( + IllegalArgumentException.class, + () -> + nullableIn.apply( + "NullableSeq", + IcebergIO.writeCdcRows(bogusCatalogConfig()) + .to(id) + .withSequenceNumberColumn("seq"))); + assertThat(nullable.getMessage(), containsString("sequence-number column 'seq'")); + assertThat(nullable.getMessage(), containsString("must be non-nullable")); + } + + /** + * The change-type column must exist (it is the cross-language default path, so a typo otherwise + * fails every record at runtime), be a non-nullable STRING, and be distinct from the + * sequence-number column. + */ + @Test + public void constructionRejectsInvalidChangeTypeColumnConfigs() { + p.enableAbandonedNodeEnforcement(false); + TableIdentifier id = TableIdentifier.of("db", "badct_" + System.nanoTime()); + PCollection in = boundedInput(INPUT_SCHEMA, KV.of(INSERT, row(1, "a", "x", 1L))); + + // facet: names a column the input does not have. + IllegalArgumentException missing = + assertThrows( + IllegalArgumentException.class, + () -> + in.apply( + "MissingChangeType", + IcebergIO.writeCdcRows(bogusCatalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withChangeTypeColumn("op_typo"))); + assertThat(missing.getMessage(), containsString("change-type column 'op_typo'")); + assertThat(missing.getMessage(), containsString("change_type_column")); + + // facet: present but INT32. + Schema intOp = Schema.builder().addFields(INPUT_SCHEMA.getFields()).addInt32Field("op").build(); + PCollection badOp = + boundedInput( + intOp, KV.of(INSERT, Row.withSchema(intOp).addValues(1, "a", "x", 1L, 7).build())); + IllegalArgumentException nonString = + assertThrows( + IllegalArgumentException.class, + () -> + badOp.apply( + "IntChangeType", + IcebergIO.writeCdcRows(bogusCatalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withChangeTypeColumn("op"))); + assertThat(nonString.getMessage(), containsString("change-type column 'op'")); + assertThat(nonString.getMessage(), containsString("must be STRING")); + assertThat(nonString.getMessage(), containsString("INT32")); + + // facet: STRING but declared nullable. + Schema nullableOp = + Schema.builder() + .addFields(INPUT_SCHEMA.getFields()) + .addNullableField("op", Schema.FieldType.STRING) + .build(); + PCollection nullableIn = + boundedInput( + nullableOp, + KV.of(INSERT, Row.withSchema(nullableOp).addValues(1, "a", "x", 1L, "I").build())); + IllegalArgumentException nullable = + assertThrows( + IllegalArgumentException.class, + () -> + nullableIn.apply( + "NullableChangeType", + IcebergIO.writeCdcRows(bogusCatalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withChangeTypeColumn("op"))); + assertThat(nullable.getMessage(), containsString("change-type column 'op'")); + assertThat(nullable.getMessage(), containsString("must be non-nullable")); + + // facet: collides with the sequence-number column. + IllegalArgumentException collides = + assertThrows( + IllegalArgumentException.class, + () -> + in.apply( + "ChangeTypeIsSeq", + IcebergIO.writeCdcRows(bogusCatalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withChangeTypeColumn("seq"))); + assertThat(collides.getMessage(), containsString("must be distinct")); + } + + /** + * The shard-config bounds: {@code num_shards >= 1}, {@code 1 <= shards_per_partition <= + * num_shards}, with the boundary {@code shards_per_partition == num_shards} (the uncapped + * default) accepted. + */ + @Test + public void constructionValidatesShardConfigBounds() { + p.enableAbandonedNodeEnforcement(false); + TableIdentifier id = TableIdentifier.of("db", "shards_" + System.nanoTime()); + PCollection in = boundedInput(INPUT_SCHEMA, KV.of(INSERT, row(1, "a", "x", 1L))); + + // facet: num_shards = 0. + IllegalArgumentException zeroShards = + assertThrows( + IllegalArgumentException.class, + () -> + in.apply( + "ZeroShards", + IcebergIO.writeCdcRows(bogusCatalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withNumShards(0))); + assertThat(zeroShards.getMessage(), containsString("num_shards must be >= 1")); + + // facet: shards_per_partition = 0, which names the option and both bounds (num_shards + // defaults to 16 here). + IllegalArgumentException sppZero = + assertThrows( + IllegalArgumentException.class, + () -> + in.apply( + "ZeroShardsPerPartition", + IcebergIO.writeCdcRows(bogusCatalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withShardsPerPartition(0))); + assertThat( + sppZero.getMessage(), + containsString("shards_per_partition must be between 1 and num_shards")); + assertThat(sppZero.getMessage(), containsString("(16)")); + assertThat(sppZero.getMessage(), containsString("got 0")); + + // facet: shards_per_partition above num_shards. + IllegalArgumentException sppAbove = + assertThrows( + IllegalArgumentException.class, + () -> + in.apply( + "ShardsPerPartitionAboveNumShards", + IcebergIO.writeCdcRows(bogusCatalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withNumShards(16) + .withShardsPerPartition(32))); + assertThat(sppAbove.getMessage(), containsString("shards_per_partition")); + assertThat(sppAbove.getMessage(), containsString("32")); + assertThat(sppAbove.getMessage(), containsString("16")); + + // facet: shards_per_partition == num_shards must build (never run). + in.apply( + "ShardsPerPartitionEqualNumShards", + IcebergIO.writeCdcRows(bogusCatalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withNumShards(16) + .withShardsPerPartition(16)); + } + + /** + * A reserved {@code beam.cdc.} snapshot-property key and an explicitly empty equality-column list + * are each rejected at construction. + */ + @Test + public void constructionRejectsReservedSnapshotPropertiesAndEmptyEqualityColumns() { + p.enableAbandonedNodeEnforcement(false); + TableIdentifier id = TableIdentifier.of("db", "props_" + System.nanoTime()); + PCollection in = boundedInput(INPUT_SCHEMA, KV.of(INSERT, row(1, "a", "x", 1L))); + + IllegalArgumentException reserved = + assertThrows( + IllegalArgumentException.class, + () -> + in.apply( + "ReservedProperty", + IcebergIO.writeCdcRows(bogusCatalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withSnapshotProperties(ImmutableMap.of("beam.cdc.foo", "x")))); + assertThat(reserved.getMessage(), containsString("beam.cdc.")); + + IllegalArgumentException emptyEquality = + assertThrows( + IllegalArgumentException.class, + () -> + in.apply( + "EmptyEqualityColumns", + IcebergIO.writeCdcRows(bogusCatalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withEqualityColumns(ImmutableList.of()))); + assertThat(emptyEquality.getMessage(), containsString("non-empty or unset")); + } + + /** {@code withSorterMemoryMB} accepts a positive value and rejects a non-positive one eagerly. */ + @Test + public void sorterMemoryMbValidatedAtConstruction() { + TableIdentifier id = TableIdentifier.of("db", "sorter_" + System.nanoTime()); + IcebergIO.writeCdcRows(bogusCatalogConfig()).to(id).withSorterMemoryMB(50); + IllegalArgumentException ex = + assertThrows( + IllegalArgumentException.class, + () -> IcebergIO.writeCdcRows(bogusCatalogConfig()).to(id).withSorterMemoryMB(0)); + assertThat(ex.getMessage(), containsString("sorter_memory_mb")); + } + + // --------------------------------------------------------------------------------------------- + // 2. Launcher touches no catalog + // --------------------------------------------------------------------------------------------- + + /** + * Neither the single-table nor the dynamic-destination branch may touch the catalog at {@code + * expand()}: building against an unreachable warehouse (and a nonexistent table) must succeed; + * schema resolution is deferred to the workers. The pipeline is never run. + */ + @Test + public void expandTouchesNoCatalog() { + p.enableAbandonedNodeEnforcement(false); + TableIdentifier id = TableIdentifier.of("db", "no_such_" + System.nanoTime()); + PCollection in = boundedInput(INPUT_SCHEMA, KV.of(INSERT, row(1, "a", "x", 1L))); + in.apply( + "SingleTable", + IcebergIO.writeCdcRows(bogusCatalogConfig()).to(id).withSequenceNumberColumn("seq")); + in.apply( + "DynamicDestinations", + IcebergIO.writeCdcRows(bogusCatalogConfig()) + .to(CdcSinkTestUtils.templatedDestinations("db.{name}", INPUT_SCHEMA, "seq")) + .withSequenceNumberColumn("seq")); + } + + // --------------------------------------------------------------------------------------------- + // 3. Batch end-to-end + // --------------------------------------------------------------------------------------------- + + @Test + public void batchCdcAppliesInsertsUpdatesDeletes() { + setUpCatalog(); + TableIdentifier id = v2Table(); + Table t = catalog.loadTable(id); + + PCollection in = + boundedInput( + INPUT_SCHEMA, + KV.of(INSERT, row(1, "a", "x", 1L)), + KV.of(INSERT, row(2, "b", "y", 1L)), + KV.of(UPDATE_BEFORE, row(1, "a", "x", 2L)), + KV.of(UPDATE_AFTER, row(1, "a", "z", 2L)), + KV.of(DELETE, row(2, "b", "y", 3L))); + IcebergWriteResult result = + in.apply(IcebergIO.writeCdcRows(catalogConfig()).to(id).withSequenceNumberColumn("seq")); + + PAssert.that(result.getSnapshots()) + .satisfies( + snapshots -> { + assertFalse(Iterables.isEmpty(snapshots)); + return null; + }); + p.run().waitUntilFinish(); + + // id=1 updated to (a,z); id=2 deleted. + assertThat(readRows(t), contains("1:a:z")); + } + + /** + * A bounded input carrying a NON-default upstream trigger must still commit every row exactly + * once: {@code CommitWindows}' bounded branch has to pin its trigger explicitly, or the inherited + * trigger fires several panes per key and every pane past the first is skipped as a same-end + * twin: rows in neither the table nor the dead letters. {@code withNumShards(1)} funnels all rows + * through one key so a pane split cannot hide in the sharding. + */ + @Test + public void boundedInputWithUpstreamTriggerCommitsAllRowsExactlyOnce() { + setUpCatalog(); + TableIdentifier id = v2Table(); + Table t = catalog.loadTable(id); + + PCollection in = + boundedInput( + INPUT_SCHEMA, + KV.of(INSERT, row(1, "a", "x1", 1L)), + KV.of(INSERT, row(2, "b", "x2", 1L)), + KV.of(INSERT, row(3, "c", "x3", 1L)), + KV.of(INSERT, row(4, "d", "x4", 1L)), + KV.of(INSERT, row(5, "e", "x5", 1L)), + KV.of(INSERT, row(6, "f", "x6", 1L)), + KV.of(INSERT, row(7, "g", "x7", 1L)), + KV.of(INSERT, row(8, "h", "x8", 1L))) + .apply( + "NonDefaultUpstreamTrigger", + Window.into(new GlobalWindows()) + .triggering(Repeatedly.forever(AfterPane.elementCountAtLeast(1))) + .discardingFiredPanes()); + + in.apply( + IcebergIO.writeCdcRows(catalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withNumShards(1)); + PipelineResult result = p.run(); + result.waitUntilFinish(); + + // readRows lists ALL live rows: a lost pane or a double-commit both fail this. + assertThat( + readRows(t), + contains("1:a:x1", "2:b:x2", "3:c:x3", "4:d:x4", "5:e:x5", "6:f:x6", "7:g:x7", "8:h:x8")); + assertThat(committerCounter(result, "alreadyCommittedWindowsSkipped"), equalTo(0L)); + } + + // --------------------------------------------------------------------------------------------- + // 4. Streaming two-window end-to-end + // --------------------------------------------------------------------------------------------- + + /** Two event-time windows commit as two snapshots with ascending tokens; final state correct. */ + @Test + public void streamingTwoWindowsCommitsPerWindow() { + setUpCatalog(); + TableIdentifier id = v2Table(); + Table t = catalog.loadTable(id); + String sinkId = "sink-" + System.nanoTime(); + Duration window = Duration.standardSeconds(60); + Instant base = new Instant(0); + + TestStream> stream = + TestStream.create( + KvCoder.of(SerializableCoder.of(ValueKind.class), RowCoder.of(INPUT_SCHEMA))) + .advanceWatermarkTo(base) + .addElements( + TimestampedValue.of( + KV.of(INSERT, row(1, "a", "x", 1L)), base.plus(Duration.standardSeconds(1))), + TimestampedValue.of( + KV.of(INSERT, row(2, "b", "y", 1L)), base.plus(Duration.standardSeconds(2)))) + .advanceWatermarkTo(base.plus(Duration.standardSeconds(70))) // window 0 commits + .addElements( + TimestampedValue.of( + KV.of(UPDATE_BEFORE, row(1, "a", "x", 2L)), + base.plus(Duration.standardSeconds(61))), + TimestampedValue.of( + KV.of(UPDATE_AFTER, row(1, "a2", "z", 2L)), + base.plus(Duration.standardSeconds(61))), + TimestampedValue.of( + KV.of(DELETE, row(2, "b", "y", 2L)), base.plus(Duration.standardSeconds(62)))) + .advanceWatermarkTo(base.plus(Duration.standardSeconds(130))) // window 1 commits + .advanceWatermarkToInfinity(); + + PCollection in = CdcSinkTestUtils.withKinds(p.apply(stream)).setRowSchema(INPUT_SCHEMA); + in.apply( + IcebergIO.writeCdcRows(catalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withTriggeringFrequency(window) + .withSinkId(sinkId)); + p.run().waitUntilFinish(); + + List tokens = committedThroughTokens(t, sinkId); + assertThat(tokens, hasSize(2)); + assertThat(tokens.get(0), lessThan(tokens.get(1))); + assertThat(readRows(t), contains("1:a2:z")); + } + + // --------------------------------------------------------------------------------------------- + // 5. Dynamic destinations (template) + control-column strip + // --------------------------------------------------------------------------------------------- + + /** + * A {@code db.{dest}} template routes rows to two tables; the default sequence-number column is + * stripped from the written rows, so neither table needs (or gains) it. + */ + @Test + public void dynamicDestinationsRouteAndStripSequenceColumn() { + setUpCatalog(); + long suffix = System.nanoTime(); + String tableA = "tmpl_a" + suffix; + String tableB = "tmpl_b" + suffix; + // Tables whose columns are (id, dest): the routing column is also a data column. + CdcSinkTestUtils.createDestTables(catalog, tableA, tableB); + + Schema inputSchema = + Schema.builder() + .addInt32Field("id") + .addNullableField("dest", Schema.FieldType.STRING) + .addInt64Field(DEFAULT_SEQ_COL) + .build(); + Row rowA = Row.withSchema(inputSchema).addValues(1, tableA, 1L).build(); + Row rowB = Row.withSchema(inputSchema).addValues(2, tableB, 1L).build(); + + PCollection in = boundedInput(inputSchema, KV.of(INSERT, rowA), KV.of(INSERT, rowB)); + in.apply( + IcebergIO.writeCdcRows(catalogConfig()) + .to(CdcSinkTestUtils.templatedDestinations("db.{dest}", inputSchema, DEFAULT_SEQ_COL))); + p.run().waitUntilFinish(); + + Table a = catalog.loadTable(TableIdentifier.of("db", tableA)); + Table b = catalog.loadTable(TableIdentifier.of("db", tableB)); + assertThat( + ImmutableList.copyOf(IcebergGenerics.read(a).build()).stream() + .map(r -> r.getField("id") + ":" + r.getField("dest")) + .collect(ImmutableList.toImmutableList()), + contains("1:" + tableA)); + assertThat( + ImmutableList.copyOf(IcebergGenerics.read(b).build()).stream() + .map(r -> r.getField("id") + ":" + r.getField("dest")) + .collect(ImmutableList.toImmutableList()), + contains("2:" + tableB)); + a.refresh(); + b.refresh(); + assertNull(a.schema().findField(DEFAULT_SEQ_COL)); + assertNull(b.schema().findField(DEFAULT_SEQ_COL)); + } + + /** + * Streaming + dynamic destinations, the combination the suite otherwise never runs. A receives + * data in windows 0 and 2, B in 1 and 2, so each destination's committer state and commit timer + * must advance on its own schedule; state shared across destinations would skip a window as + * "already committed" for the OTHER destination's end and silently lose its rows. + */ + @Test + public void streamingDynamicDestinationsCommitPerDestinationWindows() { + setUpCatalog(); + long suffix = System.nanoTime(); + String tableA = "stream_a" + suffix; + String tableB = "stream_b" + suffix; + CdcSinkTestUtils.createDestTables(catalog, tableA, tableB); + + Schema inputSchema = + Schema.builder() + .addInt32Field("id") + .addNullableField("dest", Schema.FieldType.STRING) + .addInt64Field(DEFAULT_SEQ_COL) + .build(); + String sinkId = "sink-" + suffix; + Duration window = Duration.standardSeconds(60); + Instant base = new Instant(0); + + TestStream> stream = + TestStream.create( + KvCoder.of(SerializableCoder.of(ValueKind.class), RowCoder.of(inputSchema))) + .advanceWatermarkTo(base) + // Window 0: destination A only. + .addElements( + TimestampedValue.of( + KV.of(INSERT, destRow(inputSchema, 1, tableA, 1L)), + base.plus(Duration.standardSeconds(1))), + TimestampedValue.of( + KV.of(INSERT, destRow(inputSchema, 5, tableA, 1L)), + base.plus(Duration.standardSeconds(2)))) + .advanceWatermarkTo(base.plus(Duration.standardSeconds(70))) + // Window 1: destination B only. + .addElements( + TimestampedValue.of( + KV.of(INSERT, destRow(inputSchema, 2, tableB, 2L)), + base.plus(Duration.standardSeconds(61)))) + .advanceWatermarkTo(base.plus(Duration.standardSeconds(130))) + // Window 2: both destinations. A's DELETE removes a key committed in window 0, so it + // also proves A's cross-window equality delete landed in the right order. + .addElements( + TimestampedValue.of( + KV.of(DELETE, destRow(inputSchema, 1, tableA, 3L)), + base.plus(Duration.standardSeconds(121))), + TimestampedValue.of( + KV.of(INSERT, destRow(inputSchema, 3, tableB, 3L)), + base.plus(Duration.standardSeconds(122)))) + .advanceWatermarkTo(base.plus(Duration.standardSeconds(190))) + .advanceWatermarkToInfinity(); + + PCollection in = CdcSinkTestUtils.withKinds(p.apply(stream)).setRowSchema(inputSchema); + IcebergWriteResult result = + in.apply( + IcebergIO.writeCdcRows(catalogConfig()) + .to( + CdcSinkTestUtils.templatedDestinations( + "db.{dest}", inputSchema, DEFAULT_SEQ_COL)) + .withTriggeringFrequency(window) + .withSinkId(sinkId)); + PAssert.that(result.getDeadLetterRows()).empty(); + p.run().waitUntilFinish(); + + Table a = catalog.loadTable(TableIdentifier.of("db", tableA)); + Table b = catalog.loadTable(TableIdentifier.of("db", tableB)); + + // Each destination committed exactly its OWN windows, ascending: A's first is window 0, B's + // is window 1 (not in lockstep), and both saw window 2. + List tokensA = committedThroughTokens(a, sinkId); + List tokensB = committedThroughTokens(b, sinkId); + assertThat(tokensA, hasSize(2)); + assertThat(tokensB, hasSize(2)); + assertThat(tokensA.get(0), lessThan(tokensA.get(1))); + assertThat(tokensB.get(0), lessThan(tokensB.get(1))); + assertThat(tokensA.get(0), lessThan(tokensB.get(0))); + assertThat(tokensA.get(1), equalTo(tokensB.get(1))); + + // A: window 0's id=5 survives, window 2's DELETE removed window 0's id=1. + assertThat(destRows(a), contains("5:" + tableA)); + assertThat(destRows(b), contains("2:" + tableB, "3:" + tableB)); + } + + /** A routing input row for the {@code (id, dest, seq)} dynamic-destination schema. */ + private static Row destRow(Schema schema, int id, String dest, long seq) { + return Row.withSchema(schema).addValues(id, dest, seq).build(); + } + + /** Reads all live rows of a dynamic-destination table as sorted {@code id:dest} strings. */ + private static List destRows(Table table) { + table.refresh(); + return ImmutableList.copyOf(IcebergGenerics.read(table).build()).stream() + .map(r -> r.getField("id") + ":" + r.getField("dest")) + .sorted() + .collect(ImmutableList.toImmutableList()); + } + + // --------------------------------------------------------------------------------------------- + // 6. Upsert end-to-end + // --------------------------------------------------------------------------------------------- + + @Test + public void upsertModeAppliesAfterImageOnly() { + setUpCatalog(); + TableIdentifier id = v2Table(); + Table t = catalog.loadTable(id); + + // Upsert mode: only INSERT / UPDATE_AFTER, no before-images at all. + PCollection in = + boundedInput( + INPUT_SCHEMA, + KV.of(INSERT, row(1, "a", "x", 1L)), + KV.of(INSERT, row(2, "b", "y", 1L)), + KV.of(UPDATE_AFTER, row(1, "a", "z", 2L))); + in.apply( + IcebergIO.writeCdcRows(catalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withUpsert(true)); + p.run().waitUntilFinish(); + + // id=1's UPDATE_AFTER (seq 2) replaces its INSERT (seq 1); id=2 unaffected. + assertThat(readRows(t), contains("1:a:z", "2:b:y")); + } + + /** + * Upsert mode still applies DELETEs: widening the upsert drop predicate from "UPDATE_BEFORE" to + * "anything not an after-image" would swallow every DELETE, and nothing else in the suite fails. + */ + @Test + public void upsertModeStillAppliesDeletes() { + setUpCatalog(); + TableIdentifier id = v2Table(); + Table t = catalog.loadTable(id); + + PCollection in = + boundedInput( + INPUT_SCHEMA, + KV.of(INSERT, row(1, "a", "x", 1L)), + KV.of(INSERT, row(2, "b", "y", 1L)), + KV.of(UPDATE_AFTER, row(1, "a", "z", 2L)), + KV.of(DELETE, row(2, "b", "y", 3L))); + in.apply( + IcebergIO.writeCdcRows(catalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withUpsert(true)); + p.run().waitUntilFinish(); + + // id=2 is genuinely gone; id=1 carries its after-image. + assertThat(readRows(t), contains("1:a:z")); + } + + // --------------------------------------------------------------------------------------------- + // 8. Dead letters: exposure, first-late diversion, replayability + // --------------------------------------------------------------------------------------------- + + /** + * A FIRST-late pane (the shard-window's only record arrives after the watermark passed) is + * dead-lettered, not committed: pane timing is per shard-window, the committer's skip is per + * destination-window, so the pane cannot prove its window uncommitted. + */ + @Test + public void firstLatePaneIsDeadLetteredNotCommitted() { + setUpCatalog(); + TableIdentifier id = v2Table(); + Table t = catalog.loadTable(id); + + Duration window = Duration.standardSeconds(60); + Instant base = new Instant(0); + + TestStream> stream = + TestStream.create( + KvCoder.of(SerializableCoder.of(ValueKind.class), RowCoder.of(INPUT_SCHEMA))) + .advanceWatermarkTo(base) + // Advance PAST window [0, 60s) with no data, then deliver its first (late) element. + .advanceWatermarkTo(base.plus(Duration.standardSeconds(120))) + .addElements( + TimestampedValue.of( + KV.of(INSERT, row(1, "a", "x", 1L)), base.plus(Duration.standardSeconds(1)))) + .advanceWatermarkToInfinity(); + + PCollection in = CdcSinkTestUtils.withKinds(p.apply(stream)).setRowSchema(INPUT_SCHEMA); + IcebergWriteResult result = + in.apply( + IcebergIO.writeCdcRows(catalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withTriggeringFrequency(window) + .withAllowedLateness(Duration.standardDays(7))); + + // Captured by the PAssert lambda: TableIdentifier itself is not Serializable. + String destString = id.toString(); + PAssert.that(result.getDeadLetterRows()) + .satisfies( + rows -> { + Row dl = Iterables.getOnlyElement(rows); + assertThat(dl.getRow("record"), equalTo(rowNoSeq(1, "a", "x"))); + assertThat(dl.getString("change_type"), equalTo("INSERT")); + assertThat(dl.getInt64("sequence_number"), equalTo(1L)); + assertThat(dl.getString("destination"), equalTo(destString)); + return null; + }); + p.run().waitUntilFinish(); + + // The first-late record was dead-lettered (replayable), not committed. + assertThat(readRows(t), hasSize(0)); + } + + /** + * A late record whose key hashes to a previously-untouched shard of a window that already + * committed opens a fresh key+window whose first pane is late: the case a {@code !pane.isFirst()} + * test lets through, after which the committer discards it and the row reaches no output. It must + * be dead-lettered. + */ + @Test + public void firstLatePaneOnAnUntouchedShardIsDeadLettered() { + setUpCatalog(); + TableIdentifier id = v2Table(); + Table t = catalog.loadTable(id); + + int onTimeId = 1; + int lateId = idOnADifferentShardThan(onTimeId, CdcWriteConfig.DEFAULT_NUM_SHARDS); + + Duration window = Duration.standardSeconds(60); + Instant base = new Instant(0); + + TestStream> stream = + TestStream.create( + KvCoder.of(SerializableCoder.of(ValueKind.class), RowCoder.of(INPUT_SCHEMA))) + .advanceWatermarkTo(base) + // On-time data commits window [0, 60s); then a late record for the SAME window on a + // shard that saw nothing. + .addElements( + TimestampedValue.of( + KV.of(INSERT, row(onTimeId, "a", "x", 1L)), + base.plus(Duration.standardSeconds(1)))) + .advanceWatermarkTo(base.plus(Duration.standardSeconds(70))) + .addElements( + TimestampedValue.of( + KV.of(INSERT, row(lateId, "late", "z", 2L)), + base.plus(Duration.standardSeconds(2)))) + .advanceWatermarkToInfinity(); + + PCollection in = CdcSinkTestUtils.withKinds(p.apply(stream)).setRowSchema(INPUT_SCHEMA); + IcebergWriteResult result = + in.apply( + IcebergIO.writeCdcRows(catalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withTriggeringFrequency(window) + .withAllowedLateness(Duration.standardDays(7))); + + // Captured by the PAssert lambda: TableIdentifier itself is not Serializable. + String destString = id.toString(); + PAssert.that(result.getDeadLetterRows()) + .satisfies( + rows -> { + Row dl = Iterables.getOnlyElement(rows); + assertThat(dl.getRow("record"), equalTo(rowNoSeq(lateId, "late", "z"))); + assertThat(dl.getString("change_type"), equalTo("INSERT")); + assertThat(dl.getInt64("sequence_number"), equalTo(2L)); + assertThat(dl.getString("destination"), equalTo(destString)); + return null; + }); + p.run().waitUntilFinish(); + + // NOT silently lost: it is absent from the table (above) and present in the dead letters. + assertThat(readRows(t), contains(onTimeId + ":a:x")); + } + + /** + * The lowest id above {@code id} mapping to a different shard: computed, not hard-coded, so the + * test survives hash or pk-encoding changes. + */ + private static int idOnADifferentShardThan(int id, int numShards) { + int target = shardForId(id, numShards); + for (int candidate = id + 1; candidate < id + 10_000; candidate++) { + if (shardForId(candidate, numShards) != target) { + return candidate; + } + } + throw new AssertionError("no id maps to a shard other than " + target); + } + + /** The shard {@code AssignCdcKeys} assigns to a row with primary key {@code id}. */ + private static int shardForId(int id, int numShards) { + Schema pkSchema = Schema.builder().addInt32Field("id").build(); + try { + byte[] pkBytes = + CoderUtils.encodeToByteArray( + RowCoder.of(pkSchema), Row.withSchema(pkSchema).addValue(id).build()); + return TableSetup.shardFor(pkBytes, numShards); + } catch (CoderException e) { + throw new AssertionError(e); + } + } + + /** + * Pins that {@link WriteCdcRows#withAllowedLateness} is honored: a record late but within the + * bound is dead-lettered, one past the bound is dropped ENTIRELY by the runner before {@code + * SplitLateData} sees it: no dead letter, no counter, no row. That documented asymmetry is + * exactly what a wrong or ignored lateness value would change. + */ + @Test + public void recordsBeyondAllowedLatenessAreDroppedNotDeadLettered() { + setUpCatalog(); + TableIdentifier id = v2Table(); + Table t = catalog.loadTable(id); + + Duration window = Duration.standardSeconds(60); + Duration allowedLateness = Duration.standardMinutes(10); + Instant base = new Instant(0); + + TestStream> stream = + TestStream.create( + KvCoder.of(SerializableCoder.of(ValueKind.class), RowCoder.of(INPUT_SCHEMA))) + .advanceWatermarkTo(base) + // Window [0, 60s) closes empty; ~5 min late is inside the 10-min bound. + .advanceWatermarkTo(base.plus(Duration.standardMinutes(5))) + .addElements( + TimestampedValue.of( + KV.of(INSERT, row(1, "within", "x", 1L)), + base.plus(Duration.standardSeconds(1)))) + // Past window end + the whole allowed lateness: dropped outright. + .advanceWatermarkTo(base.plus(Duration.standardMinutes(30))) + .addElements( + TimestampedValue.of( + KV.of(INSERT, row(2, "beyond", "y", 2L)), + base.plus(Duration.standardSeconds(2)))) + .advanceWatermarkToInfinity(); + + PCollection in = CdcSinkTestUtils.withKinds(p.apply(stream)).setRowSchema(INPUT_SCHEMA); + IcebergWriteResult result = + in.apply( + IcebergIO.writeCdcRows(catalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withTriggeringFrequency(window) + .withAllowedLateness(allowedLateness)); + + PAssert.that(result.getDeadLetterRows()) + .satisfies( + rows -> { + // Exactly the within-bound record; the beyond-bound one never reaches the sink. + Row dl = Iterables.getOnlyElement(rows); + assertThat(dl.getRow("record"), equalTo(rowNoSeq(1, "within", "x"))); + return null; + }); + p.run().waitUntilFinish(); + + assertThat(readRows(t), hasSize(0)); + } + + /** + * A NON-first late pane (its shard-window already fired on time) is diverted to the replayable + * dead-letter output with the three metadata columns, and is NOT committed. + */ + @Test + public void deadLetterRowsAreExposed() { + setUpCatalog(); + TableIdentifier id = v2Table(); + Table t = catalog.loadTable(id); + + Duration window = Duration.standardSeconds(60); + Instant base = new Instant(0); + + TestStream> stream = + TestStream.create( + KvCoder.of(SerializableCoder.of(ValueKind.class), RowCoder.of(INPUT_SCHEMA))) + .advanceWatermarkTo(base) + .addElements( + TimestampedValue.of( + KV.of(INSERT, row(1, "a", "x", 1L)), base.plus(Duration.standardSeconds(1)))) + .advanceWatermarkTo(base.plus(Duration.standardSeconds(70))) // close window [0, 60s) + // The same key again, timestamped INSIDE the closed window: a NON-first late pane. + .addElements( + TimestampedValue.of( + KV.of(UPDATE_AFTER, row(1, "late", "z", 2L)), + base.plus(Duration.standardSeconds(2)))) + .advanceWatermarkToInfinity(); + + PCollection in = CdcSinkTestUtils.withKinds(p.apply(stream)).setRowSchema(INPUT_SCHEMA); + IcebergWriteResult result = + in.apply( + IcebergIO.writeCdcRows(catalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withTriggeringFrequency(window) + .withAllowedLateness(Duration.standardDays(7))); + + // Captured by the PAssert lambda: TableIdentifier itself is not Serializable. + String destString = id.toString(); + PAssert.that(result.getDeadLetterRows()) + .satisfies( + rows -> { + Row dl = Iterables.getOnlyElement(rows); + assertThat(dl.getRow("record"), equalTo(rowNoSeq(1, "late", "z"))); + assertThat(dl.getString("change_type"), equalTo("UPDATE_AFTER")); + assertThat(dl.getInt64("sequence_number"), equalTo(2L)); + assertThat(dl.getString("destination"), equalTo(destString)); + return null; + }); + p.run().waitUntilFinish(); + + // Invariant: dead-lettered => NOT committed. The table has only the on-time INSERT (1:a:x). + assertThat(readRows(t), contains("1:a:x")); + } + + /** + * The dead letters are REPLAYABLE through the real API: unnest {@code record}, map {@code + * change_type}/{@code sequence_number} as the replay sink's control columns, drop {@code + * destination}, and the row lands. The late record uses a different key than the on-time one so + * the final state is independent of commit interleaving. + */ + @Test + public void deadLettersAreReplayable() { + setUpCatalog(); + TableIdentifier id = v2Table(); + Table t = catalog.loadTable(id); + + Duration window = Duration.standardSeconds(60); + Instant base = new Instant(0); + + TestStream> stream = + TestStream.create( + KvCoder.of(SerializableCoder.of(ValueKind.class), RowCoder.of(INPUT_SCHEMA))) + .advanceWatermarkTo(base) + .addElements( + TimestampedValue.of( + KV.of(INSERT, row(1, "a", "x", 1L)), base.plus(Duration.standardSeconds(1)))) + .advanceWatermarkTo(base.plus(Duration.standardSeconds(70))) // window [0,60s) fires + .addElements( + TimestampedValue.of( + KV.of(UPDATE_AFTER, row(7, "late", "z", 2L)), + base.plus(Duration.standardSeconds(2)))) + .advanceWatermarkToInfinity(); + + PCollection in = CdcSinkTestUtils.withKinds(p.apply(stream)).setRowSchema(INPUT_SCHEMA); + IcebergWriteResult result = + in.apply( + "OriginalSink", + IcebergIO.writeCdcRows(catalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withTriggeringFrequency(window) + .withAllowedLateness(Duration.standardDays(7))); + + PCollection deadLetters = result.getDeadLetterRows(); + assertNotNull(deadLetters); + // Captured by the PAssert lambda: TableIdentifier itself is not Serializable. + String destString = id.toString(); + PAssert.that(deadLetters) + .satisfies( + rows -> { + Row dl = Iterables.getOnlyElement(rows); + assertThat(dl.getRow("record"), equalTo(rowNoSeq(7, "late", "z"))); + assertThat(dl.getString("change_type"), equalTo("UPDATE_AFTER")); + assertThat(dl.getInt64("sequence_number"), equalTo(2L)); + assertThat(dl.getString("destination"), equalTo(destString)); + return null; + }); + + // Re-feed the ACTUAL dead letters. + Schema replaySchema = + Schema.builder() + .addFields(DATA_SCHEMA.getFields()) + .addStringField("change_type") + .addInt64Field("sequence_number") + .build(); + PCollection replayInput = + deadLetters + .apply( + "UnnestRecord", + MapElements.into(TypeDescriptors.rows()) + .via( + dl -> + Row.withSchema(replaySchema) + .addValues(checkStateNotNull(dl.getRow("record")).getValues()) + .addValue(dl.getString("change_type")) + .addValue(dl.getInt64("sequence_number")) + .build())) + .setRowSchema(replaySchema) + // A dead letter's own event time is behind the watermark by definition; re-fed as-is + // it would just be late again, so a replay re-stamps it as fresh input. + .apply("RestampForReplay", ParDo.of(new RestampTo(REPLAY_EVENT_TIME))) + .setRowSchema(replaySchema); + replayInput.apply( + "ReplaySink", + IcebergIO.writeCdcRows(catalogConfig()) + .to(id) + .withChangeTypeColumn("change_type") + .withSequenceNumberColumn("sequence_number") + .withTriggeringFrequency(window) + .withAllowedLateness(Duration.standardDays(7))); + + p.run().waitUntilFinish(); + + assertThat(readRows(t), containsInAnyOrder("1:a:x", "7:late:z")); + } + + // --------------------------------------------------------------------------------------------- + // 9. Error handling (failed rows) + // --------------------------------------------------------------------------------------------- + + /** + * With {@code withErrorHandling()}, a poison record (an unknown change-type value) is diverted to + * {@link IcebergWriteResult#getFailedRows()}; the other records commit normally. + */ + @Test + public void errorHandlingDivertsPoisonRecords() { + setUpCatalog(); + TableIdentifier id = v2Table(); + Table t = catalog.loadTable(id); + + Schema inputWithOp = + Schema.builder() + .addFields(DATA_SCHEMA.getFields()) + .addInt64Field("seq") + .addStringField("op") + .build(); + Row r1 = Row.withSchema(inputWithOp).addValues(1, "a", "x", 1L, "INSERT").build(); + Row r2 = Row.withSchema(inputWithOp).addValues(2, "b", "y", 1L, "BOGUS").build(); // poison + Row r3 = Row.withSchema(inputWithOp).addValues(3, "c", "z", 1L, "INSERT").build(); + + PCollection in = + boundedInput(inputWithOp, KV.of(INSERT, r1), KV.of(INSERT, r2), KV.of(INSERT, r3)); + IcebergWriteResult result = + in.apply( + IcebergIO.writeCdcRows(catalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withChangeTypeColumn("op") + .withErrorHandling()); + + PCollection failedRows = result.getFailedRows(); + assertNotNull(failedRows); + PAssert.that(failedRows) + .satisfies( + rows -> { + Row err = Iterables.getOnlyElement(rows); + Row failedRow = err.getRow("failed_row"); + assertThat(failedRow.getInt32("id"), equalTo(2)); + assertThat(failedRow.getString("op"), equalTo("BOGUS")); + // Operators must see WHICH change-type value was bad, not just that something failed. + assertThat(err.getString("error_message"), containsString("BOGUS")); + return null; + }); + p.run().waitUntilFinish(); + + assertThat(readRows(t), contains("1:a:x", "3:c:z")); + } + + /** Without {@code withErrorHandling()}, the failed-rows accessor is {@code null}. */ + @Test + public void failedRowsNullWithoutErrorHandling() { + p.enableAbandonedNodeEnforcement(false); + TableIdentifier id = TableIdentifier.of("db", "nofail_" + System.nanoTime()); + PCollection in = boundedInput(INPUT_SCHEMA, KV.of(INSERT, row(1, "a", "x", 1L))); + IcebergWriteResult result = + in.apply( + IcebergIO.writeCdcRows(bogusCatalogConfig()).to(id).withSequenceNumberColumn("seq")); + assertNull(result.getFailedRows()); + assertNotNull(result.getDeadLetterRows()); + assertNotNull(result.getSnapshots()); + } + + // --------------------------------------------------------------------------------------------- + // 10. Dead-letter column names pinned + // --------------------------------------------------------------------------------------------- + + /** The dead-letter metadata column names and types are a public contract: pinned literally. */ + @Test + public void deadLetterColumnNamesPinned() { + p.enableAbandonedNodeEnforcement(false); + TableIdentifier id = TableIdentifier.of("db", "dlpin_" + System.nanoTime()); + PCollection in = boundedInput(INPUT_SCHEMA, KV.of(INSERT, row(1, "a", "x", 1L))); + IcebergWriteResult result = + in.apply( + IcebergIO.writeCdcRows(bogusCatalogConfig()).to(id).withSequenceNumberColumn("seq")); + + PCollection deadLetters = result.getDeadLetterRows(); + assertNotNull(deadLetters); + Schema dl = deadLetters.getSchema(); + assertThat( + dl.getFieldNames(), contains("record", "change_type", "sequence_number", "destination")); + assertThat(dl.getField("record").getType(), equalTo(Schema.FieldType.row(DATA_SCHEMA))); + assertThat(dl.getField("change_type").getType(), equalTo(Schema.FieldType.STRING)); + assertThat(dl.getField("sequence_number").getType(), equalTo(Schema.FieldType.INT64)); + assertThat(dl.getField("destination").getType(), equalTo(Schema.FieldType.STRING)); + } + + // --------------------------------------------------------------------------------------------- + // 12. Two-pipeline batch rerun with stable sink id + // --------------------------------------------------------------------------------------------- + + /** + * A true two-pipeline batch rerun (not a hand-seeded token): the second pipeline, same table and + * {@code sink_id} but a different row, must write nothing: the batch token is recovered from the + * real prior commit and the rerun's window skipped loudly. + */ + @Test + public void secondBatchRunWithStableSinkIdWritesNothing() { + setUpCatalog(); + TableIdentifier id = v2Table(); + Table t = catalog.loadTable(id); + String sinkId = "stable-rerun"; + + // P1: first batch run, one INSERT under the stable sink id. + boundedInput(INPUT_SCHEMA, KV.of(INSERT, row(1, "a", "x", 1L))) + .apply( + IcebergIO.writeCdcRows(catalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withSinkId(sinkId)); + p.run().waitUntilFinish(); + + t.refresh(); + assertThat(Lists.newArrayList(t.snapshots()), hasSize(1)); + assertThat(readRows(t), contains("1:a:x")); + + // P2: a fresh batch pipeline, SAME table + SAME sink id, DIFFERENT row. + Pipeline p2 = Pipeline.create(); + CdcSinkTestUtils.withKinds(p2.apply(Create.of(KV.of(INSERT, row(2, "b", "y", 2L))))) + .setRowSchema(INPUT_SCHEMA) + .apply( + IcebergIO.writeCdcRows(catalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withSinkId(sinkId)); + PipelineResult r2 = p2.run(); + r2.waitUntilFinish(); + + t.refresh(); + // Still one snapshot, still only the first row, and the skip was loud. + assertThat(Lists.newArrayList(t.snapshots()), hasSize(1)); + assertThat(readRows(t), contains("1:a:x")); + assertThat(committerCounter(r2, "alreadyCommittedWindowsSkipped"), greaterThanOrEqualTo(1L)); + } + + // --------------------------------------------------------------------------------------------- + // 13. Token heartbeat gated to streaming + // --------------------------------------------------------------------------------------------- + + /** + * A heartbeat configured on BOUNDED input must be dropped with a warn at {@code expand()}: a + * self-rescheduling processing-time timer can keep a runner from quiescing. The run terminates + * promptly with exactly one snapshot. + */ + @Test(timeout = 120_000) + public void tokenHeartbeatIgnoredForBoundedInput() { + setUpCatalog(); + TableIdentifier id = v2Table(); + Table t = catalog.loadTable(id); + + PCollection in = boundedInput(INPUT_SCHEMA, KV.of(INSERT, row(1, "a", "x", 1L))); + in.apply( + IcebergIO.writeCdcRows(catalogConfig()) + .to(id) + .withSequenceNumberColumn("seq") + .withTokenHeartbeat(Duration.millis(1))); + + // expand() runs during apply(); on bounded input it must warn that the heartbeat is dropped. + expectedLogs.verifyWarn("heartbeat is ignored for bounded input"); + + p.run().waitUntilFinish(); + + t.refresh(); + assertThat(Lists.newArrayList(t.snapshots()), hasSize(1)); + } + + /** Re-stamps every element to one fixed event time (a replay assigns fresh event times). */ + private static final class RestampTo extends DoFn { + private final Instant timestamp; + + RestampTo(Instant timestamp) { + this.timestamp = timestamp; + } + + @ProcessElement + public void process(@Element Row row, OutputReceiver out) { + out.outputWithTimestamp(row, timestamp); + } + } +} From ec89ca83982cf1ae5f29590062dd4462796a4f74 Mon Sep 17 00:00:00 2001 From: Ahmed Abualsaud Date: Tue, 22 Sep 2026 11:28:45 -0700 Subject: [PATCH 2/9] addreress comments --- .../sdk/io/iceberg/IcebergWriteResult.java | 18 +++++++------ .../sdk/io/iceberg/cdc/sink/WriteCdcRows.java | 27 +++++++++++++++++++ 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergWriteResult.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergWriteResult.java index 8db954428b3f..d08a0acb5ab4 100644 --- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergWriteResult.java +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergWriteResult.java @@ -17,6 +17,8 @@ */ package org.apache.beam.sdk.io.iceberg; +import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull; + import java.util.Map; import org.apache.beam.sdk.Pipeline; import org.apache.beam.sdk.annotations.Internal; @@ -75,11 +77,11 @@ public PCollection> getSnapshots() { * change for those keys has committed; a stale replay's equality delete removes the newer row. * * @return the dead-letter {@code PCollection} for results produced by {@code - * IcebergIO.writeCdcRows}/{@link #cdc}, or {@code null} for results produced by the - * append-only sink ({@code IcebergIO.writeRows}), which has no dead-letter output. + * IcebergIO.writeCdcRows}/{@link #cdc}. Throws when attempting to retrieve it for the + * append-only sink ({@code IcebergIO.writeRows}) as has no dead-letter output. */ - public @Nullable PCollection getDeadLetterRows() { - return deadLetterRows; + public PCollection getDeadLetterRows() { + return checkStateNotNull(deadLetterRows); } /** @@ -88,11 +90,11 @@ public PCollection> getSnapshots() { * {@link org.apache.beam.sdk.schemas.transforms.providers.ErrorHandling}). Distinct from {@link * #getDeadLetterRows()} (which carries late-but-valid records). * - * @return the failed-rows {@code PCollection}, or {@code null} when error handling was not - * enabled (or for the append-only sink). + * @return the failed-rows {@code PCollection} if error handling was enabled for the CDC + * sink. Will throw otherwise. */ - public @Nullable PCollection getFailedRows() { - return failedRows; + public PCollection getFailedRows() { + return checkStateNotNull(failedRows); } IcebergWriteResult(Pipeline pipeline, PCollection> snapshots) { diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRows.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRows.java index 20a44dc1c7cb..cc707076a35d 100644 --- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRows.java +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRows.java @@ -31,11 +31,13 @@ import org.apache.beam.sdk.io.iceberg.SnapshotInfo; import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.transforms.display.DisplayData; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionTuple; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.ValueKind; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Predicates; import org.apache.iceberg.catalog.TableIdentifier; @@ -380,6 +382,8 @@ private CdcWriteConfig cdcWriteConfig() { /** Construction-time validation. */ private void validate(PCollection input, CdcWriteConfig config) { + Preconditions.checkState( + input.hasSchema(), "CDC input must have a schema; set one with setRowSchema(...)."); Preconditions.checkArgument( 1 == Stream.of(getTableIdentifier(), getDynamicDestinations()) @@ -401,7 +405,15 @@ private void validate(PCollection input, CdcWriteConfig config) { triggeringFrequency.isLongerThan(Duration.ZERO), "triggering frequency must be positive for streaming CDC writes, got %s", triggeringFrequency); + } else if (triggeringFrequency != null) { + LOG.warn( + "Triggering frequency is ignored for bounded input. Batch loads commit once at the end."); } + @Nullable Duration allowedLateness = getAllowedLateness(); + Preconditions.checkArgument( + allowedLateness == null || !allowedLateness.isShorterThan(Duration.ZERO), + "allowed lateness must not be negative, got %s", + allowedLateness); @Nullable String changeTypeColumn = config.getChangeTypeColumn(); if (changeTypeColumn != null) { @@ -461,6 +473,21 @@ private void validate(PCollection input, CdcWriteConfig config) { } } + @Override + public void populateDisplayData(DisplayData.Builder builder) { + super.populateDisplayData(builder); + builder.add(DisplayData.item("sinkId", getSinkId())); + builder.add(DisplayData.item("upsert", getUpsert())); + builder.add(DisplayData.item("numShards", getNumShards())); + builder.addIfNotNull(DisplayData.item("shardsPerPartition", getShardsPerPartition())); + builder.add(DisplayData.item("sorterMemoryMB", getSorterMemoryMB())); + builder.addIfNotNull(DisplayData.item("triggeringFrequency", getTriggeringFrequency())); + builder.add( + DisplayData.item( + "allowedLateness", + MoreObjects.firstNonNull(getAllowedLateness(), DEFAULT_ALLOWED_LATENESS))); + } + @Override public IcebergWriteResult expand(PCollection input) { CdcWriteConfig config = cdcWriteConfig(); From d243862cfa1dc98869beeca1e000205b5a915de7 Mon Sep 17 00:00:00 2001 From: Ahmed Abualsaud Date: Tue, 22 Sep 2026 11:31:57 -0700 Subject: [PATCH 3/9] address comments --- .../apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRows.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRows.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRows.java index cc707076a35d..33a11d554f96 100644 --- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRows.java +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRows.java @@ -20,6 +20,8 @@ import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull; import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import java.util.List; import java.util.Map; import java.util.UUID; @@ -214,7 +216,7 @@ public WriteCdcRows to(DynamicDestinations destinations) { * {@link #withShardsPerPartition} cap. */ public WriteCdcRows withEqualityColumns(List columns) { - return toBuilder().setEqualityColumns(columns).build(); + return toBuilder().setEqualityColumns(ImmutableList.copyOf(columns)).build(); } /** @@ -242,7 +244,7 @@ public WriteCdcRows withChangeTypeColumn(String column) { * #withChangeTypeColumn} to also be set. */ public WriteCdcRows withChangeTypeMap(Map changeTypeMap) { - return toBuilder().setChangeTypeMap(changeTypeMap).build(); + return toBuilder().setChangeTypeMap(ImmutableMap.copyOf(changeTypeMap)).build(); } /** @@ -328,7 +330,7 @@ public WriteCdcRows withErrorHandling() { * rejected at construction. */ public WriteCdcRows withSnapshotProperties(Map snapshotProperties) { - return toBuilder().setSnapshotProperties(snapshotProperties).build(); + return toBuilder().setSnapshotProperties(ImmutableMap.copyOf(snapshotProperties)).build(); } /** From 0f96bdf46fd62770468232db0c3cabc20c9d7f29 Mon Sep 17 00:00:00 2001 From: Ahmed Abualsaud Date: Tue, 22 Sep 2026 11:49:53 -0700 Subject: [PATCH 4/9] spotless --- .../org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRows.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRows.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRows.java index 33a11d554f96..9d240c0e7d76 100644 --- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRows.java +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRows.java @@ -20,8 +20,6 @@ import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull; import com.google.auto.value.AutoValue; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; import java.util.List; import java.util.Map; import java.util.UUID; @@ -42,6 +40,8 @@ import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Predicates; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.apache.iceberg.catalog.TableIdentifier; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; From 488b7452405714e984971a0ddcb757de0233e185 Mon Sep 17 00:00:00 2001 From: Ahmed Abualsaud Date: Tue, 22 Sep 2026 11:52:25 -0700 Subject: [PATCH 5/9] update test --- .../apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRowsTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRowsTest.java b/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRowsTest.java index 1e3a2a03d121..e093440225c9 100644 --- a/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRowsTest.java +++ b/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRowsTest.java @@ -1374,7 +1374,7 @@ public void failedRowsNullWithoutErrorHandling() { IcebergWriteResult result = in.apply( IcebergIO.writeCdcRows(bogusCatalogConfig()).to(id).withSequenceNumberColumn("seq")); - assertNull(result.getFailedRows()); + assertThrows(IllegalStateException.class, result::getFailedRows); assertNotNull(result.getDeadLetterRows()); assertNotNull(result.getSnapshots()); } From a6115ede558689a1b8e7654a2168ae9073b6d4b7 Mon Sep 17 00:00:00 2001 From: Ahmed Abualsaud Date: Tue, 22 Sep 2026 16:05:40 -0700 Subject: [PATCH 6/9] add cdc option to existing schematransform --- .../IcebergWriteSchemaTransformProvider.java | 377 +++++++++- ...IcebergSchemaTransformTranslationTest.java | 37 + ...rgWriteSchemaTransformProviderCdcTest.java | 666 ++++++++++++++++++ 3 files changed, 1072 insertions(+), 8 deletions(-) create mode 100644 sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergWriteSchemaTransformProviderCdcTest.java diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergWriteSchemaTransformProvider.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergWriteSchemaTransformProvider.java index e5a81e41a8bb..831db3559019 100644 --- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergWriteSchemaTransformProvider.java +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergWriteSchemaTransformProvider.java @@ -18,14 +18,23 @@ package org.apache.beam.sdk.io.iceberg; import static org.apache.beam.sdk.io.iceberg.IcebergWriteSchemaTransformProvider.Configuration; +import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull; import static org.apache.beam.sdk.util.construction.BeamUrns.getUrn; import com.google.auto.service.AutoService; import com.google.auto.value.AutoValue; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Set; import org.apache.beam.model.pipeline.v1.ExternalTransforms; +import org.apache.beam.sdk.io.iceberg.cdc.IcebergCdcMetadataColumns; +import org.apache.beam.sdk.io.iceberg.cdc.sink.WriteCdcRows; import org.apache.beam.sdk.schemas.AutoValueSchema; import org.apache.beam.sdk.schemas.NoSuchSchemaException; import org.apache.beam.sdk.schemas.Schema; @@ -35,6 +44,7 @@ import org.apache.beam.sdk.schemas.transforms.SchemaTransform; import org.apache.beam.sdk.schemas.transforms.SchemaTransformProvider; import org.apache.beam.sdk.schemas.transforms.TypedSchemaTransformProvider; +import org.apache.beam.sdk.schemas.transforms.providers.ErrorHandling; import org.apache.beam.sdk.transforms.MapElements; import org.apache.beam.sdk.transforms.SimpleFunction; import org.apache.beam.sdk.values.KV; @@ -42,14 +52,16 @@ import org.apache.beam.sdk.values.PCollectionRowTuple; import org.apache.beam.sdk.values.Row; import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.apache.iceberg.DistributionMode; import org.apache.iceberg.FileFormat; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; /** - * SchemaTransform implementation for {@link IcebergIO#writeRows}. Writes Beam Rows to Iceberg and - * outputs a {@code PCollection} representing snapshots created in the process. + * SchemaTransform implementation for {@link IcebergIO#writeRows} and, when the {@code cdc} block is + * set, {@link IcebergIO#writeCdcRows}. Outputs a {@code PCollection} representing the + * snapshots created in the process; CDC writes add a {@code dead_letter} output of late records. */ @AutoService(SchemaTransformProvider.class) public class IcebergWriteSchemaTransformProvider @@ -57,6 +69,12 @@ public class IcebergWriteSchemaTransformProvider static final String INPUT_TAG = "input"; static final String SNAPSHOTS_TAG = "snapshots"; + static final String DEAD_LETTER_TAG = "dead_letter"; + static final String ERRORS_TAG = "errors"; + + /** The default sequence-number column: what the CDC read source emits. */ + private static final String DEFAULT_SEQUENCE_NUMBER_COLUMN = + IcebergCdcMetadataColumns.COMMIT_SNAPSHOT_SEQUENCE_NUMBER; static final Schema OUTPUT_SCHEMA = Schema.builder() @@ -66,9 +84,14 @@ public class IcebergWriteSchemaTransformProvider @Override public String description() { - return "Writes Beam Rows to Iceberg.\n" - + "Returns a PCollection representing the snapshots produced in the process, with the following schema:\n" - + "{\"table\" (str), \"operation\" (str), \"summary\" (map[str, str]), \"manifestListLocation\" (str)}"; + return "Writes Beam Rows to Iceberg, appending them by default. Set the 'cdc' block to apply " + + "them as a stream of row-level changes (INSERT/UPDATE_BEFORE/UPDATE_AFTER/DELETE) by " + + "primary key instead.\n" + + "Returns a 'snapshots' PCollection representing the snapshots produced in the process, " + + "with the following schema:\n" + + "{\"table\" (str), \"operation\" (str), \"summary\" (map[str, str]), \"manifestListLocation\" (str)}\n" + + "CDC mode also returns a 'dead_letter' PCollection representing late data, and a 'errors' PCollection " + + "representing invalid records."; } @DefaultSchema(AutoValueSchema.class) @@ -100,14 +123,33 @@ public static Builder builder() { "For a streaming pipeline, sets the limit for lifting bundles into the direct write path.") public abstract @Nullable Integer getDirectWriteByteLimit(); + @SchemaFieldDescription( + "A config map that, when present, enables writing the input as a stream of row-level " + + "changes applied by primary key. An empty map will use the defaults. Config options include:\n" + + "- `sequence_number_column`: the required column name representing the" + + "monotonic sequence number used to order a single key's changes. Default" + + "column name is '_commit_snapshot_sequence_number'. This column will be" + + "stripped from the data row before writing to Iceberg.\n" + + "- `change_type_column`: the optional column name representing the row's" + + "change type (INSERT, UPDATE_BEFORE, UPDATE_AFTER, or DELETE). This" + + "column will be stripped from the data row before writing to Iceberg.\n" + + "- `change_type_map`: optional map from the `change_type_column` value to" + + "the canonical change type name (see above).\n" + + "- `upsert`: if true, only the after-image of each change" + + "(INSERT/UPDATE_AFTER) is applied as an upsert. UPDATE_BEFORE records" + + "are dropped. Default: false.") + public abstract @Nullable Cdc getCdc(); + @SchemaFieldDescription( "A list of field names to keep in the input record. All other fields are dropped before writing. " - + "Is mutually exclusive with 'drop' and 'only'.") + + "Is mutually exclusive with 'drop' and 'only'. In CDC mode the control columns are " + + "dropped unless listed here.") public abstract @Nullable List getKeep(); @SchemaFieldDescription( "A list of field names to drop from the input record before writing. " - + "Is mutually exclusive with 'keep' and 'only'.") + + "Is mutually exclusive with 'keep' and 'only'. In CDC mode the control columns are " + + "always dropped.") public abstract @Nullable List getDrop(); @SchemaFieldDescription( @@ -180,6 +222,62 @@ public static Builder builder() { "Sets the number of parallel buckets/workers used to query the Iceberg catalog during refreshes. Defaults to 1.") public abstract @Nullable Integer getPollingBuckets(); + @SchemaFieldDescription( + "Columns defining row identity (equality-delete fields). Defaults to the destination table's " + + "identifier (primary-key) fields. Required if the table doesn't exist yet. Currently only supported in CDC mode.") + public abstract @Nullable List getEqualityColumns(); + + @SchemaFieldDescription( + "The number of deterministic primary-key-hash shards per destination, i.e. the max " + + "write parallelism per destination. Too low may bottleneck writes, and too high may " + + "produce more files. Defaults to 16. Currently only supported in CDC mode.") + public abstract @Nullable Integer getNumShards(); + + @SchemaFieldDescription( + "Maximum number of shards a single partition's rows may occupy. Lower values " + + "write fewer files per commit, but also reduces per-partition write parallelism. A value " + + "of 1 pins each partition to one writer. Ignored for unpartitioned tables. Must be between 1 " + + "and `num_shards`; defaults to `num_shards`. Currently only supported in CDC mode.") + public abstract @Nullable Integer getShardsPerPartition(); + + @SchemaFieldDescription( + "How long a late record may lag behind the watermark before it is " + + "dropped entirely, rather than routed to the dead_letter output. Defaults to 21600 " + + "(6 hours). Currently only supported in CDC mode.") + public abstract @Nullable Integer getAllowedLatenessSeconds(); + + @SchemaFieldDescription( + "A stable identifier for this sink, used to namespace the idempotency tokens " + + "written to each commit's Iceberg snapshot summary. Defaults to a unique per-write UUID. " + + "Set it explicitly (and keep it stable across relaunches) for exactly-once commits across " + + "relaunches of a particular streaming write. A batch load with a stable sink_id " + + "commits only once (later batch loads with the same sink_id are skipped). " + + "Currently only supported in CDC mode.") + public abstract @Nullable String getSinkId(); + + @SchemaFieldDescription( + "Streaming only. If set, the sink will emit a periodic empty token-refresh commit while idle, " + + "so its thread of `sink_id` stamped snapshot stays recent and is less likely to be " + + "lost to `expire_snapshots`. Disabled by default. Currently only supported in CDC mode.") + public abstract @Nullable Integer getTokenHeartbeatSeconds(); + + @SchemaFieldDescription( + "Extra key/value properties to add to every commit's Iceberg snapshot summary. " + + "Keys prefixed with 'beam.cdc.' are reserved and rejected. Currently only supported in CDC mode.") + public abstract @Nullable Map getSnapshotProperties(); + + @SchemaFieldDescription( + "Whether and where to output per-record invalid rows (null or missing sequence " + + "value, unknown change type, null equality value, unresolvable destination). Fails the pipeline " + + "if unset (default). Distinct from the `dead_letter` output, which is for late-but-valid " + + "rows. Currently only supported in CDC mode.") + public abstract @Nullable ErrorHandling getErrorHandling(); + + @SchemaFieldDescription( + "The in-memory buffer size (MB) for the pre-write sort; groups larger than this " + + "spill to disk. Must be >= 1. Defaults to 100. Currently only supported in CDC mode.") + public abstract @Nullable Integer getSorterMemoryMb(); + @AutoValue.Builder public abstract static class Builder { public abstract Builder setTable(String table); @@ -220,6 +318,26 @@ public abstract static class Builder { public abstract Builder setPollingBuckets(Integer pollingBuckets); + public abstract Builder setCdc(Cdc cdc); + + public abstract Builder setEqualityColumns(List equalityColumns); + + public abstract Builder setNumShards(Integer numShards); + + public abstract Builder setShardsPerPartition(Integer shardsPerPartition); + + public abstract Builder setAllowedLatenessSeconds(Integer allowedLatenessSeconds); + + public abstract Builder setSinkId(String sinkId); + + public abstract Builder setTokenHeartbeatSeconds(Integer tokenHeartbeatSeconds); + + public abstract Builder setSnapshotProperties(Map snapshotProperties); + + public abstract Builder setErrorHandling(ErrorHandling errorHandling); + + public abstract Builder setSorterMemoryMb(Integer sorterMemoryMb); + public abstract Configuration build(); } @@ -230,6 +348,121 @@ public IcebergCatalogConfig getIcebergCatalog() { .setConfigProperties(getConfigProperties()) .build(); } + + enum Mode { + APPEND, + CDC + } + + /** The write mode this configuration selects. */ + Mode mode() { + return getCdc() == null ? Mode.APPEND : Mode.CDC; + } + + /** + * Options that only one mode supports today; everything else applies to both. Once an option + * becomes available to the other mode, just remove it from here. + */ + private static final ImmutableMap SUPPORTED_MODES = + ImmutableMap.builder() + .put("equality_columns", Mode.CDC) + .put("num_shards", Mode.CDC) + .put("shards_per_partition", Mode.CDC) + .put("allowed_lateness_seconds", Mode.CDC) + .put("sink_id", Mode.CDC) + .put("token_heartbeat_seconds", Mode.CDC) + .put("snapshot_properties", Mode.CDC) + .put("error_handling", Mode.CDC) + .put("sorter_memory_mb", Mode.CDC) + .put("direct_write_byte_limit", Mode.APPEND) + .put("distribution_mode", Mode.APPEND) + .put("autosharding", Mode.APPEND) + .put("write_properties", Mode.APPEND) + .put("using_side_input_table_cache", Mode.APPEND) + .put("table_refresh_interval_seconds", Mode.APPEND) + .put("maximum_cache_size", Mode.APPEND) + .put("polling_buckets", Mode.APPEND) + .build(); + + /** Rejects every set option that the selected mode does not support. */ + void validateModeOptions() { + Mode mode = mode(); + Map values = new LinkedHashMap<>(); + values.put("equality_columns", getEqualityColumns()); + values.put("num_shards", getNumShards()); + values.put("shards_per_partition", getShardsPerPartition()); + values.put("allowed_lateness_seconds", getAllowedLatenessSeconds()); + values.put("sink_id", getSinkId()); + values.put("token_heartbeat_seconds", getTokenHeartbeatSeconds()); + values.put("snapshot_properties", getSnapshotProperties()); + values.put("error_handling", getErrorHandling()); + values.put("sorter_memory_mb", getSorterMemoryMb()); + values.put("direct_write_byte_limit", getDirectWriteByteLimit()); + values.put("distribution_mode", getDistributionMode()); + values.put("autosharding", getAutosharding()); + values.put("write_properties", getWriteProperties()); + values.put("using_side_input_table_cache", getUsingSideInputTableCache()); + values.put("table_refresh_interval_seconds", getTableRefreshIntervalSeconds()); + values.put("maximum_cache_size", getMaximumCacheSize()); + values.put("polling_buckets", getPollingBuckets()); + List invalidOptions = new ArrayList<>(); + for (Map.Entry option : SUPPORTED_MODES.entrySet()) { + String name = option.getKey(); + if (values.get(name) == null || option.getValue() == mode) { + continue; + } + invalidOptions.add(name); + } + if (!invalidOptions.isEmpty()) { + throw new IllegalArgumentException( + String.format( + "The following options are not supported by %s writes yet: %s", + mode.name().toLowerCase(), invalidOptions)); + } + } + } + + /** The options that only make sense for a stream of row-level changes. */ + @DefaultSchema(AutoValueSchema.class) + @AutoValue + public abstract static class Cdc implements Serializable { + public static Builder builder() { + return new AutoValue_IcebergWriteSchemaTransformProvider_Cdc.Builder(); + } + + @SchemaFieldDescription( + "The required column name representing the monotonic sequence number used to order a single key's " + + "changes. Default column name is '_commit_snapshot_sequence_number'. This column will be " + + "stripped from the data row before writing to Iceberg.") + public abstract @Nullable String getSequenceNumberColumn(); + + @SchemaFieldDescription( + "The optional column name representing the row's change type (INSERT, UPDATE_BEFORE, UPDATE_AFTER, " + + "or DELETE). This column will be stripped from the data row before writing to Iceberg. If not " + + "specified, the sink will use the element's native ValueKind") + public abstract @Nullable String getChangeTypeColumn(); + + @SchemaFieldDescription( + "Optional map from the `change_type_column` value to the canonical change type name (see above).") + public abstract @Nullable Map getChangeTypeMap(); + + @SchemaFieldDescription( + "If true, only the after-image of each change (INSERT/UPDATE_AFTER) is applied as an upsert. " + + "UPDATE_BEFORE records are dropped. Default: false") + public abstract @Nullable Boolean getUpsert(); + + @AutoValue.Builder + public abstract static class Builder { + public abstract Builder setSequenceNumberColumn(String sequenceNumberColumn); + + public abstract Builder setChangeTypeColumn(String changeTypeColumn); + + public abstract Builder setChangeTypeMap(Map changeTypeMap); + + public abstract Builder setUpsert(Boolean upsert); + + public abstract Cdc build(); + } } @Override @@ -244,7 +477,7 @@ public List inputCollectionNames() { @Override public List outputCollectionNames() { - return Collections.singletonList(SNAPSHOTS_TAG); + return Arrays.asList(SNAPSHOTS_TAG, DEAD_LETTER_TAG, ERRORS_TAG); } @Override @@ -276,7 +509,12 @@ Row getConfigurationRow() { @Override public PCollectionRowTuple expand(PCollectionRowTuple input) { PCollection rows = input.get(INPUT_TAG); + configuration.validateModeOptions(); + @Nullable Cdc cdc = configuration.getCdc(); + return cdc == null ? expandAppend(rows) : expandCdc(rows, cdc); + } + private PCollectionRowTuple expandAppend(PCollection rows) { IcebergIO.WriteRows writeTransform = IcebergIO.writeRows(configuration.getIcebergCatalog()) .to( @@ -360,6 +598,129 @@ public PCollectionRowTuple expand(PCollectionRowTuple input) { return PCollectionRowTuple.of(SNAPSHOTS_TAG, snapshots); } + private PCollectionRowTuple expandCdc(PCollection rows, Cdc cdc) { + Schema inputSchema = rows.getSchema(); + @Nullable List drop = configuration.getDrop(); + @Nullable List keep = configuration.getKeep(); + @Nullable String only = configuration.getOnly(); + @Nullable String changeTypeColumn = cdc.getChangeTypeColumn(); + @Nullable String configuredSeq = cdc.getSequenceNumberColumn(); + String seqColumn = configuredSeq != null ? configuredSeq : DEFAULT_SEQUENCE_NUMBER_COLUMN; + + // The sink reads the control columns from the raw element, so by default they are dropped + // from the written row. Listing them in keep writes them too. + if (keep == null && only == null) { + Set effectiveDrop = + new LinkedHashSet<>(controlColumnsPresent(inputSchema, changeTypeColumn, seqColumn)); + if (drop != null) { + effectiveDrop.addAll(drop); + } + drop = effectiveDrop.isEmpty() ? null : new ArrayList<>(effectiveDrop); + } + + WriteCdcRows write = + IcebergIO.writeCdcRows(configuration.getIcebergCatalog()) + .to( + new PortableIcebergDestinations( + configuration.getTable(), + FileFormat.PARQUET.toString(), + inputSchema, + configuration.getPartitionFields(), + configuration.getSortFields(), + configuration.getTableProperties(), + drop, + keep, + only)); + IcebergWriteResult result = rows.apply(applyOptions(write, cdc)); + + PCollection snapshots = + result + .getSnapshots() + .apply(MapElements.via(new SnapshotToRow())) + .setRowSchema(OUTPUT_SCHEMA); + PCollectionRowTuple output = + PCollectionRowTuple.of(SNAPSHOTS_TAG, snapshots) + .and(DEAD_LETTER_TAG, result.getDeadLetterRows()); + @Nullable ErrorHandling errorHandling = configuration.getErrorHandling(); + if (ErrorHandling.hasOutput(errorHandling)) { + output = output.and(checkStateNotNull(errorHandling).getOutput(), result.getFailedRows()); + } + return output; + } + + /** Threads every set option onto {@code write}. */ + private WriteCdcRows applyOptions(WriteCdcRows write, Cdc cdc) { + @Nullable List equalityColumns = configuration.getEqualityColumns(); + if (equalityColumns != null) { + write = write.withEqualityColumns(equalityColumns); + } + @Nullable String sequenceNumberColumn = cdc.getSequenceNumberColumn(); + if (sequenceNumberColumn != null) { + write = write.withSequenceNumberColumn(sequenceNumberColumn); + } + @Nullable String changeTypeColumn = cdc.getChangeTypeColumn(); + if (changeTypeColumn != null) { + write = write.withChangeTypeColumn(changeTypeColumn); + } + @Nullable Map changeTypeMap = cdc.getChangeTypeMap(); + if (changeTypeMap != null) { + write = write.withChangeTypeMap(changeTypeMap); + } + @Nullable Boolean upsert = cdc.getUpsert(); + if (upsert != null) { + write = write.withUpsert(upsert); + } + @Nullable Integer sorterMemoryMb = configuration.getSorterMemoryMb(); + if (sorterMemoryMb != null) { + write = write.withSorterMemoryMB(sorterMemoryMb); + } + @Nullable Integer numShards = configuration.getNumShards(); + if (numShards != null) { + write = write.withNumShards(numShards); + } + @Nullable Integer shardsPerPartition = configuration.getShardsPerPartition(); + if (shardsPerPartition != null) { + write = write.withShardsPerPartition(shardsPerPartition); + } + @Nullable String sinkId = configuration.getSinkId(); + if (sinkId != null) { + write = write.withSinkId(sinkId); + } + @Nullable Integer triggeringFrequencySeconds = configuration.getTriggeringFrequencySeconds(); + if (triggeringFrequencySeconds != null) { + write = write.withTriggeringFrequency(Duration.standardSeconds(triggeringFrequencySeconds)); + } + @Nullable Integer allowedLatenessSeconds = configuration.getAllowedLatenessSeconds(); + if (allowedLatenessSeconds != null) { + write = write.withAllowedLateness(Duration.standardSeconds(allowedLatenessSeconds)); + } + if (ErrorHandling.hasOutput(configuration.getErrorHandling())) { + write = write.withErrorHandling(); + } + @Nullable Map snapshotProperties = configuration.getSnapshotProperties(); + if (snapshotProperties != null) { + write = write.withSnapshotProperties(snapshotProperties); + } + @Nullable Integer tokenHeartbeatSeconds = configuration.getTokenHeartbeatSeconds(); + if (tokenHeartbeatSeconds != null) { + write = write.withTokenHeartbeat(Duration.standardSeconds(tokenHeartbeatSeconds)); + } + return write; + } + + /** The control columns present in the input; a missing one is left to the sink to report. */ + private static List controlColumnsPresent( + Schema inputSchema, @Nullable String changeTypeColumn, String seqColumn) { + List controls = new ArrayList<>(); + if (changeTypeColumn != null && inputSchema.hasField(changeTypeColumn)) { + controls.add(changeTypeColumn); + } + if (inputSchema.hasField(seqColumn)) { + controls.add(seqColumn); + } + return controls; + } + @VisibleForTesting static class SnapshotToRow extends SimpleFunction, Row> { @Override diff --git a/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergSchemaTransformTranslationTest.java b/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergSchemaTransformTranslationTest.java index 675b4aafe76a..553a768a5cc1 100644 --- a/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergSchemaTransformTranslationTest.java +++ b/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergSchemaTransformTranslationTest.java @@ -21,6 +21,7 @@ import static org.apache.beam.sdk.io.iceberg.IcebergReadSchemaTransformProvider.IcebergReadSchemaTransform; import static org.apache.beam.sdk.io.iceberg.IcebergWriteSchemaTransformProvider.INPUT_TAG; import static org.apache.beam.sdk.io.iceberg.IcebergWriteSchemaTransformProvider.IcebergWriteSchemaTransform; +import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull; import static org.junit.Assert.assertEquals; import java.io.IOException; @@ -98,6 +99,27 @@ public class IcebergSchemaTransformTranslationTest { .withFieldValue("keep", Collections.singletonList("str")) .build(); + /** A CDC write config: the nested {@code cdc} row plus the options gated to CDC mode. */ + private static final Row WRITE_CDC_CONFIG_ROW = + Row.withSchema(WRITE_PROVIDER.configurationSchema()) + .withFieldValue("table", "test_table_identifier") + .withFieldValue("catalog_properties", CATALOG_PROPERTIES) + .withFieldValue("equality_columns", Collections.singletonList("id")) + .withFieldValue("sink_id", "stable-sink") + .withFieldValue( + "cdc", + Row.withSchema( + checkStateNotNull( + WRITE_PROVIDER + .configurationSchema() + .getField("cdc") + .getType() + .getRowSchema())) + .withFieldValue("change_type_column", "op") + .withFieldValue("upsert", true) + .build()) + .build(); + private static final Row READ_CONFIG_ROW = Row.withSchema(READ_PROVIDER.configurationSchema()) .withFieldValue("table", "test_table_identifier") @@ -136,6 +158,21 @@ public void testReCreateWriteTransformFromRow() { assertEquals(WRITE_CONFIG_ROW, writeTransformFromRow.getConfigurationRow()); } + @Test + public void testReCreateCdcWriteTransformFromRow() { + IcebergWriteSchemaTransform writeTransform = + (IcebergWriteSchemaTransform) WRITE_PROVIDER.from(WRITE_CDC_CONFIG_ROW); + + IcebergSchemaTransformTranslation.IcebergWriteSchemaTransformTranslator translator = + new IcebergSchemaTransformTranslation.IcebergWriteSchemaTransformTranslator(); + Row row = translator.toConfigRow(writeTransform); + + IcebergWriteSchemaTransform writeTransformFromRow = + translator.fromConfigRow(row, PipelineOptionsFactory.create()); + + assertEquals(WRITE_CDC_CONFIG_ROW, writeTransformFromRow.getConfigurationRow()); + } + @Test public void testWriteTransformProtoTranslation() throws InvalidProtocolBufferException, IOException { diff --git a/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergWriteSchemaTransformProviderCdcTest.java b/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergWriteSchemaTransformProviderCdcTest.java new file mode 100644 index 000000000000..ad59436e6f4b --- /dev/null +++ b/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergWriteSchemaTransformProviderCdcTest.java @@ -0,0 +1,666 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.iceberg; + +import static org.apache.beam.sdk.io.iceberg.IcebergWriteSchemaTransformProvider.Cdc; +import static org.apache.beam.sdk.io.iceberg.IcebergWriteSchemaTransformProvider.Configuration; +import static org.apache.beam.sdk.io.iceberg.IcebergWriteSchemaTransformProvider.DEAD_LETTER_TAG; +import static org.apache.beam.sdk.io.iceberg.IcebergWriteSchemaTransformProvider.INPUT_TAG; +import static org.apache.beam.sdk.io.iceberg.IcebergWriteSchemaTransformProvider.SNAPSHOTS_TAG; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.equalTo; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.util.List; +import java.util.Map; +import org.apache.beam.sdk.managed.Managed; +import org.apache.beam.sdk.schemas.Schema; +import org.apache.beam.sdk.schemas.transforms.SchemaTransform; +import org.apache.beam.sdk.schemas.transforms.providers.ErrorHandling; +import org.apache.beam.sdk.testing.PAssert; +import org.apache.beam.sdk.testing.TestPipeline; +import org.apache.beam.sdk.transforms.Create; +import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.values.KV; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.PCollectionRowTuple; +import org.apache.beam.sdk.values.Row; +import org.apache.beam.sdk.values.ValueKind; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.iceberg.CatalogUtil; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Table; +import org.apache.iceberg.catalog.Catalog; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.data.IcebergGenerics; +import org.apache.iceberg.hadoop.HadoopCatalog; +import org.apache.iceberg.types.Types; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Tests for the {@code cdc} mode of {@link IcebergWriteSchemaTransformProvider}. End-to-end tests + * drive a real {@link HadoopCatalog} V2 table; each test uses a unique table identifier so the + * process-wide TableCache never sees a repeat. + */ +@RunWith(JUnit4.class) +public class IcebergWriteSchemaTransformProviderCdcTest { + + @Rule public transient TestPipeline p = TestPipeline.create(); + @Rule public transient TemporaryFolder tmp = new TemporaryFolder(); + + /** Canonical test table schema (id INT primary key, name/data STRING). */ + private static final org.apache.iceberg.Schema ICEBERG_SCHEMA = + new org.apache.iceberg.Schema( + Types.NestedField.required(1, "id", Types.IntegerType.get()), + Types.NestedField.optional(2, "name", Types.StringType.get()), + Types.NestedField.optional(3, "data", Types.StringType.get())); + + private static final Schema DATA_SCHEMA = + Schema.builder() + .addInt32Field("id") + .addNullableField("name", Schema.FieldType.STRING) + .addNullableField("data", Schema.FieldType.STRING) + .build(); + + /** The default sequence-number column name (see {@code cdc.sink.CdcWriteConfig}). */ + private static final String DEFAULT_SEQ_COL = "_commit_snapshot_sequence_number"; + + /** Input schema = data schema + a {@code change_type} column (string op code). */ + private static final Schema INPUT_SCHEMA_WITH_CHANGE_TYPE = + Schema.builder().addFields(DATA_SCHEMA.getFields()).addStringField("change_type").build(); + + /** {@link #INPUT_SCHEMA_WITH_CHANGE_TYPE} + the default sequence-number column. */ + private static final Schema INPUT_SCHEMA_WITH_CHANGE_TYPE_AND_SEQ = + Schema.builder() + .addFields(INPUT_SCHEMA_WITH_CHANGE_TYPE.getFields()) + .addInt64Field(DEFAULT_SEQ_COL) + .build(); + + private Catalog catalog; + private String warehousePath; + + @Before + public void setUp() { + warehousePath = tmp.getRoot().getAbsolutePath(); + catalog = new HadoopCatalog(new org.apache.hadoop.conf.Configuration(), warehousePath); + } + + private Map catalogProperties() { + return ImmutableMap.of( + "type", CatalogUtil.ICEBERG_CATALOG_TYPE_HADOOP, "warehouse", "file:" + warehousePath); + } + + /** Creates a fresh unpartitioned V2 table (PK = {@code id}) with a unique identifier. */ + private TableIdentifier v2Table() { + TableIdentifier id = TableIdentifier.of("db", "t" + System.nanoTime()); + createV2Table(id); + return id; + } + + private void createV2Table(TableIdentifier id) { + org.apache.iceberg.Schema schemaWithIds = + new org.apache.iceberg.Schema(ICEBERG_SCHEMA.columns(), ImmutableSet.of(1)); + catalog.createTable( + id, schemaWithIds, PartitionSpec.unpartitioned(), ImmutableMap.of("format-version", "2")); + } + + /** A config builder pre-wired to this test's catalog and a fresh single table. */ + private Configuration.Builder configFor(TableIdentifier id) { + return Configuration.builder() + .setTable(id.toString()) + .setCatalogProperties(catalogProperties()); + } + + /** {@link #configFor} in CDC mode, with the standard {@code change_type} op column. */ + private Configuration.Builder cdcConfigFor(TableIdentifier id) { + return configFor(id).setCdc(changeTypeCdc().build()); + } + + /** A {@code cdc} block reading kinds from the standard {@code change_type} op column. */ + private static Cdc.Builder changeTypeCdc() { + return Cdc.builder().setChangeTypeColumn("change_type"); + } + + /** A CDC config builder for the {@code db.{name}} destination template. */ + private Configuration.Builder templateConfig() { + return Configuration.builder() + .setTable("db.{name}") + .setCatalogProperties(catalogProperties()) + .setCdc(changeTypeCdc().build()); + } + + /** + * Two fresh V2 tables, {@code db.a_} and {@code db.b_}, for the template path. + */ + private List templateTables(String suffix) { + TableIdentifier idA = TableIdentifier.of("db", "a_" + suffix); + TableIdentifier idB = TableIdentifier.of("db", "b_" + suffix); + createV2Table(idA); + createV2Table(idB); + return ImmutableList.of(catalog.loadTable(idA), catalog.loadTable(idB)); + } + + /** The provider's {@link SchemaTransform} for {@code config}. */ + private static SchemaTransform transformFor(Configuration config) { + return new IcebergWriteSchemaTransformProvider().from(config); + } + + /** Applies {@code config}'s CDC write to {@code rows}, returning the output tuple. */ + private PCollectionRowTuple applyCdcWrite(Configuration config, Schema schema, Row... rows) { + PCollection input = p.apply(Create.of(ImmutableList.copyOf(rows)).withRowSchema(schema)); + return PCollectionRowTuple.of(INPUT_TAG, input).apply("CdcWrite", transformFor(config)); + } + + /** {@link #applyCdcWrite} followed by a pipeline run to completion. */ + private void runCdcWrite(Configuration config, Schema schema, Row... rows) { + applyCdcWrite(config, schema, rows); + p.run().waitUntilFinish(); + } + + /** Reads all live rows of {@code table} as sorted {@code id:name:data} strings. */ + private static List readRows(Table table) { + table.refresh(); + return ImmutableList.copyOf(IcebergGenerics.read(table).build()).stream() + .map(r -> r.getField("id") + ":" + r.getField("name") + ":" + r.getField("data")) + .sorted() + .collect(ImmutableList.toImmutableList()); + } + + /** An input row with a {@code change_type} op column and an explicit sequence number. */ + private static Row rowWithSeq(int id, String name, String data, String changeType, long seq) { + return Row.withSchema(INPUT_SCHEMA_WITH_CHANGE_TYPE_AND_SEQ) + .addValues(id, name, data, changeType, seq) + .build(); + } + + // --------------------------------------------------------------------------------------------- + // Config values with an observable effect + // --------------------------------------------------------------------------------------------- + + /** + * {@code equality_columns} is the most dangerous field to lose (a drop falls back to the table's + * identifier fields and deletes key on the WRONG column): the identifier field is {@code id}, + * {@code equality_columns} names {@code code}, and the DELETE carries a non-matching {@code id} , + * only the configured key can apply it. + */ + @Test + public void equalityColumnsConfigDecidesTheDeleteKey() { + TableIdentifier id = TableIdentifier.of("db", "eqcols" + System.nanoTime()); + org.apache.iceberg.Schema schema = + new org.apache.iceberg.Schema( + ImmutableList.of( + Types.NestedField.required(1, "id", Types.IntegerType.get()), + Types.NestedField.required(2, "code", Types.StringType.get()), + Types.NestedField.optional(3, "val", Types.StringType.get())), + ImmutableSet.of(1)); // identifier field = id, deliberately NOT the equality column + catalog.createTable( + id, schema, PartitionSpec.unpartitioned(), ImmutableMap.of("format-version", "2")); + Table table = catalog.loadTable(id); + + Schema inputSchema = + Schema.builder() + .addInt32Field("id") + .addStringField("code") + .addNullableField("val", Schema.FieldType.STRING) + .addStringField("change_type") + .addInt64Field(DEFAULT_SEQ_COL) + .build(); + + runCdcWrite( + cdcConfigFor(id).setEqualityColumns(ImmutableList.of("code")).build(), + inputSchema, + Row.withSchema(inputSchema).addValues(1, "k1", "x", "INSERT", 1L).build(), + Row.withSchema(inputSchema).addValues(2, "k2", "y", "INSERT", 1L).build(), + // A DELETE whose id (99) matches nothing: only the `code` key can apply it. + Row.withSchema(inputSchema).addValues(99, "k1", "x", "DELETE", 2L).build()); + + table.refresh(); + assertThat( + ImmutableList.copyOf(IcebergGenerics.read(table).build()).stream() + .map(r -> r.getField("id") + ":" + r.getField("code")) + .collect(ImmutableList.toImmutableList()), + containsInAnyOrder("2:k2")); + } + + // --------------------------------------------------------------------------------------------- + // Projection + // --------------------------------------------------------------------------------------------- + + /** + * The control columns are dropped from the written row by default; listing them in {@code keep} + * writes their raw values too, for tables that carry a last-change or last-sequence column. + */ + @Test + public void keepCanWriteTheControlColumns() { + TableIdentifier id = TableIdentifier.of("db", "ctl" + System.nanoTime()); + org.apache.iceberg.Schema schema = + new org.apache.iceberg.Schema( + ImmutableList.of( + Types.NestedField.required(1, "id", Types.IntegerType.get()), + Types.NestedField.optional(2, "name", Types.StringType.get()), + Types.NestedField.optional(3, "data", Types.StringType.get()), + Types.NestedField.required(4, "change_type", Types.StringType.get()), + Types.NestedField.required(5, DEFAULT_SEQ_COL, Types.LongType.get())), + ImmutableSet.of(1)); + catalog.createTable( + id, schema, PartitionSpec.unpartitioned(), ImmutableMap.of("format-version", "2")); + Table table = catalog.loadTable(id); + + runCdcWrite( + cdcConfigFor(id) + .setKeep(ImmutableList.of("id", "name", "data", "change_type", DEFAULT_SEQ_COL)) + .build(), + INPUT_SCHEMA_WITH_CHANGE_TYPE_AND_SEQ, + rowWithSeq(1, "a", "x", "INSERT", 1L), + rowWithSeq(1, "a", "z", "UPDATE_AFTER", 2L), + rowWithSeq(2, "b", "y", "INSERT", 1L), + rowWithSeq(2, "b", "y", "DELETE", 2L)); + + // The sink still read both columns: id=1 shows its seq=2 image and id=2 was deleted. + table.refresh(); + assertThat( + ImmutableList.copyOf(IcebergGenerics.read(table).build()).stream() + .map( + r -> + r.getField("id") + + ":" + + r.getField("data") + + ":" + + r.getField("change_type") + + ":" + + r.getField(DEFAULT_SEQ_COL)) + .collect(ImmutableList.toImmutableList()), + equalTo(ImmutableList.of("1:z:UPDATE_AFTER:2"))); + } + + /** + * The unnested dead-letter shape re-feeds into a single-table sink: {@code destination} dropped + * by the projection, the control columns consumed (then stripped) by the sink. + */ + @Test + public void singleTableReplaysDeadLetterShape() { + TableIdentifier id = v2Table(); + Table table = catalog.loadTable(id); + + Schema replaySchema = + Schema.builder() + .addFields(DATA_SCHEMA.getFields()) + .addStringField("change_type") + .addInt64Field("sequence_number") + .addStringField("destination") + .build(); + runCdcWrite( + configFor(id) + .setCdc(changeTypeCdc().setSequenceNumberColumn("sequence_number").build()) + .setDrop(ImmutableList.of("destination")) + .build(), + replaySchema, + Row.withSchema(replaySchema).addValues(1, "a", "x", "INSERT", 1L, id.toString()).build()); + + assertThat(readRows(table), equalTo(ImmutableList.of("1:a:x"))); + } + + /** + * A {@code keep} whitelist that omits the control columns must still let both through to the + * sink: the UPDATE lands and the DELETE deletes only if kinds and order arrived. + */ + @Test + public void controlColumnsSurviveKeepProjection() { + TableIdentifier id = v2Table(); + Table table = catalog.loadTable(id); + + Schema inputSchema = + Schema.builder() + .addFields(INPUT_SCHEMA_WITH_CHANGE_TYPE_AND_SEQ.getFields()) + .addStringField("source") + .build(); + + // keep lists only the data columns: no control columns, no "source" envelope column. + runCdcWrite( + cdcConfigFor(id).setKeep(ImmutableList.of("id", "name", "data")).build(), + inputSchema, + Row.withSchema(inputSchema).addValues(1, "a", "x", "INSERT", 1L, "m").build(), + Row.withSchema(inputSchema).addValues(2, "b", "y", "INSERT", 1L, "m").build(), + Row.withSchema(inputSchema).addValues(1, "a", "x", "UPDATE_BEFORE", 2L, "m").build(), + Row.withSchema(inputSchema).addValues(1, "a", "z", "UPDATE_AFTER", 2L, "m").build(), + Row.withSchema(inputSchema).addValues(2, "b", "y", "DELETE", 2L, "m").build()); + + assertThat(readRows(table), equalTo(ImmutableList.of("1:a:z"))); + } + + /** + * An empty {@code cdc} block enables CDC with every default: kinds come from each element's + * native {@link ValueKind}, which the projection must preserve (a plain {@code output()} re-emit + * would stamp everything INSERT). + */ + @Test + public void emptyCdcBlockUsesDefaultsAndNativeValueKinds() { + TableIdentifier id = v2Table(); + Table table = catalog.loadTable(id); + + Schema inputSchema = + Schema.builder() + .addFields(DATA_SCHEMA.getFields()) + .addInt64Field(DEFAULT_SEQ_COL) + .addStringField("source") + .build(); + + // No change_type_column: change kinds come only from each element's native ValueKind. + SchemaTransform transform = + transformFor( + configFor(id) + .setCdc(Cdc.builder().build()) + .setDrop(ImmutableList.of("source")) + .build()); + + PCollection input = + withKinds( + p.apply( + Create.of( + ImmutableList.of( + KV.of(ValueKind.INSERT, kindRow(inputSchema, 1, "a", "x", 1L)), + KV.of(ValueKind.INSERT, kindRow(inputSchema, 2, "b", "y", 1L)), + KV.of(ValueKind.UPDATE_BEFORE, kindRow(inputSchema, 1, "a", "x", 2L)), + KV.of(ValueKind.UPDATE_AFTER, kindRow(inputSchema, 1, "a", "z", 2L)), + KV.of(ValueKind.DELETE, kindRow(inputSchema, 2, "b", "y", 2L)))))) + .setRowSchema(inputSchema); + + PCollectionRowTuple.of(INPUT_TAG, input).apply("CdcWrite", transform); + p.run().waitUntilFinish(); + + // id=1 updated, id=2 deleted , possible only if the native kinds survived the ParDo. + assertThat(readRows(table), equalTo(ImmutableList.of("1:a:z"))); + } + + /** Mirrors {@code cdc.sink.CdcSinkTestUtils#withKinds} (package-private there). */ + private static PCollection withKinds(PCollection> tagged) { + return tagged.apply( + "AttachKinds", + ParDo.of( + new DoFn, Row>() { + @ProcessElement + public void process(@Element KV e, OutputReceiver out) { + out.builder(e.getValue()).setValueKind(e.getKey()).output(); + } + })); + } + + private static Row kindRow(Schema schema, int id, String name, String data, long seq) { + return Row.withSchema(schema).addValues(id, name, data, seq, "src").build(); + } + + /** + * The {@code only} projection extracts a nested payload row (the Debezium {@code after} pattern) + * while carrying the top-level control columns through to the sink. + */ + @Test + public void onlyProjectionExtractsPayloadAndPreservesControlColumns() { + TableIdentifier id = v2Table(); + Table table = catalog.loadTable(id); + + Schema envelopeSchema = envelopeSchema(/* nullablePayload= */ false); + + runCdcWrite( + configFor(id) + .setOnly("after") + .setCdc( + Cdc.builder() + .setChangeTypeColumn("op") + .setChangeTypeMap( + ImmutableMap.of("c", "INSERT", "u", "UPDATE_AFTER", "d", "DELETE")) + .setSequenceNumberColumn("seq") + .setUpsert(true) + .build()) + .build(), + envelopeSchema, + envelopeRow(envelopeSchema, 1, "a", "x", "c", 1L), + envelopeRow(envelopeSchema, 2, "b", "y", "c", 1L), + envelopeRow(envelopeSchema, 1, "a", "z", "u", 2L), + envelopeRow(envelopeSchema, 2, "b", "y", "d", 2L)); + + // "u" upserted id=1, "d" deleted id=2; ts_ms and op never reached the table. + assertThat(readRows(table), equalTo(ImmutableList.of("1:a:z"))); + } + + /** A Debezium-ish envelope: {@code after} payload row + {@code op}/{@code seq}/{@code ts_ms}. */ + private static Schema envelopeSchema(boolean nullablePayload) { + Schema.Builder builder = Schema.builder(); + if (nullablePayload) { + builder.addNullableField("after", Schema.FieldType.row(DATA_SCHEMA)); + } else { + builder.addRowField("after", DATA_SCHEMA); + } + return builder.addStringField("op").addInt64Field("seq").addInt64Field("ts_ms").build(); + } + + private static Row envelopeRow( + Schema envelopeSchema, int id, String name, String data, String op, long seq) { + return Row.withSchema(envelopeSchema) + .addValues(Row.withSchema(DATA_SCHEMA).addValues(id, name, data).build(), op, seq, 1234L) + .build(); + } + + /** + * {@code drop} removes extra Debezium-ish envelope columns on the templated path too, so the + * write succeeds and only the data columns land. + */ + @Test + public void dynamicDestinationDropsExtraEnvelopeColumns() { + String suffix = "t" + System.nanoTime(); + List
tables = templateTables(suffix); + + // Input carries two extra Debezium envelope columns not present in the table. + Schema envelopeSchema = + Schema.builder() + .addFields(INPUT_SCHEMA_WITH_CHANGE_TYPE_AND_SEQ.getFields()) + .addInt64Field("ts_ms") + .addStringField("source") + .build(); + + runCdcWrite( + templateConfig().setDrop(ImmutableList.of("ts_ms", "source")).build(), + envelopeSchema, + Row.withSchema(envelopeSchema) + .addValues(1, "a_" + suffix, "x", "INSERT", 1L, 1234L, "mysql") + .build(), + Row.withSchema(envelopeSchema) + .addValues(2, "b_" + suffix, "y", "INSERT", 1L, 5678L, "postgres") + .build()); + + // The extra envelope columns were dropped; only id:name:data landed. + assertThat(readRows(tables.get(0)), equalTo(ImmutableList.of("1:a_" + suffix + ":x"))); + assertThat(readRows(tables.get(1)), equalTo(ImmutableList.of("2:b_" + suffix + ":y"))); + } + + // --------------------------------------------------------------------------------------------- + // Destinations: single table and dynamic templates + // --------------------------------------------------------------------------------------------- + + /** + * A destination template routes by a data column, and each table sees its own changes applied: + * the sink consumes both control columns, so neither leaks into the written data. + */ + @Test + public void dynamicDestinationTemplateRoutesAndAppliesChanges() { + String suffix = "t" + System.nanoTime(); + List
tables = templateTables(suffix); + + PCollectionRowTuple output = + applyCdcWrite( + templateConfig().build(), + INPUT_SCHEMA_WITH_CHANGE_TYPE_AND_SEQ, + rowWithSeq(1, "a_" + suffix, "x", "INSERT", 1L), + rowWithSeq(2, "b_" + suffix, "y", "INSERT", 1L), + rowWithSeq(1, "a_" + suffix, "z", "UPDATE_AFTER", 2L)); + assertTrue(output.has(SNAPSHOTS_TAG)); + assertTrue(output.has(DEAD_LETTER_TAG)); + + p.run().waitUntilFinish(); + + // id=1 reflects the seq=2 UPDATE_AFTER; no control column leaked into either table. + assertThat(readRows(tables.get(0)), equalTo(ImmutableList.of("1:a_" + suffix + ":z"))); + assertThat(readRows(tables.get(1)), equalTo(ImmutableList.of("2:b_" + suffix + ":y"))); + } + + /** + * A destination template may reference a field the projection drops: routing sees the raw record + * while the written row is the filtered one, so a routing-only column never lands in the table. + */ + @Test + public void templateRoutesByDroppedField() { + String suffix = "t" + System.nanoTime(); + List
tables = templateTables(suffix); + Schema inputSchema = + Schema.builder() + .addFields(INPUT_SCHEMA_WITH_CHANGE_TYPE_AND_SEQ.getFields()) + .addStringField("source") + .build(); + + runCdcWrite( + templateConfig().setTable("db.{source}").setDrop(ImmutableList.of("source")).build(), + inputSchema, + Row.withSchema(inputSchema).addValues(1, "a", "x", "INSERT", 1L, "a_" + suffix).build(), + Row.withSchema(inputSchema).addValues(2, "b", "y", "INSERT", 1L, "b_" + suffix).build()); + + assertThat(readRows(tables.get(0)), equalTo(ImmutableList.of("1:a:x"))); + assertThat(readRows(tables.get(1)), equalTo(ImmutableList.of("2:b:y"))); + } + + // --------------------------------------------------------------------------------------------- + // Change kinds and defaults + // --------------------------------------------------------------------------------------------- + + @Test + public void schemaTransformAppliesCdcEndToEnd() { + TableIdentifier id = v2Table(); + Table table = catalog.loadTable(id); + + PCollectionRowTuple output = + applyCdcWrite( + cdcConfigFor(id).build(), + INPUT_SCHEMA_WITH_CHANGE_TYPE_AND_SEQ, + rowWithSeq(1, "a", "x", "INSERT", 1L), + rowWithSeq(2, "b", "y", "INSERT", 1L), + rowWithSeq(1, "a", "x", "UPDATE_BEFORE", 2L), + rowWithSeq(1, "a", "z", "UPDATE_AFTER", 2L), + rowWithSeq(2, "b", "y", "DELETE", 2L)); + + // Output tags are pinned: snapshots + dead_letter, and no error output when error_handling + // is not configured. + assertTrue(output.has(SNAPSHOTS_TAG)); + assertTrue(output.has(DEAD_LETTER_TAG)); + assertFalse(output.has("errors")); + + PAssert.that(output.get(DEAD_LETTER_TAG)).empty(); + + p.run().waitUntilFinish(); + + // id=1 updated to (a,z); id=2 deleted. + assertThat(readRows(table), equalTo(ImmutableList.of("1:a:z"))); + } + + // --------------------------------------------------------------------------------------------- + // Error handling + // --------------------------------------------------------------------------------------------- + + /** + * With {@code error_handling}, an invalid record (unknown change-type value) is routed to the + * configured named output; the good records commit. + */ + @Test + public void errorHandlingRoutesInvalidRecords() { + TableIdentifier id = v2Table(); + Table table = catalog.loadTable(id); + + PCollectionRowTuple output = + applyCdcWrite( + cdcConfigFor(id) + .setErrorHandling(ErrorHandling.builder().setOutput("errors").build()) + .build(), + INPUT_SCHEMA_WITH_CHANGE_TYPE_AND_SEQ, + rowWithSeq(1, "a", "x", "INSERT", 1L), + rowWithSeq(2, "b", "y", "BOGUS", 1L), + rowWithSeq(3, "c", "z", "INSERT", 1L)); + assertTrue(output.has("errors")); + assertTrue(output.has(SNAPSHOTS_TAG)); + assertTrue(output.has(DEAD_LETTER_TAG)); + + PAssert.that(output.get("errors")) + .satisfies( + rows -> { + Row err = ImmutableList.copyOf(rows).get(0); + Row failedRow = err.getRow("failed_row"); + assertNotNull(failedRow); + assertEquals(Integer.valueOf(2), failedRow.getInt32("id")); + return null; + }); + + p.run().waitUntilFinish(); + + assertThat(readRows(table), containsInAnyOrder("1:a:x", "3:c:z")); + } + + // --------------------------------------------------------------------------------------------- + // Managed + // --------------------------------------------------------------------------------------------- + + /** + * {@code Managed.write(Managed.ICEBERG)} with a {@code cdc} map writes CDC end-to-end through the + * snake_case config surface. + */ + @Test + public void managedIcebergWithCdcMapWritesEndToEnd() { + TableIdentifier id = v2Table(); + Table table = catalog.loadTable(id); + + Map configMap = + ImmutableMap.builder() + .put("table", id.toString()) + .put("catalog_properties", catalogProperties()) + .put("cdc", ImmutableMap.of("change_type_column", "change_type")) + .build(); + + PCollection input = + p.apply( + Create.of( + rowWithSeq(1, "a", "x", "INSERT", 1L), + rowWithSeq(1, "a", "x", "UPDATE_BEFORE", 2L), + rowWithSeq(1, "a", "z", "UPDATE_AFTER", 2L))) + .setRowSchema(INPUT_SCHEMA_WITH_CHANGE_TYPE_AND_SEQ); + + PCollectionRowTuple output = input.apply(Managed.write(Managed.ICEBERG).withConfig(configMap)); + assertTrue(output.has(SNAPSHOTS_TAG)); + assertTrue(output.has(DEAD_LETTER_TAG)); + + p.run().waitUntilFinish(); + + assertThat(readRows(table), equalTo(ImmutableList.of("1:a:z"))); + } +} From 822e4188a32da8af8b19a5909a9dde760484d5cb Mon Sep 17 00:00:00 2001 From: Ahmed Abualsaud Date: Tue, 22 Sep 2026 16:05:54 -0700 Subject: [PATCH 7/9] YAML and Changes --- CHANGES.md | 1 + sdks/python/apache_beam/yaml/yaml_io.py | 65 ++++++++++++++++++++++++- 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 4866998bb05f..7e57804c4bd0 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -102,6 +102,7 @@ * ClickHouseIO: support writing `Decimal(P, S)` / `Decimal32/64/128/256` columns (Java) ([#39840](https://github.com/apache/beam/issues/39840)). * SolaceIO now supports reading and writing user properties (message metadata) (Java) ([#40099](https://github.com/apache/beam/issues/40099)). * [IcebergIO] AddFiles (`IcebergAddFiles` in YAML) can evolve the table schema before registering files, with `schema_evolution_options`, `required_columns`, `incompatible_schema_handling` and `unverifiable_file_handling` (Java/YAML, batch only) ([#40144](https://github.com/apache/beam/issues/40144)). +* [IcebergIO] Added batch and streaming CDC writes that applies INSERT/UPDATE_BEFORE/UPDATE_AFTER/DELETE changes to Iceberg V2+ tables by primary key. Invoke with `IcebergIO.writeCdcRows` (Java) or by setting the `cdc` option of Managed `ICEBERG` write (Java, Python, YAML) ([#X](https://github.com/apache/beam/issues/X)). ## New Features / Improvements diff --git a/sdks/python/apache_beam/yaml/yaml_io.py b/sdks/python/apache_beam/yaml/yaml_io.py index b7ce98b35997..166e2f053bd0 100644 --- a/sdks/python/apache_beam/yaml/yaml_io.py +++ b/sdks/python/apache_beam/yaml/yaml_io.py @@ -651,6 +651,16 @@ def write_to_iceberg( only: Optional[str] = None, distribution_mode: Optional[str] = None, autosharding: Optional[bool] = None, + cdc: Optional[Mapping[str, Any]] = None, + equality_columns: Optional[Iterable[str]] = None, + num_shards: Optional[int] = None, + shards_per_partition: Optional[int] = None, + allowed_lateness_seconds: Optional[int] = None, + sink_id: Optional[str] = None, + token_heartbeat_seconds: Optional[int] = None, + snapshot_properties: Optional[Mapping[str, str]] = None, + error_handling: Optional[Mapping[str, Any]] = None, + sorter_memory_mb: Optional[int] = None, ): # TODO(robertwb): It'd be nice to derive this list of parameters, along with # their types and docs, programmatically from the iceberg (or managed) @@ -709,6 +719,49 @@ def write_to_iceberg( further sub-dividing partitions into multiple shards to prevent bottlenecks during high-throughput writes. Only available with 'hash' distribution mode. + cdc: A config map that, when present, enables writing the input as a stream + of row-level changes applied by primary key. An empty map will use the + defaults. Config options include: + - sequence_number_column: the required column name representing the + monotonic sequence number used to order a single key's changes. Default + column name is '_commit_snapshot_sequence_number'. This column will be + stripped from the data row before writing to Iceberg. + - change_type_column: the optional column name representing the row's + change type (INSERT, UPDATE_BEFORE, UPDATE_AFTER, or DELETE). This + column will be stripped from the data row before writing to Iceberg. + - change_type_map: optional map from the `change_type_column` value to + the canonical change type name (see above). + - upsert: if true, only the after-image of each change + (INSERT/UPDATE_AFTER) is applied as an upsert. UPDATE_BEFORE records + are dropped. Default: false. + equality_columns: Columns defining row identity (equality-delete fields). + Defaults to the destination table's identifier (primary-key) fields. + Required if the table doesn't exist yet. Currently only supported in CDC + mode. + num_shards: The number of deterministic primary-key-hash shards per + destination, i.e. the max write parallelism per destination. Defaults to + 16. Currently only supported in CDC mode. + shards_per_partition: Maximum number of shards a single partition's rows + may occupy. Defaults to `num_shards`. Currently only supported in CDC + mode. + allowed_lateness_seconds: How long a late record may lag behind the + watermark before it is dropped entirely. Defaults to 21600 (6 hours). + Currently only supported in CDC mode. + sink_id: A stable identifier namespacing the idempotency tokens written to + each commit. Defaults to a unique per-write UUID. Override with a stable + ID for exactly-once commits across relaunches of a particular streaming + write. Currently only relevant in CDC mode. + token_heartbeat_seconds: Streaming only. Refresh each idle + destination's commit token every this many seconds. Disabled by default. + Currently only relevant in CDC mode. + snapshot_properties: Extra properties added to every commit's snapshot + summary. Currently only supported in CDC mode. + error_handling: Where to send records the sink cannot apply (e.g. an + unknown change type) instead of failing the pipeline. Currently only + supported in CDC mode. + sorter_memory_mb: The in-memory buffer size (MB) for the pre-write sort. + Groups larger than this spill to disk. Default: 100MB. Currently only + relevant in CDC mode """ return beam.managed.Write( "iceberg", @@ -724,7 +777,17 @@ def write_to_iceberg( drop=drop, only=only, distribution_mode=distribution_mode, - autosharding=autosharding)) + autosharding=autosharding, + cdc=cdc, + equality_columns=equality_columns, + num_shards=num_shards, + shards_per_partition=shards_per_partition, + allowed_lateness_seconds=allowed_lateness_seconds, + sink_id=sink_id, + token_heartbeat_seconds=token_heartbeat_seconds, + snapshot_properties=snapshot_properties, + error_handling=error_handling, + sorter_memory_mb=sorter_memory_mb)) def io_providers(): From 0963b3b52f7c139b0c589d182bd1dcb9876708c9 Mon Sep 17 00:00:00 2001 From: Ahmed Abualsaud Date: Tue, 22 Sep 2026 17:09:50 -0700 Subject: [PATCH 8/9] pydocs --- sdks/python/apache_beam/yaml/yaml_io.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/sdks/python/apache_beam/yaml/yaml_io.py b/sdks/python/apache_beam/yaml/yaml_io.py index 166e2f053bd0..8882dca3646a 100644 --- a/sdks/python/apache_beam/yaml/yaml_io.py +++ b/sdks/python/apache_beam/yaml/yaml_io.py @@ -722,18 +722,18 @@ def write_to_iceberg( cdc: A config map that, when present, enables writing the input as a stream of row-level changes applied by primary key. An empty map will use the defaults. Config options include: - - sequence_number_column: the required column name representing the - monotonic sequence number used to order a single key's changes. Default - column name is '_commit_snapshot_sequence_number'. This column will be - stripped from the data row before writing to Iceberg. - - change_type_column: the optional column name representing the row's - change type (INSERT, UPDATE_BEFORE, UPDATE_AFTER, or DELETE). This - column will be stripped from the data row before writing to Iceberg. - - change_type_map: optional map from the `change_type_column` value to - the canonical change type name (see above). - - upsert: if true, only the after-image of each change - (INSERT/UPDATE_AFTER) is applied as an upsert. UPDATE_BEFORE records - are dropped. Default: false. + - `sequence_number_column`: the required column name representing the + monotonic sequence number used to order a single key's changes. Default + column name is '_commit_snapshot_sequence_number'. This column will be + stripped from the data row before writing to Iceberg. + - `change_type_column`: the optional column name representing the row's + change type (INSERT, UPDATE_BEFORE, UPDATE_AFTER, or DELETE). This + column will be stripped from the data row before writing to Iceberg. + - `change_type_map`: optional map from the `change_type_column` value to + the canonical change type name (see above). + - `upsert`: if true, only the after-image of each change + (INSERT/UPDATE_AFTER) is applied as an upsert. UPDATE_BEFORE records + are dropped. Default: false. equality_columns: Columns defining row identity (equality-delete fields). Defaults to the destination table's identifier (primary-key) fields. Required if the table doesn't exist yet. Currently only supported in CDC From c55228361e04dc18dd5ea5747d609bbecf692276 Mon Sep 17 00:00:00 2001 From: Ahmed Abualsaud Date: Wed, 23 Sep 2026 15:48:03 -0700 Subject: [PATCH 9/9] address comments --- CHANGES.md | 2 +- .../IcebergWriteSchemaTransformProvider.java | 289 +++++++++--------- .../io/iceberg/cdc/sink/CdcWriteConfig.java | 7 +- .../sdk/io/iceberg/cdc/sink/WriteCdcRows.java | 5 +- ...IcebergSchemaTransformTranslationTest.java | 18 +- ...rgWriteSchemaTransformProviderCdcTest.java | 86 ++++-- .../iceberg/cdc/sink/AssignCdcKeysTest.java | 3 +- .../iceberg/cdc/sink/CdcWriteConfigTest.java | 4 +- sdks/python/apache_beam/yaml/yaml_io.py | 60 ++-- 9 files changed, 244 insertions(+), 230 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 1d2d028b59d2..63f84e249ecd 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -102,7 +102,7 @@ * ClickHouseIO: support writing `Decimal(P, S)` / `Decimal32/64/128/256` columns (Java) ([#39840](https://github.com/apache/beam/issues/39840)). * SolaceIO now supports reading and writing user properties (message metadata) (Java) ([#40099](https://github.com/apache/beam/issues/40099)). * [IcebergIO] AddFiles (`IcebergAddFiles` in YAML) can evolve the table schema before registering files, with `schema_evolution_options`, `required_columns`, `incompatible_schema_handling` and `unverifiable_file_handling` (Java/YAML, batch only) ([#40144](https://github.com/apache/beam/issues/40144)). -* [IcebergIO] Added batch and streaming CDC writes that applies INSERT/UPDATE_BEFORE/UPDATE_AFTER/DELETE changes to Iceberg V2+ tables by primary key. Invoke with `IcebergIO.writeCdcRows` (Java) or by setting the `cdc` option of Managed `ICEBERG` write (Java, Python, YAML) ([#X](https://github.com/apache/beam/issues/X)). +* [IcebergIO] Added batch and streaming CDC writes that applies INSERT/UPDATE_BEFORE/UPDATE_AFTER/DELETE changes to Iceberg V2+ tables by primary key. Invoke with `IcebergIO.writeCdcRows` (Java) or by setting `mode: merge-on-read` on the Managed `ICEBERG` write (Java, Python, YAML) ([#39979](https://github.com/apache/beam/issues/39979)). ## New Features / Improvements diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergWriteSchemaTransformProvider.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergWriteSchemaTransformProvider.java index 831db3559019..4f568fb65620 100644 --- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergWriteSchemaTransformProvider.java +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergWriteSchemaTransformProvider.java @@ -23,11 +23,9 @@ import com.google.auto.service.AutoService; import com.google.auto.value.AutoValue; -import java.io.Serializable; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; @@ -52,16 +50,16 @@ import org.apache.beam.sdk.values.PCollectionRowTuple; import org.apache.beam.sdk.values.Row; import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.apache.iceberg.DistributionMode; import org.apache.iceberg.FileFormat; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; /** - * SchemaTransform implementation for {@link IcebergIO#writeRows} and, when the {@code cdc} block is - * set, {@link IcebergIO#writeCdcRows}. Outputs a {@code PCollection} representing the - * snapshots created in the process; CDC writes add a {@code dead_letter} output of late records. + * SchemaTransform implementation for {@link IcebergIO#writeRows} and, in {@code merge-on-read} + * mode, {@link IcebergIO#writeCdcRows}. Outputs a {@code PCollection} representing the + * snapshots created in the process; merge-on-read writes add a {@code dead_letter} output of late + * records. */ @AutoService(SchemaTransformProvider.class) public class IcebergWriteSchemaTransformProvider @@ -72,10 +70,6 @@ public class IcebergWriteSchemaTransformProvider static final String DEAD_LETTER_TAG = "dead_letter"; static final String ERRORS_TAG = "errors"; - /** The default sequence-number column: what the CDC read source emits. */ - private static final String DEFAULT_SEQUENCE_NUMBER_COLUMN = - IcebergCdcMetadataColumns.COMMIT_SNAPSHOT_SEQUENCE_NUMBER; - static final Schema OUTPUT_SCHEMA = Schema.builder() .addStringField("table") @@ -84,14 +78,14 @@ public class IcebergWriteSchemaTransformProvider @Override public String description() { - return "Writes Beam Rows to Iceberg, appending them by default. Set the 'cdc' block to apply " - + "them as a stream of row-level changes (INSERT/UPDATE_BEFORE/UPDATE_AFTER/DELETE) by " - + "primary key instead.\n" + return "Writes Beam Rows to Iceberg, appending them by default. Set mode to 'merge-on-read' to " + + "apply them as a stream of row-level changes (INSERT/UPDATE_BEFORE/UPDATE_AFTER/DELETE) " + + "by primary key instead.\n" + "Returns a 'snapshots' PCollection representing the snapshots produced in the process, " + "with the following schema:\n" + "{\"table\" (str), \"operation\" (str), \"summary\" (map[str, str]), \"manifestListLocation\" (str)}\n" - + "CDC mode also returns a 'dead_letter' PCollection representing late data, and a 'errors' PCollection " - + "representing invalid records."; + + "Merge-on-read mode also returns a 'dead_letter' PCollection representing late data, and an " + + "'errors' PCollection representing invalid records."; } @DefaultSchema(AutoValueSchema.class) @@ -124,31 +118,42 @@ public static Builder builder() { public abstract @Nullable Integer getDirectWriteByteLimit(); @SchemaFieldDescription( - "A config map that, when present, enables writing the input as a stream of row-level " - + "changes applied by primary key. An empty map will use the defaults. Config options include:\n" - + "- `sequence_number_column`: the required column name representing the" - + "monotonic sequence number used to order a single key's changes. Default" - + "column name is '_commit_snapshot_sequence_number'. This column will be" - + "stripped from the data row before writing to Iceberg.\n" - + "- `change_type_column`: the optional column name representing the row's" - + "change type (INSERT, UPDATE_BEFORE, UPDATE_AFTER, or DELETE). This" - + "column will be stripped from the data row before writing to Iceberg.\n" - + "- `change_type_map`: optional map from the `change_type_column` value to" - + "the canonical change type name (see above).\n" - + "- `upsert`: if true, only the after-image of each change" - + "(INSERT/UPDATE_AFTER) is applied as an upsert. UPDATE_BEFORE records" - + "are dropped. Default: false.") - public abstract @Nullable Cdc getCdc(); + "Controls how rows are written. 'append' (default) appends every row as new data. " + + "'merge-on-read' treats each row as a change (INSERT, UPDATE_BEFORE, UPDATE_AFTER, " + + "or DELETE) applied to the table by primary key.") + public abstract @Nullable String getMode(); + + @SchemaFieldDescription( + "Merge-on-read only. The required column name representing the monotonic sequence number used to " + + "order a single key's changes. Defaults to '_commit_snapshot_sequence_number'. This column will be " + + "stripped from the data row before writing to Iceberg.") + public abstract @Nullable String getSequenceNumberColumn(); + + @SchemaFieldDescription( + "Merge-on-read only. The optional column name representing the row's change type (INSERT, " + + "UPDATE_BEFORE, UPDATE_AFTER, or DELETE). This column will be stripped from the data row " + + "before writing to Iceberg. If unset, the sink will use the element's native ValueKind") + public abstract @Nullable String getChangeTypeColumn(); + + @SchemaFieldDescription( + "Merge-on-read only. Optional map from a change_type_column value to the canonical change " + + "type name (see above).") + public abstract @Nullable Map getChangeTypeMap(); + + @SchemaFieldDescription( + "Merge-on-read only. If true, only the after-image of each change (INSERT/UPDATE_AFTER) " + + "is applied, as an upsert; UPDATE_BEFORE records are dropped. Default: false.") + public abstract @Nullable Boolean getUpsert(); @SchemaFieldDescription( "A list of field names to keep in the input record. All other fields are dropped before writing. " - + "Is mutually exclusive with 'drop' and 'only'. In CDC mode the control columns are " + + "Is mutually exclusive with 'drop' and 'only'. In merge-on-read mode the control columns are " + "dropped unless listed here.") public abstract @Nullable List getKeep(); @SchemaFieldDescription( "A list of field names to drop from the input record before writing. " - + "Is mutually exclusive with 'keep' and 'only'. In CDC mode the control columns are " + + "Is mutually exclusive with 'keep' and 'only'. In merge-on-read mode the control columns are " + "always dropped.") public abstract @Nullable List getDrop(); @@ -224,26 +229,27 @@ public static Builder builder() { @SchemaFieldDescription( "Columns defining row identity (equality-delete fields). Defaults to the destination table's " - + "identifier (primary-key) fields. Required if the table doesn't exist yet. Currently only supported in CDC mode.") + + "identifier (primary-key) fields. Required if the table doesn't exist yet. " + + "Currently only supported in 'merge-on-read' mode.") public abstract @Nullable List getEqualityColumns(); @SchemaFieldDescription( "The number of deterministic primary-key-hash shards per destination, i.e. the max " + "write parallelism per destination. Too low may bottleneck writes, and too high may " - + "produce more files. Defaults to 16. Currently only supported in CDC mode.") + + "produce more files. Defaults to 16. Currently only supported in 'merge-on-read' mode.") public abstract @Nullable Integer getNumShards(); @SchemaFieldDescription( "Maximum number of shards a single partition's rows may occupy. Lower values " + "write fewer files per commit, but also reduces per-partition write parallelism. A value " + "of 1 pins each partition to one writer. Ignored for unpartitioned tables. Must be between 1 " - + "and `num_shards`; defaults to `num_shards`. Currently only supported in CDC mode.") + + "and `num_shards`; defaults to `num_shards`. Currently only supported in 'merge-on-read' mode.") public abstract @Nullable Integer getShardsPerPartition(); @SchemaFieldDescription( "How long a late record may lag behind the watermark before it is " + "dropped entirely, rather than routed to the dead_letter output. Defaults to 21600 " - + "(6 hours). Currently only supported in CDC mode.") + + "(6 hours). Currently only supported in 'merge-on-read' mode.") public abstract @Nullable Integer getAllowedLatenessSeconds(); @SchemaFieldDescription( @@ -252,30 +258,30 @@ public static Builder builder() { + "Set it explicitly (and keep it stable across relaunches) for exactly-once commits across " + "relaunches of a particular streaming write. A batch load with a stable sink_id " + "commits only once (later batch loads with the same sink_id are skipped). " - + "Currently only supported in CDC mode.") + + "Currently only supported in 'merge-on-read' mode.") public abstract @Nullable String getSinkId(); @SchemaFieldDescription( "Streaming only. If set, the sink will emit a periodic empty token-refresh commit while idle, " + "so its thread of `sink_id` stamped snapshot stays recent and is less likely to be " - + "lost to `expire_snapshots`. Disabled by default. Currently only supported in CDC mode.") + + "lost to `expire_snapshots`. Disabled by default. Currently only supported in 'merge-on-read' mode.") public abstract @Nullable Integer getTokenHeartbeatSeconds(); @SchemaFieldDescription( "Extra key/value properties to add to every commit's Iceberg snapshot summary. " - + "Keys prefixed with 'beam.cdc.' are reserved and rejected. Currently only supported in CDC mode.") + + "Keys prefixed with 'beam.cdc.' are reserved and rejected. Currently only supported in 'merge-on-read' mode.") public abstract @Nullable Map getSnapshotProperties(); @SchemaFieldDescription( "Whether and where to output per-record invalid rows (null or missing sequence " + "value, unknown change type, null equality value, unresolvable destination). Fails the pipeline " + "if unset (default). Distinct from the `dead_letter` output, which is for late-but-valid " - + "rows. Currently only supported in CDC mode.") + + "rows. Currently only supported in 'merge-on-read' mode.") public abstract @Nullable ErrorHandling getErrorHandling(); @SchemaFieldDescription( "The in-memory buffer size (MB) for the pre-write sort; groups larger than this " - + "spill to disk. Must be >= 1. Defaults to 100. Currently only supported in CDC mode.") + + "spill to disk. Must be >= 1. Defaults to 100. Currently only supported in 'merge-on-read' mode.") public abstract @Nullable Integer getSorterMemoryMb(); @AutoValue.Builder @@ -318,7 +324,15 @@ public abstract static class Builder { public abstract Builder setPollingBuckets(Integer pollingBuckets); - public abstract Builder setCdc(Cdc cdc); + public abstract Builder setMode(String mode); + + public abstract Builder setSequenceNumberColumn(String sequenceNumberColumn); + + public abstract Builder setChangeTypeColumn(String changeTypeColumn); + + public abstract Builder setChangeTypeMap(Map changeTypeMap); + + public abstract Builder setUpsert(Boolean upsert); public abstract Builder setEqualityColumns(List equalityColumns); @@ -350,118 +364,83 @@ public IcebergCatalogConfig getIcebergCatalog() { } enum Mode { - APPEND, - CDC + APPEND("append"), + MERGE_ON_READ("merge-on-read"); + + /** The value users set {@code mode} to. */ + final String optionValue; + + Mode(String optionValue) { + this.optionValue = optionValue; + } } - /** The write mode this configuration selects. */ + /** The write mode this configuration selects; unset means append. */ Mode mode() { - return getCdc() == null ? Mode.APPEND : Mode.CDC; + @Nullable String mode = getMode(); + if (mode == null || mode.equalsIgnoreCase(Mode.APPEND.optionValue)) { + return Mode.APPEND; + } + if (mode.equalsIgnoreCase(Mode.MERGE_ON_READ.optionValue)) { + return Mode.MERGE_ON_READ; + } + throw new IllegalArgumentException( + String.format( + "Unknown mode '%s'; expected '%s' or '%s'.", + mode, Mode.APPEND.optionValue, Mode.MERGE_ON_READ.optionValue)); } - /** - * Options that only one mode supports today; everything else applies to both. Once an option - * becomes available to the other mode, just remove it from here. - */ - private static final ImmutableMap SUPPORTED_MODES = - ImmutableMap.builder() - .put("equality_columns", Mode.CDC) - .put("num_shards", Mode.CDC) - .put("shards_per_partition", Mode.CDC) - .put("allowed_lateness_seconds", Mode.CDC) - .put("sink_id", Mode.CDC) - .put("token_heartbeat_seconds", Mode.CDC) - .put("snapshot_properties", Mode.CDC) - .put("error_handling", Mode.CDC) - .put("sorter_memory_mb", Mode.CDC) - .put("direct_write_byte_limit", Mode.APPEND) - .put("distribution_mode", Mode.APPEND) - .put("autosharding", Mode.APPEND) - .put("write_properties", Mode.APPEND) - .put("using_side_input_table_cache", Mode.APPEND) - .put("table_refresh_interval_seconds", Mode.APPEND) - .put("maximum_cache_size", Mode.APPEND) - .put("polling_buckets", Mode.APPEND) - .build(); - /** Rejects every set option that the selected mode does not support. */ void validateModeOptions() { + // Resolve the mode first so an unknown value fails here, not only once an option trips it. Mode mode = mode(); - Map values = new LinkedHashMap<>(); - values.put("equality_columns", getEqualityColumns()); - values.put("num_shards", getNumShards()); - values.put("shards_per_partition", getShardsPerPartition()); - values.put("allowed_lateness_seconds", getAllowedLatenessSeconds()); - values.put("sink_id", getSinkId()); - values.put("token_heartbeat_seconds", getTokenHeartbeatSeconds()); - values.put("snapshot_properties", getSnapshotProperties()); - values.put("error_handling", getErrorHandling()); - values.put("sorter_memory_mb", getSorterMemoryMb()); - values.put("direct_write_byte_limit", getDirectWriteByteLimit()); - values.put("distribution_mode", getDistributionMode()); - values.put("autosharding", getAutosharding()); - values.put("write_properties", getWriteProperties()); - values.put("using_side_input_table_cache", getUsingSideInputTableCache()); - values.put("table_refresh_interval_seconds", getTableRefreshIntervalSeconds()); - values.put("maximum_cache_size", getMaximumCacheSize()); - values.put("polling_buckets", getPollingBuckets()); - List invalidOptions = new ArrayList<>(); - for (Map.Entry option : SUPPORTED_MODES.entrySet()) { - String name = option.getKey(); - if (values.get(name) == null || option.getValue() == mode) { - continue; - } - invalidOptions.add(name); - } - if (!invalidOptions.isEmpty()) { + List unsupported = new ArrayList<>(); + // Merge-on-read only: the change-stream contract. + requireMode( + Mode.MERGE_ON_READ, "sequence_number_column", getSequenceNumberColumn(), unsupported); + requireMode(Mode.MERGE_ON_READ, "change_type_column", getChangeTypeColumn(), unsupported); + requireMode(Mode.MERGE_ON_READ, "change_type_map", getChangeTypeMap(), unsupported); + requireMode(Mode.MERGE_ON_READ, "upsert", getUpsert(), unsupported); + // Merge-on-read only, until the append write grows these features. + requireMode(Mode.MERGE_ON_READ, "equality_columns", getEqualityColumns(), unsupported); + requireMode(Mode.MERGE_ON_READ, "num_shards", getNumShards(), unsupported); + requireMode(Mode.MERGE_ON_READ, "shards_per_partition", getShardsPerPartition(), unsupported); + requireMode( + Mode.MERGE_ON_READ, "allowed_lateness_seconds", getAllowedLatenessSeconds(), unsupported); + requireMode(Mode.MERGE_ON_READ, "sink_id", getSinkId(), unsupported); + requireMode( + Mode.MERGE_ON_READ, "token_heartbeat_seconds", getTokenHeartbeatSeconds(), unsupported); + requireMode(Mode.MERGE_ON_READ, "snapshot_properties", getSnapshotProperties(), unsupported); + requireMode(Mode.MERGE_ON_READ, "error_handling", getErrorHandling(), unsupported); + requireMode(Mode.MERGE_ON_READ, "sorter_memory_mb", getSorterMemoryMb(), unsupported); + // Append only: merge-on-read has neither a direct-write path nor a side-input table cache. + requireMode(Mode.APPEND, "direct_write_byte_limit", getDirectWriteByteLimit(), unsupported); + requireMode(Mode.APPEND, "distribution_mode", getDistributionMode(), unsupported); + requireMode(Mode.APPEND, "autosharding", getAutosharding(), unsupported); + requireMode(Mode.APPEND, "write_properties", getWriteProperties(), unsupported); + requireMode( + Mode.APPEND, "using_side_input_table_cache", getUsingSideInputTableCache(), unsupported); + requireMode( + Mode.APPEND, + "table_refresh_interval_seconds", + getTableRefreshIntervalSeconds(), + unsupported); + requireMode(Mode.APPEND, "maximum_cache_size", getMaximumCacheSize(), unsupported); + requireMode(Mode.APPEND, "polling_buckets", getPollingBuckets(), unsupported); + if (!unsupported.isEmpty()) { throw new IllegalArgumentException( String.format( - "The following options are not supported by %s writes yet: %s", - mode.name().toLowerCase(), invalidOptions)); + "The following options are not supported in '%s' mode yet: %s", + mode.optionValue, unsupported)); } } - } - - /** The options that only make sense for a stream of row-level changes. */ - @DefaultSchema(AutoValueSchema.class) - @AutoValue - public abstract static class Cdc implements Serializable { - public static Builder builder() { - return new AutoValue_IcebergWriteSchemaTransformProvider_Cdc.Builder(); - } - - @SchemaFieldDescription( - "The required column name representing the monotonic sequence number used to order a single key's " - + "changes. Default column name is '_commit_snapshot_sequence_number'. This column will be " - + "stripped from the data row before writing to Iceberg.") - public abstract @Nullable String getSequenceNumberColumn(); - - @SchemaFieldDescription( - "The optional column name representing the row's change type (INSERT, UPDATE_BEFORE, UPDATE_AFTER, " - + "or DELETE). This column will be stripped from the data row before writing to Iceberg. If not " - + "specified, the sink will use the element's native ValueKind") - public abstract @Nullable String getChangeTypeColumn(); - - @SchemaFieldDescription( - "Optional map from the `change_type_column` value to the canonical change type name (see above).") - public abstract @Nullable Map getChangeTypeMap(); - - @SchemaFieldDescription( - "If true, only the after-image of each change (INSERT/UPDATE_AFTER) is applied as an upsert. " - + "UPDATE_BEFORE records are dropped. Default: false") - public abstract @Nullable Boolean getUpsert(); - - @AutoValue.Builder - public abstract static class Builder { - public abstract Builder setSequenceNumberColumn(String sequenceNumberColumn); - public abstract Builder setChangeTypeColumn(String changeTypeColumn); - - public abstract Builder setChangeTypeMap(Map changeTypeMap); - - public abstract Builder setUpsert(Boolean upsert); - - public abstract Cdc build(); + /** Records {@code option} as unsupported when it is set under a mode other than its own. */ + private void requireMode( + Mode supported, String option, @Nullable Object value, List unsupported) { + if (value != null && mode() != supported) { + unsupported.add(option); + } } } @@ -510,8 +489,9 @@ Row getConfigurationRow() { public PCollectionRowTuple expand(PCollectionRowTuple input) { PCollection rows = input.get(INPUT_TAG); configuration.validateModeOptions(); - @Nullable Cdc cdc = configuration.getCdc(); - return cdc == null ? expandAppend(rows) : expandCdc(rows, cdc); + return configuration.mode() == Configuration.Mode.MERGE_ON_READ + ? expandCdc(rows) + : expandAppend(rows); } private PCollectionRowTuple expandAppend(PCollection rows) { @@ -598,14 +578,17 @@ private PCollectionRowTuple expandAppend(PCollection rows) { return PCollectionRowTuple.of(SNAPSHOTS_TAG, snapshots); } - private PCollectionRowTuple expandCdc(PCollection rows, Cdc cdc) { + private PCollectionRowTuple expandCdc(PCollection rows) { Schema inputSchema = rows.getSchema(); @Nullable List drop = configuration.getDrop(); @Nullable List keep = configuration.getKeep(); @Nullable String only = configuration.getOnly(); - @Nullable String changeTypeColumn = cdc.getChangeTypeColumn(); - @Nullable String configuredSeq = cdc.getSequenceNumberColumn(); - String seqColumn = configuredSeq != null ? configuredSeq : DEFAULT_SEQUENCE_NUMBER_COLUMN; + @Nullable String changeTypeColumn = configuration.getChangeTypeColumn(); + @Nullable String configuredSeq = configuration.getSequenceNumberColumn(); + String seqColumn = + configuredSeq != null + ? configuredSeq + : IcebergCdcMetadataColumns.COMMIT_SNAPSHOT_SEQUENCE_NUMBER; // The sink reads the control columns from the raw element, so by default they are dropped // from the written row. Listing them in keep writes them too. @@ -631,7 +614,7 @@ private PCollectionRowTuple expandCdc(PCollection rows, Cdc cdc) { drop, keep, only)); - IcebergWriteResult result = rows.apply(applyOptions(write, cdc)); + IcebergWriteResult result = rows.apply(applyOptions(write)); PCollection snapshots = result @@ -649,24 +632,24 @@ private PCollectionRowTuple expandCdc(PCollection rows, Cdc cdc) { } /** Threads every set option onto {@code write}. */ - private WriteCdcRows applyOptions(WriteCdcRows write, Cdc cdc) { + private WriteCdcRows applyOptions(WriteCdcRows write) { @Nullable List equalityColumns = configuration.getEqualityColumns(); if (equalityColumns != null) { write = write.withEqualityColumns(equalityColumns); } - @Nullable String sequenceNumberColumn = cdc.getSequenceNumberColumn(); + @Nullable String sequenceNumberColumn = configuration.getSequenceNumberColumn(); if (sequenceNumberColumn != null) { write = write.withSequenceNumberColumn(sequenceNumberColumn); } - @Nullable String changeTypeColumn = cdc.getChangeTypeColumn(); + @Nullable String changeTypeColumn = configuration.getChangeTypeColumn(); if (changeTypeColumn != null) { write = write.withChangeTypeColumn(changeTypeColumn); } - @Nullable Map changeTypeMap = cdc.getChangeTypeMap(); + @Nullable Map changeTypeMap = configuration.getChangeTypeMap(); if (changeTypeMap != null) { write = write.withChangeTypeMap(changeTypeMap); } - @Nullable Boolean upsert = cdc.getUpsert(); + @Nullable Boolean upsert = configuration.getUpsert(); if (upsert != null) { write = write.withUpsert(upsert); } diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/CdcWriteConfig.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/CdcWriteConfig.java index 537ff6aaebb8..32335dd14fbd 100644 --- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/CdcWriteConfig.java +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/CdcWriteConfig.java @@ -31,9 +31,6 @@ /** Configuration for the CDC sink. */ @AutoValue abstract class CdcWriteConfig implements Serializable { - static final String DEFAULT_SEQUENCE_NUMBER_COLUMN = - IcebergCdcMetadataColumns.COMMIT_SNAPSHOT_SEQUENCE_NUMBER; - /** * Every shard that touches a partition writes a file per commit window, so {@code num_shards x * touched partitions x windows per day} files. On a partitioned table {@link @@ -52,7 +49,7 @@ abstract class CdcWriteConfig implements Serializable { /** * The column holding the per-primary-key monotonic sequence number used to order a single key's - * changes. Defaults to {@value #DEFAULT_SEQUENCE_NUMBER_COLUMN}. + * changes. Defaults to {@value IcebergCdcMetadataColumns#COMMIT_SNAPSHOT_SEQUENCE_NUMBER}. */ abstract String getSequenceNumberColumn(); @@ -131,7 +128,7 @@ abstract class CdcWriteConfig implements Serializable { static Builder builder() { return new AutoValue_CdcWriteConfig.Builder() - .setSequenceNumberColumn(DEFAULT_SEQUENCE_NUMBER_COLUMN) + .setSequenceNumberColumn(IcebergCdcMetadataColumns.COMMIT_SNAPSHOT_SEQUENCE_NUMBER) .setNumShards(DEFAULT_NUM_SHARDS) .setShardsPerPartition(DEFAULT_NUM_SHARDS) .setSorterMemoryMB(DEFAULT_SORTER_MEMORY_MB) diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRows.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRows.java index 9d240c0e7d76..e32ad2f8d5ab 100644 --- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRows.java +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRows.java @@ -29,6 +29,7 @@ import org.apache.beam.sdk.io.iceberg.IcebergCatalogConfig; import org.apache.beam.sdk.io.iceberg.IcebergWriteResult; import org.apache.beam.sdk.io.iceberg.SnapshotInfo; +import org.apache.beam.sdk.io.iceberg.cdc.IcebergCdcMetadataColumns; import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.transforms.display.DisplayData; @@ -184,7 +185,7 @@ abstract static class Builder { public static WriteCdcRows of(IcebergCatalogConfig catalogConfig) { return new AutoValue_WriteCdcRows.Builder() .setCatalogConfig(catalogConfig) - .setSequenceNumberColumn(CdcWriteConfig.DEFAULT_SEQUENCE_NUMBER_COLUMN) + .setSequenceNumberColumn(IcebergCdcMetadataColumns.COMMIT_SNAPSHOT_SEQUENCE_NUMBER) .setNumShards(CdcWriteConfig.DEFAULT_NUM_SHARDS) .setSorterMemoryMB(CdcWriteConfig.DEFAULT_SORTER_MEMORY_MB) .setUpsert(false) @@ -223,7 +224,7 @@ public WriteCdcRows withEqualityColumns(List columns) { * The column holding the per-primary-key monotonic sequence number used to order a single key's * changes. Must be declared as a non-nullable {@code INT64} in the input schema. The column is * stripped from the written rows. Defaults to {@value - * CdcWriteConfig#DEFAULT_SEQUENCE_NUMBER_COLUMN}. + * IcebergCdcMetadataColumns#COMMIT_SNAPSHOT_SEQUENCE_NUMBER}. */ public WriteCdcRows withSequenceNumberColumn(String column) { return toBuilder().setSequenceNumberColumn(column).build(); diff --git a/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergSchemaTransformTranslationTest.java b/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergSchemaTransformTranslationTest.java index 553a768a5cc1..3c07c97b4a52 100644 --- a/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergSchemaTransformTranslationTest.java +++ b/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergSchemaTransformTranslationTest.java @@ -21,7 +21,6 @@ import static org.apache.beam.sdk.io.iceberg.IcebergReadSchemaTransformProvider.IcebergReadSchemaTransform; import static org.apache.beam.sdk.io.iceberg.IcebergWriteSchemaTransformProvider.INPUT_TAG; import static org.apache.beam.sdk.io.iceberg.IcebergWriteSchemaTransformProvider.IcebergWriteSchemaTransform; -import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull; import static org.junit.Assert.assertEquals; import java.io.IOException; @@ -99,25 +98,16 @@ public class IcebergSchemaTransformTranslationTest { .withFieldValue("keep", Collections.singletonList("str")) .build(); - /** A CDC write config: the nested {@code cdc} row plus the options gated to CDC mode. */ + /** A merge-on-read write config: the mode plus the options gated to it. */ private static final Row WRITE_CDC_CONFIG_ROW = Row.withSchema(WRITE_PROVIDER.configurationSchema()) .withFieldValue("table", "test_table_identifier") .withFieldValue("catalog_properties", CATALOG_PROPERTIES) + .withFieldValue("mode", "merge-on-read") + .withFieldValue("change_type_column", "op") + .withFieldValue("upsert", true) .withFieldValue("equality_columns", Collections.singletonList("id")) .withFieldValue("sink_id", "stable-sink") - .withFieldValue( - "cdc", - Row.withSchema( - checkStateNotNull( - WRITE_PROVIDER - .configurationSchema() - .getField("cdc") - .getType() - .getRowSchema())) - .withFieldValue("change_type_column", "op") - .withFieldValue("upsert", true) - .build()) .build(); private static final Row READ_CONFIG_ROW = diff --git a/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergWriteSchemaTransformProviderCdcTest.java b/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergWriteSchemaTransformProviderCdcTest.java index ad59436e6f4b..8f2199e6e525 100644 --- a/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergWriteSchemaTransformProviderCdcTest.java +++ b/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergWriteSchemaTransformProviderCdcTest.java @@ -17,17 +17,18 @@ */ package org.apache.beam.sdk.io.iceberg; -import static org.apache.beam.sdk.io.iceberg.IcebergWriteSchemaTransformProvider.Cdc; import static org.apache.beam.sdk.io.iceberg.IcebergWriteSchemaTransformProvider.Configuration; import static org.apache.beam.sdk.io.iceberg.IcebergWriteSchemaTransformProvider.DEAD_LETTER_TAG; import static org.apache.beam.sdk.io.iceberg.IcebergWriteSchemaTransformProvider.INPUT_TAG; import static org.apache.beam.sdk.io.iceberg.IcebergWriteSchemaTransformProvider.SNAPSHOTS_TAG; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import java.util.List; @@ -138,22 +139,18 @@ private Configuration.Builder configFor(TableIdentifier id) { .setCatalogProperties(catalogProperties()); } - /** {@link #configFor} in CDC mode, with the standard {@code change_type} op column. */ + /** {@link #configFor} in merge-on-read mode, reading kinds from a {@code change_type} column. */ private Configuration.Builder cdcConfigFor(TableIdentifier id) { - return configFor(id).setCdc(changeTypeCdc().build()); + return configFor(id).setMode("merge-on-read").setChangeTypeColumn("change_type"); } - /** A {@code cdc} block reading kinds from the standard {@code change_type} op column. */ - private static Cdc.Builder changeTypeCdc() { - return Cdc.builder().setChangeTypeColumn("change_type"); - } - - /** A CDC config builder for the {@code db.{name}} destination template. */ + /** A merge-on-read config builder for the {@code db.{name}} destination template. */ private Configuration.Builder templateConfig() { return Configuration.builder() .setTable("db.{name}") .setCatalogProperties(catalogProperties()) - .setCdc(changeTypeCdc().build()); + .setMode("merge-on-read") + .setChangeTypeColumn("change_type"); } /** @@ -200,6 +197,44 @@ private static Row rowWithSeq(int id, String name, String data, String changeTyp .build(); } + // --------------------------------------------------------------------------------------------- + // Mode gating + // --------------------------------------------------------------------------------------------- + + /** Options of the other mode are rejected together; shared options pass in both modes. */ + @Test + public void validateModeOptionsRejectsOptionsOfTheOtherMode() { + TableIdentifier id = TableIdentifier.of("db", "t"); + + Configuration append = configFor(id).setUpsert(true).setNumShards(4).setSinkId("s").build(); + IllegalArgumentException appendError = + assertThrows(IllegalArgumentException.class, append::validateModeOptions); + assertThat( + appendError.getMessage(), + equalTo( + "The following options are not supported in 'append' mode yet: " + + "[upsert, num_shards, sink_id]")); + + Configuration mergeOnRead = + cdcConfigFor(id).setDistributionMode("hash").setAutosharding(true).build(); + IllegalArgumentException mergeOnReadError = + assertThrows(IllegalArgumentException.class, mergeOnRead::validateModeOptions); + assertThat( + mergeOnReadError.getMessage(), + equalTo( + "The following options are not supported in 'merge-on-read' mode yet: " + + "[distribution_mode, autosharding]")); + + configFor(id).setTriggeringFrequencySeconds(30).build().validateModeOptions(); + cdcConfigFor(id).setTriggeringFrequencySeconds(30).build().validateModeOptions(); + + IllegalArgumentException unknown = + assertThrows( + IllegalArgumentException.class, + () -> configFor(id).setMode("copy-on-write").build().validateModeOptions()); + assertThat(unknown.getMessage(), containsString("Unknown mode 'copy-on-write'")); + } + // --------------------------------------------------------------------------------------------- // Config values with an observable effect // --------------------------------------------------------------------------------------------- @@ -317,8 +352,8 @@ public void singleTableReplaysDeadLetterShape() { .addStringField("destination") .build(); runCdcWrite( - configFor(id) - .setCdc(changeTypeCdc().setSequenceNumberColumn("sequence_number").build()) + cdcConfigFor(id) + .setSequenceNumberColumn("sequence_number") .setDrop(ImmutableList.of("destination")) .build(), replaySchema, @@ -356,12 +391,12 @@ public void controlColumnsSurviveKeepProjection() { } /** - * An empty {@code cdc} block enables CDC with every default: kinds come from each element's + * Merge-on-read mode with nothing else set uses every default: kinds come from each element's * native {@link ValueKind}, which the projection must preserve (a plain {@code output()} re-emit * would stamp everything INSERT). */ @Test - public void emptyCdcBlockUsesDefaultsAndNativeValueKinds() { + public void mergeOnReadDefaultsUseNativeValueKinds() { TableIdentifier id = v2Table(); Table table = catalog.loadTable(id); @@ -375,10 +410,7 @@ public void emptyCdcBlockUsesDefaultsAndNativeValueKinds() { // No change_type_column: change kinds come only from each element's native ValueKind. SchemaTransform transform = transformFor( - configFor(id) - .setCdc(Cdc.builder().build()) - .setDrop(ImmutableList.of("source")) - .build()); + configFor(id).setMode("merge-on-read").setDrop(ImmutableList.of("source")).build()); PCollection input = withKinds( @@ -430,14 +462,11 @@ public void onlyProjectionExtractsPayloadAndPreservesControlColumns() { runCdcWrite( configFor(id) .setOnly("after") - .setCdc( - Cdc.builder() - .setChangeTypeColumn("op") - .setChangeTypeMap( - ImmutableMap.of("c", "INSERT", "u", "UPDATE_AFTER", "d", "DELETE")) - .setSequenceNumberColumn("seq") - .setUpsert(true) - .build()) + .setMode("merge-on-read") + .setChangeTypeColumn("op") + .setChangeTypeMap(ImmutableMap.of("c", "INSERT", "u", "UPDATE_AFTER", "d", "DELETE")) + .setSequenceNumberColumn("seq") + .setUpsert(true) .build(), envelopeSchema, envelopeRow(envelopeSchema, 1, "a", "x", "c", 1L), @@ -636,7 +665,7 @@ public void errorHandlingRoutesInvalidRecords() { * snake_case config surface. */ @Test - public void managedIcebergWithCdcMapWritesEndToEnd() { + public void managedIcebergMergeOnReadWritesEndToEnd() { TableIdentifier id = v2Table(); Table table = catalog.loadTable(id); @@ -644,7 +673,8 @@ public void managedIcebergWithCdcMapWritesEndToEnd() { ImmutableMap.builder() .put("table", id.toString()) .put("catalog_properties", catalogProperties()) - .put("cdc", ImmutableMap.of("change_type_column", "change_type")) + .put("mode", "merge-on-read") + .put("change_type_column", "change_type") .build(); PCollection input = diff --git a/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/cdc/sink/AssignCdcKeysTest.java b/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/cdc/sink/AssignCdcKeysTest.java index 7d7ae19230e4..232ca3d6bc84 100644 --- a/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/cdc/sink/AssignCdcKeysTest.java +++ b/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/cdc/sink/AssignCdcKeysTest.java @@ -47,6 +47,7 @@ import org.apache.beam.sdk.io.iceberg.DynamicDestinations; import org.apache.beam.sdk.io.iceberg.IcebergCatalogConfig; import org.apache.beam.sdk.io.iceberg.IcebergUtils; +import org.apache.beam.sdk.io.iceberg.cdc.IcebergCdcMetadataColumns; import org.apache.beam.sdk.metrics.MetricNameFilter; import org.apache.beam.sdk.metrics.MetricResult; import org.apache.beam.sdk.metrics.MetricsFilter; @@ -85,7 +86,7 @@ public class AssignCdcKeysTest { @Rule public transient TemporaryFolder tmp = new TemporaryFolder(); @Rule public final TestName testName = new TestName(); - private static final String SEQ_COL = CdcWriteConfig.DEFAULT_SEQUENCE_NUMBER_COLUMN; + private static final String SEQ_COL = IcebergCdcMetadataColumns.COMMIT_SNAPSHOT_SEQUENCE_NUMBER; private static final int NUM_SHARDS = 8; private static final org.apache.iceberg.Schema ICEBERG_SCHEMA = diff --git a/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/cdc/sink/CdcWriteConfigTest.java b/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/cdc/sink/CdcWriteConfigTest.java index fb8ab133b49a..dc55147eb8b4 100644 --- a/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/cdc/sink/CdcWriteConfigTest.java +++ b/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/cdc/sink/CdcWriteConfigTest.java @@ -27,6 +27,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; +import org.apache.beam.sdk.io.iceberg.cdc.IcebergCdcMetadataColumns; import org.apache.beam.sdk.util.SerializableUtils; import org.apache.beam.sdk.values.ValueKind; import org.junit.Test; @@ -44,7 +45,8 @@ public void builderAppliesDefaults() { CdcWriteConfig config = CdcWriteConfig.builder().setSinkId(SINK_ID).build(); assertThat( - config.getSequenceNumberColumn(), equalTo(CdcWriteConfig.DEFAULT_SEQUENCE_NUMBER_COLUMN)); + config.getSequenceNumberColumn(), + equalTo(IcebergCdcMetadataColumns.COMMIT_SNAPSHOT_SEQUENCE_NUMBER)); assertThat(config.getNumShards(), equalTo(CdcWriteConfig.DEFAULT_NUM_SHARDS)); // Unset shards_per_partition resolves to num_shards: the resolved int carries no cap. assertThat(config.getShardsPerPartition(), equalTo(CdcWriteConfig.DEFAULT_NUM_SHARDS)); diff --git a/sdks/python/apache_beam/yaml/yaml_io.py b/sdks/python/apache_beam/yaml/yaml_io.py index 8882dca3646a..3fa15044b60d 100644 --- a/sdks/python/apache_beam/yaml/yaml_io.py +++ b/sdks/python/apache_beam/yaml/yaml_io.py @@ -651,7 +651,11 @@ def write_to_iceberg( only: Optional[str] = None, distribution_mode: Optional[str] = None, autosharding: Optional[bool] = None, - cdc: Optional[Mapping[str, Any]] = None, + mode: Optional[str] = None, + sequence_number_column: Optional[str] = None, + change_type_column: Optional[str] = None, + change_type_map: Optional[Mapping[str, str]] = None, + upsert: Optional[bool] = None, equality_columns: Optional[Iterable[str]] = None, num_shards: Optional[int] = None, shards_per_partition: Optional[int] = None, @@ -719,49 +723,51 @@ def write_to_iceberg( further sub-dividing partitions into multiple shards to prevent bottlenecks during high-throughput writes. Only available with 'hash' distribution mode. - cdc: A config map that, when present, enables writing the input as a stream - of row-level changes applied by primary key. An empty map will use the - defaults. Config options include: - - `sequence_number_column`: the required column name representing the - monotonic sequence number used to order a single key's changes. Default - column name is '_commit_snapshot_sequence_number'. This column will be - stripped from the data row before writing to Iceberg. - - `change_type_column`: the optional column name representing the row's - change type (INSERT, UPDATE_BEFORE, UPDATE_AFTER, or DELETE). This + mode: Controls how rows are written. 'append' (default) appends every row + as new data. 'merge-on-read' treats each row as a change (INSERT, + UPDATE_BEFORE, UPDATE_AFTER, or DELETE) applied to the table by primary + key. + sequence_number_column: Merge-on-read only. the required column name + representing the monotonic sequence number used to order a single key's + changes. Default column name is '_commit_snapshot_sequence_number'. This column will be stripped from the data row before writing to Iceberg. - - `change_type_map`: optional map from the `change_type_column` value to - the canonical change type name (see above). - - `upsert`: if true, only the after-image of each change + change_type_column: Merge-on-read only. The optional column name + representing the row's change type (INSERT, UPDATE_BEFORE, UPDATE_AFTER, + or DELETE). This column will be stripped from the data row before writing + to Iceberg. + change_type_map: Merge-on-read only. Optional map from the + `change_type_column` value to the canonical change type name (see above). + upsert: Merge-on-read only. If true, only the after-image of each change (INSERT/UPDATE_AFTER) is applied as an upsert. UPDATE_BEFORE records are dropped. Default: false. equality_columns: Columns defining row identity (equality-delete fields). Defaults to the destination table's identifier (primary-key) fields. - Required if the table doesn't exist yet. Currently only supported in CDC - mode. + Required if the table doesn't exist yet. Currently only supported in + 'merge-on-read' mode. num_shards: The number of deterministic primary-key-hash shards per destination, i.e. the max write parallelism per destination. Defaults to - 16. Currently only supported in CDC mode. + 16. Currently only supported in 'merge-on-read' mode. shards_per_partition: Maximum number of shards a single partition's rows - may occupy. Defaults to `num_shards`. Currently only supported in CDC - mode. + may occupy. Defaults to `num_shards`. Currently only supported in + 'merge-on-read' mode. allowed_lateness_seconds: How long a late record may lag behind the watermark before it is dropped entirely. Defaults to 21600 (6 hours). - Currently only supported in CDC mode. + Currently only supported in 'merge-on-read' mode. sink_id: A stable identifier namespacing the idempotency tokens written to each commit. Defaults to a unique per-write UUID. Override with a stable ID for exactly-once commits across relaunches of a particular streaming - write. Currently only relevant in CDC mode. + write. Currently only relevant in 'merge-on-read' mode. token_heartbeat_seconds: Streaming only. Refresh each idle destination's commit token every this many seconds. Disabled by default. - Currently only relevant in CDC mode. + Currently only relevant in 'merge-on-read' mode. snapshot_properties: Extra properties added to every commit's snapshot - summary. Currently only supported in CDC mode. + summary. Currently only supported in 'merge-on-read' mode. error_handling: Where to send records the sink cannot apply (e.g. an unknown change type) instead of failing the pipeline. Currently only - supported in CDC mode. + supported in 'merge-on-read' mode. sorter_memory_mb: The in-memory buffer size (MB) for the pre-write sort. Groups larger than this spill to disk. Default: 100MB. Currently only - relevant in CDC mode + relevant in 'merge-on-read' mode. """ return beam.managed.Write( "iceberg", @@ -778,7 +784,11 @@ def write_to_iceberg( only=only, distribution_mode=distribution_mode, autosharding=autosharding, - cdc=cdc, + mode=mode, + sequence_number_column=sequence_number_column, + change_type_column=change_type_column, + change_type_map=change_type_map, + upsert=upsert, equality_columns=equality_columns, num_shards=num_shards, shards_per_partition=shards_per_partition,