From a2ad862ccde0c9586d0f2a9423e27a5ef38f7159 Mon Sep 17 00:00:00 2001 From: Eunjin Song Date: Wed, 8 Jul 2026 13:07:33 -0700 Subject: [PATCH 01/29] [Spark] Reader-side conflict-time data skipping to reduce unnecessary conflicts Conflict detection filters candidate files by partition only; for unpartitioned (incl. liquid-clustered) tables the append check conflicts on any concurrently-added file. This adds column-statistics data skipping: added files whose stats prove they cannot match the current transaction's read predicates are excluded, so operations touching disjoint data ranges no longer falsely conflict. - DataSkippingReaderBase.buildDataSkippingPredicate + filterFilesByDataSkipping reuse the reader's DataFiltersBuilder / verifyStatsForFilter and the shared parseAndDecodeStats, applying 'expr || !verifyStatsForFilter(...)' for one-way safety (skip only when stats prove no match; missing stats keep the file). - ConflictChecker.getFirstFileMatchingPartitionPredicates applies skipping per read predicate and unions survivors (OR across reads; a read's own data predicates are ANDed), and never skips for a whole-table read. Emits a delta.conflictDetection.dataSkipping.filesSkipped event. - Gated by spark.databricks.delta.conflictDetection.dataSkipping.enabled (internal, default off). - Adds ConflictDataSkippingSuite (disjoint / overlapping / disabled / missing-stats / partitioned). Part of #3. Implements #4. Co-Authored-By: Claude Opus 4.8 --- .../spark/sql/delta/ConflictChecker.scala | 31 +++- .../sql/delta/sources/DeltaSQLConf.scala | 11 ++ .../sql/delta/stats/DataSkippingReader.scala | 86 ++++++++- .../sql/delta/ConflictDataSkippingSuite.scala | 171 ++++++++++++++++++ 4 files changed, 292 insertions(+), 7 deletions(-) create mode 100644 spark/src/test/scala/org/apache/spark/sql/delta/ConflictDataSkippingSuite.scala diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/ConflictChecker.scala b/spark/src/main/scala/org/apache/spark/sql/delta/ConflictChecker.scala index d5cd7be1beb..b92392e0c87 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/ConflictChecker.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/ConflictChecker.scala @@ -994,14 +994,41 @@ private[delta] class ConflictChecker( return None } + // Reader-side data skipping: narrow the candidate files using the read snapshot's column-stats + // skipping. A concurrently-added file whose stats prove it cannot match the read predicates is + // not a conflict -- this is what lets unpartitioned / liquid-clustered tables avoid conflicting + // on every added file below. Data predicates are applied *per read predicate* and the survivors + // unioned (OR across reads), because independent reads have OR, not AND, semantics: a file is a + // candidate if it could match ANY read; it is dropped only if its stats prove it fails EVERY + // read. Skipping is not applied for a whole-table read, where every added file must conflict. + val candidateFiles = + if (spark.conf.get(DeltaSQLConf.DELTA_CONFLICT_DETECTION_DATA_SKIPPING_ENABLED) && + !currentTransactionInfo.readWholeTable) { + val readSnapshot = currentTransactionInfo.readSnapshot + val survivingPaths = currentTransactionInfo.readPredicates.iterator.flatMap { rp => + readSnapshot.filterFilesByDataSkipping(files, rp.dataPredicates) + }.map(_.path).toSet + val remaining = files.filter(f => survivingPaths.contains(f.path)) + if (remaining.size < files.size) { + recordDeltaEvent(deltaLog, + opType = "delta.conflictDetection.dataSkipping.filesSkipped", + data = Map( + "candidateFiles" -> files.size, + "skippedFiles" -> (files.size - remaining.size))) + } + remaining + } else { + files + } + // There is no reason to filter files if the table is not partitioned. if (currentTransactionInfo.readWholeTable || currentTransactionInfo.readSnapshot.metadata.partitionColumns.isEmpty) { - return files.headOption + return candidateFiles.headOption } import org.apache.spark.sql.delta.implicits._ - val filesDf = files.toDF(spark) + val filesDf = candidateFiles.toDF(spark) spark.conf.get(DeltaSQLConf.DELTA_CONFLICT_DETECTION_WIDEN_NONDETERMINISTIC_PREDICATES) match { case DeltaSQLConf.NonDeterministicPredicateWidening.OFF => diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala b/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala index d4d9c813b6c..f8edb294261 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala @@ -534,6 +534,17 @@ trait DeltaSQLConfBase extends DeltaSQLConfUtils { .booleanConf .createWithDefault(true) + val DELTA_CONFLICT_DETECTION_DATA_SKIPPING_ENABLED = + buildConf("conflictDetection.dataSkipping.enabled") + .internal() + .doc( + """When enabled, conflict detection uses column-statistics data skipping to exclude + |concurrently-added files whose stats prove they cannot match the current transaction's + |read predicates, reducing unnecessary conflicts (especially on unpartitioned tables). + |One-way safe: a file is excluded only when its stats prove no match.""".stripMargin) + .booleanConf + .createWithDefault(false) + val DELTA_PROTOCOL_DEFAULT_WRITER_VERSION = buildConf("properties.defaults.minWriterVersion") .doc("The default writer protocol version to create new tables with, unless a feature " + diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/stats/DataSkippingReader.scala b/spark/src/main/scala/org/apache/spark/sql/delta/stats/DataSkippingReader.scala index 574e0e3008e..05fd8fd6fe1 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/stats/DataSkippingReader.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/stats/DataSkippingReader.scala @@ -286,22 +286,29 @@ trait DataSkippingReaderBase DeltaSQLConf.DELTA_DATASKIPPING_PARTITION_LIKE_FILTERS_ADDITIONAL_SUPPORTED_EXPRESSIONS) .toSet.flatMap((exprs: String) => exprs.split(",")) - /** Returns a DataFrame expression to obtain a list of files with parsed statistics. */ - private def withStatsInternal0: DataFrame = { - val parsedStats = from_json(col("stats"), statsSchema) + /** + * Parses (and, when the schema contains VariantType, Z85-decodes) the JSON `statsCol` into the + * `statsSchema` struct. Shared by [[withStatsInternal0]] and conflict-detection data skipping + * ([[filterFilesByDataSkipping]]) so both parse stats the same way. + */ + private def parseAndDecodeStats(statsCol: Column): Column = { + val parsedStats = from_json(statsCol, statsSchema) // Only use DecodeNestedZ85EncodedVariant if the schema contains VariantType. // This avoids performance overhead for tables without variant columns. // `DecodeNestedZ85EncodedVariant` is a temporary workaround since the Spark 4.1 from_json // expression has no way to decode a VariantVal from an encoded Z85 string. // TODO: Add Z85 decoding to Variant in Spark 4.2 and use that from_json option here. - val decodedStats = if (SchemaUtils.checkForVariantTypeColumnsRecursively(statsSchema)) { + if (SchemaUtils.checkForVariantTypeColumnsRecursively(statsSchema)) { Column(DecodeNestedZ85EncodedVariant(parsedStats.expr)) } else { parsedStats } - allFiles.withColumn("stats", decodedStats) } + /** Returns a DataFrame expression to obtain a list of files with parsed statistics. */ + private def withStatsInternal0: DataFrame = + allFiles.withColumn("stats", parseAndDecodeStats(col("stats"))) + private lazy val withStatsCache = cacheDS(withStatsInternal0, s"Delta Table State with Stats #$version - $redactedPath") @@ -641,6 +648,75 @@ trait DataSkippingReaderBase files.toSeq -> Seq(DataSize(totalSize), DataSize(partitionSize), DataSize(scanSize)) } + /** + * Builds a single [[DataSkippingPredicate]] (skipping expression + the stats it references) from + * `dataFilters`, mirroring [[filesForScan]]'s eligibility filtering, per-filter construction and + * conjunction fold. Returns None when stats skipping is unavailable or no eligible filter yields + * a predicate. The filters are AND-combined, so callers must pass filters from a single logical + * read (predicates from independent reads have OR, not AND, semantics). + */ + private[delta] def buildDataSkippingPredicate( + dataFilters: Seq[Expression]): Option[DataSkippingPredicate] = { + import DeltaTableUtils._ + if (!useStats) return None + // Mirror filesForScan eligibility: drop subquery / non-deterministic / metadata filters, so we + // never build a skipping predicate that could wrongly exclude a matching file. + val eligibleFilters = dataFilters.filterNot { f => + containsSubquery(f) || !f.deterministic || f.exists { + case MetadataAttribute(_) => true + case _ => false + } + } + val constructDataFilters = new DataFiltersBuilder( + spark = spark, + dataSkippingType = DeltaDataSkippingType.dataSkippingOnlyV1, + getStatsColumnOpt = (s: StatsColumn) => getStatsColumnOpt(s)) + eligibleFilters + .flatMap(f => constructDataFilters(f)) + .reduceOption((skip1, skip2) => DataSkippingPredicate( + skip1.expr && skip2.expr, skip1.referencedStats ++ skip2.referencedStats)) + } + + /** + * Conflict-detection helper (reader-side data skipping): returns the subset of `files` whose + * statistics do NOT prove they fail `dataFilters` -- i.e. the files that could still match and + * therefore must be treated as conflicts. Files with missing/insufficient stats (or when skipping + * is unavailable) are kept. + * + * `dataFilters` must come from a single logical read: they are AND-combined via + * [[buildDataSkippingPredicate]]. Callers with multiple independent reads must invoke this per + * read and union the survivors (OR semantics). + * + * One-way safe: it applies `expr || !verifyStatsForFilter(...)` exactly as + * [[getDataSkippedFiles]], so a file is dropped only when its stats *prove* it cannot match -- a + * real conflict is never turned into a false negative. Returns the original [[AddFile]]s (matched + * by path), stats untouched. + */ + private[delta] def filterFilesByDataSkipping( + files: Seq[AddFile], + dataFilters: Seq[Expression]): Seq[AddFile] = { + import org.apache.spark.sql.delta.implicits._ + if (files.isEmpty || dataFilters.isEmpty || schema.isEmpty) return files + buildDataSkippingPredicate(dataFilters) match { + // No usable data-skipping predicate -> keep all files (conservative). + case None => files + case Some(pred) => + val survivingPaths = recordFrameProfile( + "Delta", "DataSkippingReader.filterFilesByDataSkipping") { + // `expr || !verifyStatsForFilter(...)` keeps any file whose referenced stats are + // missing/NULL (mirrors getDataSkippedFiles): only skip when stats prove no match. + files.toDF(spark) + .withColumn("stats", parseAndDecodeStats(col("stats"))) + .where(pred.expr || !verifyStatsForFilter(pred.referencedStats)) + .select("path") + .collect() + .map(_.getString(0)) + .toSet + } + files.filter(f => survivingPaths.contains(f.path)) + } + } + private def getCorrectDataSkippingType( dataSkippingType: DeltaDataSkippingType): DeltaDataSkippingType = { dataSkippingType diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/ConflictDataSkippingSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/ConflictDataSkippingSuite.scala new file mode 100644 index 00000000000..112862ac131 --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/ConflictDataSkippingSuite.scala @@ -0,0 +1,171 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta + +import java.io.File + +import scala.concurrent.duration.Duration + +import org.apache.spark.sql.delta.concurrency.PhaseLockingTestMixin +import org.apache.spark.sql.delta.concurrency.TransactionExecutionTestMixin +import org.apache.spark.sql.delta.sources.DeltaSQLConf +import org.apache.spark.sql.delta.test.DeltaSQLCommandTest + +import org.apache.spark.SparkException +import org.apache.spark.sql.{QueryTest, Row} +import org.apache.spark.sql.functions.lit +import org.apache.spark.sql.test.SharedSparkSession +import org.apache.spark.util.ThreadUtils + +/** + * Tests for reader-side conflict-time data skipping + * ([[DeltaSQLConf.DELTA_CONFLICT_DETECTION_DATA_SKIPPING_ENABLED]]). + * + * A concurrently-added file whose column stats prove it cannot match the current transaction's read + * predicates should NOT cause an append conflict, especially on unpartitioned tables, where the + * append check would otherwise conflict on any added file. Skipping must be one-way safe: a file + * with missing stats is always kept, so a real conflict is never missed. + */ +class ConflictDataSkippingSuite extends QueryTest + with SharedSparkSession + with DeltaSQLCommandTest + with PhaseLockingTestMixin + with TransactionExecutionTestMixin { + + private def tableRef(dir: File): String = s"delta.`${dir.getCanonicalPath}`" + + /** + * Unpartitioned table with `id` in [0, 1000) across 10 files (file i covers [100i, 100i+100)), + * committed at Serializable isolation so that concurrent (blind) appends are conflict-checked. + * When `numIndexedCols` is set, stats collection is limited accordingly (0 = no stats). + */ + private def createTable(dir: File, numIndexedCols: Option[Int] = None): Unit = { + spark.range(start = 0, end = 1000, step = 1, numPartitions = 10) + .write.format("delta").mode("append").save(dir.getAbsolutePath) + val extraProps = numIndexedCols + .map(n => s", '${DeltaConfigs.DATA_SKIPPING_NUM_INDEXED_COLS.key}' = '$n'") + .getOrElse("") + sql(s"ALTER TABLE ${tableRef(dir)} SET TBLPROPERTIES " + + s"('${DeltaConfigs.ISOLATION_LEVEL.key}' = 'Serializable'$extraProps)") + } + + /** A DELETE that runs under the given data-skipping setting (evaluated on its commit thread). */ + private def deleteTxn(dir: File, condition: String, dataSkipping: Boolean): () => Array[Row] = + () => { + withSQLConf( + DeltaSQLConf.DELTA_CONFLICT_DETECTION_DATA_SKIPPING_ENABLED.key -> dataSkipping.toString) { + sql(s"DELETE FROM ${tableRef(dir)} WHERE $condition").collect() + } + Array.empty[Row] + } + + /** A blind append of `id` in [start, end), optionally into partition `p`. */ + private def appendTxn( + dir: File, start: Long, end: Long, partition: Option[Int] = None): () => Array[Row] = + () => { + var df = spark.range(start, end).toDF() + partition.foreach(p => df = df.withColumn("p", lit(p))) + df.write.format("delta").mode("append").save(dir.getAbsolutePath) + Array.empty[Row] + } + + /** Expected surviving ids after deleting id<50 and appending [1000, 1100). */ + private val disjointExpected: Seq[Row] = + ((50L until 1000L) ++ (1000L until 1100L)).map(Row(_)) + + private def assertConcurrentAppend(e: SparkException): Unit = + assert(e.getCause.isInstanceOf[io.delta.exceptions.ConcurrentAppendException], + s"Expected ConcurrentAppendException, got: ${e.getCause}") + + test("disjoint data ranges: added file is skipped, no conflict") { + withTempDir { dir => + createTable(dir) + // A (loser) deletes id<50; B (winner) appends [1000,1100), disjoint from A's predicate. + val txnA = deleteTxn(dir, "id < 50", dataSkipping = true) + val txnB = appendTxn(dir, 1000, 1100) + + val (futureA, futureB) = runTxnsWithOrder__A_Start__B__A_End(txnA, txnB) + ThreadUtils.awaitResult(futureA, Duration.Inf) + ThreadUtils.awaitResult(futureB, Duration.Inf) + + // Both committed: id<50 deleted, [1000,1100) appended. + checkAnswer( + spark.read.format("delta").load(dir.getAbsolutePath).select("id"), disjointExpected) + } + } + + test("overlapping data ranges: added file matches the predicate, still conflicts") { + withTempDir { dir => + createTable(dir) + // A's predicate id>=950 overlaps B's appended [1000,1100) file range -> not skippable. + val txnA = deleteTxn(dir, "id >= 950", dataSkipping = true) + val txnB = appendTxn(dir, 1000, 1100) + + val (futureA, futureB) = runTxnsWithOrder__A_Start__B__A_End(txnA, txnB) + ThreadUtils.awaitResult(futureB, Duration.Inf) + val e = intercept[SparkException] { ThreadUtils.awaitResult(futureA, Duration.Inf) } + assertConcurrentAppend(e) + } + } + + test("feature disabled: disjoint data ranges still conflict") { + withTempDir { dir => + createTable(dir) + val txnA = deleteTxn(dir, "id < 50", dataSkipping = false) + val txnB = appendTxn(dir, 1000, 1100) + + val (futureA, futureB) = runTxnsWithOrder__A_Start__B__A_End(txnA, txnB) + ThreadUtils.awaitResult(futureB, Duration.Inf) + val e = intercept[SparkException] { ThreadUtils.awaitResult(futureA, Duration.Inf) } + assertConcurrentAppend(e) + } + } + + test("missing stats: disjoint ranges still conflict (one-way safety)") { + withTempDir { dir => + // No indexed columns -> the appended file has no id stats -> must NOT be skipped. + createTable(dir, numIndexedCols = Some(0)) + val txnA = deleteTxn(dir, "id < 50", dataSkipping = true) + val txnB = appendTxn(dir, 1000, 1100) + + val (futureA, futureB) = runTxnsWithOrder__A_Start__B__A_End(txnA, txnB) + ThreadUtils.awaitResult(futureB, Duration.Inf) + val e = intercept[SparkException] { ThreadUtils.awaitResult(futureA, Duration.Inf) } + assertConcurrentAppend(e) + } + } + + test("partitioned table: data skipping on a non-partition column avoids the conflict") { + withTempDir { dir => + // Partitioned by `p`; the appended file shares the partition but its id range is disjoint. + spark.range(start = 0, end = 1000, step = 1, numPartitions = 10).withColumn("p", lit(0)) + .write.partitionBy("p").format("delta").mode("append").save(dir.getAbsolutePath) + sql(s"ALTER TABLE ${tableRef(dir)} SET TBLPROPERTIES " + + s"('${DeltaConfigs.ISOLATION_LEVEL.key}' = 'Serializable')") + + val txnA = deleteTxn(dir, "id < 50", dataSkipping = true) + val txnB = appendTxn(dir, 1000, 1100, partition = Some(0)) + + val (futureA, futureB) = runTxnsWithOrder__A_Start__B__A_End(txnA, txnB) + ThreadUtils.awaitResult(futureA, Duration.Inf) + ThreadUtils.awaitResult(futureB, Duration.Inf) + + checkAnswer( + spark.read.format("delta").load(dir.getAbsolutePath).select("id"), disjointExpected) + } + } +} From f6b092fd032c6291efc67ba48495225ff786cafa Mon Sep 17 00:00:00 2001 From: Eunjin Song Date: Mon, 3 Aug 2026 11:19:09 -0700 Subject: [PATCH 02/29] Guard conflict-time data skipping on non-empty read predicates Fix a Serializable soundness gap in reader-side conflict-time data skipping. A non-blind-append transaction can record no read predicates (e.g. it removes files without reading the table). With skipping enabled its empty read set produced an empty survivor set, so getFirstFileMatchingPartitionPredicates returned None and a real ConcurrentAppendException was silently suppressed. Skipping is now applied only when the transaction has at least one read predicate; otherwise every concurrently added file stays a conflict candidate, matching behavior with the feature disabled. Add ConflictDataSkippingSuite coverage: - empty read predicates on a non-blind-append txn still conflict (regression test for the fix above); - a whole-table read is never skipped and conflicts with a concurrent append; - filterFilesByDataSkipping AND-combines predicates within a single read. Co-Authored-By: Claude Opus 4.8 --- .../spark/sql/delta/ConflictChecker.scala | 9 +- .../sql/delta/ConflictDataSkippingSuite.scala | 83 ++++++++++++++++++- 2 files changed, 89 insertions(+), 3 deletions(-) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/ConflictChecker.scala b/spark/src/main/scala/org/apache/spark/sql/delta/ConflictChecker.scala index b92392e0c87..9e7db8589e1 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/ConflictChecker.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/ConflictChecker.scala @@ -1000,10 +1000,15 @@ private[delta] class ConflictChecker( // on every added file below. Data predicates are applied *per read predicate* and the survivors // unioned (OR across reads), because independent reads have OR, not AND, semantics: a file is a // candidate if it could match ANY read; it is dropped only if its stats prove it fails EVERY - // read. Skipping is not applied for a whole-table read, where every added file must conflict. + // read. Skipping is not applied for a whole-table read, nor when there are no read predicates + // at all: a transaction can be a non-blind-append (e.g. it removed files) yet have no read + // predicates, and in that case every added file must remain a conflict candidate, exactly as + // when the feature is disabled. Guarding on non-empty read predicates avoids an empty survivor + // set silently suppressing a real conflict. val candidateFiles = if (spark.conf.get(DeltaSQLConf.DELTA_CONFLICT_DETECTION_DATA_SKIPPING_ENABLED) && - !currentTransactionInfo.readWholeTable) { + !currentTransactionInfo.readWholeTable && + currentTransactionInfo.readPredicates.nonEmpty) { val readSnapshot = currentTransactionInfo.readSnapshot val survivingPaths = currentTransactionInfo.readPredicates.iterator.flatMap { rp => readSnapshot.filterFilesByDataSkipping(files, rp.dataPredicates) diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/ConflictDataSkippingSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/ConflictDataSkippingSuite.scala index 112862ac131..b003835bd13 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/ConflictDataSkippingSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/ConflictDataSkippingSuite.scala @@ -20,15 +20,18 @@ import java.io.File import scala.concurrent.duration.Duration +import org.apache.spark.sql.delta.actions.AddFile import org.apache.spark.sql.delta.concurrency.PhaseLockingTestMixin import org.apache.spark.sql.delta.concurrency.TransactionExecutionTestMixin import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.SparkException -import org.apache.spark.sql.{QueryTest, Row} +import org.apache.spark.sql.{QueryTest, Row, SaveMode} +import org.apache.spark.sql.catalyst.expressions.{AttributeReference, GreaterThanOrEqual, LessThan, Literal} import org.apache.spark.sql.functions.lit import org.apache.spark.sql.test.SharedSparkSession +import org.apache.spark.sql.types.LongType import org.apache.spark.util.ThreadUtils /** @@ -168,4 +171,82 @@ class ConflictDataSkippingSuite extends QueryTest spark.read.format("delta").load(dir.getAbsolutePath).select("id"), disjointExpected) } } + + private def manufacturedAdd(name: String): AddFile = + AddFile(name, Map.empty[String, String], size = 1L, modificationTime = 1L, dataChange = true) + + test("empty read predicates on a non-blind-append txn: added file still conflicts") { + // Soundness of the Serializable blind-append case. A transaction can be a NON-blind append + // (it removes a file, so onlyAddFiles = false) and yet record no read predicates. With data + // skipping enabled such a txn must still conflict-check every concurrently added file, exactly + // as it does with the feature disabled. Before the guard on non-empty read predicates, the + // empty read set produced an empty survivor set and the conflict was silently suppressed. + withTempDir { dir => + createTable(dir) + val log = DeltaLog.forTable(spark, dir.getCanonicalPath) + // An existing file to remove, which makes the loser a non-blind append with no reads. + val existingRemove = log.update().allFiles.collect().head.remove + withSQLConf( + DeltaSQLConf.DELTA_CONFLICT_DETECTION_DATA_SKIPPING_ENABLED.key -> "true") { + val loser = log.startTransaction() + // Winner: a blind append committed while the loser is still open. + log.startTransaction().commit( + Seq(manufacturedAdd("winner.parquet")), DeltaOperations.Write(SaveMode.Append)) + // Loser adds AND removes without reading -> not a blind append, no read predicates. + intercept[io.delta.exceptions.ConcurrentAppendException] { + loser.commit( + Seq(manufacturedAdd("loser.parquet"), existingRemove), + DeltaOperations.Write(SaveMode.Append)) + } + } + } + } + + test("whole-table read is never skipped: conflicts with a concurrent append") { + // A whole-table read must treat every concurrently added file as a conflict, even with data + // skipping enabled -- the feature is deliberately bypassed for readWholeTable. + withTempDir { dir => + createTable(dir) + val log = DeltaLog.forTable(spark, dir.getCanonicalPath) + withSQLConf( + DeltaSQLConf.DELTA_CONFLICT_DETECTION_DATA_SKIPPING_ENABLED.key -> "true") { + val loser = log.startTransaction() + loser.readWholeTable() + log.startTransaction().commit( + Seq(manufacturedAdd("winner.parquet")), DeltaOperations.Write(SaveMode.Append)) + intercept[io.delta.exceptions.ConcurrentAppendException] { + loser.commit( + Seq(manufacturedAdd("loser.parquet")), DeltaOperations.Write(SaveMode.Append)) + } + } + } + } + + test("filterFilesByDataSkipping AND-combines predicates within one read") { + // Predicates from a SINGLE read have AND semantics (OR is only across independent reads, which + // ConflictChecker unions by construction). This exercises that AND directly. + withTempDir { dir => + createTable(dir) + val log = DeltaLog.forTable(spark, dir.getCanonicalPath) + val snapshot = log.update() + val allFiles = snapshot.allFiles.collect().toSeq + assert(allFiles.size == 10) + + // Resolved catalyst predicates on the `id` column; built directly so the literal stays a + // bare Long (a parsed `id >= 200` would wrap it in a Cast, which is not skipping-eligible). + val id = AttributeReference("id", LongType)() + val ge200 = GreaterThanOrEqual(id, Literal(200L)) + val lt300 = LessThan(id, Literal(300L)) + + // Only the [200, 300) file can match `id >= 200 AND id < 300`. OR semantics would keep every + // file with id >= 200 or id < 300, i.e. all 10. + val survivors = snapshot.filterFilesByDataSkipping(allFiles, Seq(ge200, lt300)) + assert(survivors.size == 1, + s"expected exactly the [200,300) file, got ${survivors.map(_.path).sorted}") + + // Sanity: a single predicate keeps its whole matching range (files with max id >= 200). + val geOnly = snapshot.filterFilesByDataSkipping(allFiles, Seq(ge200)) + assert(geOnly.size == 8, s"expected 8 files with id >= 200, got ${geOnly.size}") + } + } } From e2f12cb539879cd2b9a05992816e5a8d759b9c4c Mon Sep 17 00:00:00 2001 From: Eunjin Song Date: Fri, 7 Aug 2026 18:16:18 -0700 Subject: [PATCH 03/29] Address review: single-job multi-read skipping + eval fallback Combine per-read data-skipping into one Spark job and fail safe on evaluation errors, per review comments on #7358. - filterFilesMatchingAnyReadPredicate(files, dataFiltersPerRead): builds one skipping predicate per read and OR-combines them into a single `where`, so all read predicates are evaluated in one Spark job instead of one job per read. A read with no usable predicate matches everything -> short-circuit and keep all files, no job. - filterFilesByDataSkipping is now the single-read case delegating to it. - Wrap build+eval in try/catch(NonFatal): on failure log a warning and return all candidate files (default conflict behavior), so a skipping error can never abort a valid commit. - ConflictChecker calls the new method once instead of flatMap-per-read. - Test: OR-combines independent reads in one call. Co-Authored-By: Claude Opus 4.8 --- .../spark/sql/delta/ConflictChecker.scala | 21 +++-- .../sql/delta/stats/DataSkippingReader.scala | 90 +++++++++++++------ .../sql/delta/ConflictDataSkippingSuite.scala | 30 ++++++- 3 files changed, 103 insertions(+), 38 deletions(-) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/ConflictChecker.scala b/spark/src/main/scala/org/apache/spark/sql/delta/ConflictChecker.scala index 9e7db8589e1..0201dfe8403 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/ConflictChecker.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/ConflictChecker.scala @@ -998,22 +998,21 @@ private[delta] class ConflictChecker( // skipping. A concurrently-added file whose stats prove it cannot match the read predicates is // not a conflict -- this is what lets unpartitioned / liquid-clustered tables avoid conflicting // on every added file below. Data predicates are applied *per read predicate* and the survivors - // unioned (OR across reads), because independent reads have OR, not AND, semantics: a file is a - // candidate if it could match ANY read; it is dropped only if its stats prove it fails EVERY - // read. Skipping is not applied for a whole-table read, nor when there are no read predicates - // at all: a transaction can be a non-blind-append (e.g. it removed files) yet have no read - // predicates, and in that case every added file must remain a conflict candidate, exactly as - // when the feature is disabled. Guarding on non-empty read predicates avoids an empty survivor - // set silently suppressing a real conflict. + // OR-combined, because independent reads have OR, not AND, semantics: a file is a candidate if + // it could match ANY read; it is dropped only if its stats prove it fails EVERY read. All reads + // are evaluated in a single Spark job by filterFilesMatchingAnyReadPredicate. Skipping is not + // applied for a whole-table read, nor when there are no read predicates at all: a transaction + // can be a non-blind-append (e.g. it removed files) yet have no read predicates, and in that + // case every added file must remain a conflict candidate, exactly as when the feature is + // disabled. Guarding on non-empty read predicates avoids an empty survivor set silently + // suppressing a real conflict. val candidateFiles = if (spark.conf.get(DeltaSQLConf.DELTA_CONFLICT_DETECTION_DATA_SKIPPING_ENABLED) && !currentTransactionInfo.readWholeTable && currentTransactionInfo.readPredicates.nonEmpty) { val readSnapshot = currentTransactionInfo.readSnapshot - val survivingPaths = currentTransactionInfo.readPredicates.iterator.flatMap { rp => - readSnapshot.filterFilesByDataSkipping(files, rp.dataPredicates) - }.map(_.path).toSet - val remaining = files.filter(f => survivingPaths.contains(f.path)) + val remaining = readSnapshot.filterFilesMatchingAnyReadPredicate( + files, currentTransactionInfo.readPredicates.map(_.dataPredicates).toSeq) if (remaining.size < files.size) { recordDeltaEvent(deltaLog, opType = "delta.conflictDetection.dataSkipping.filesSkipped", diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/stats/DataSkippingReader.scala b/spark/src/main/scala/org/apache/spark/sql/delta/stats/DataSkippingReader.scala index 05fd8fd6fe1..6c9673c55a4 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/stats/DataSkippingReader.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/stats/DataSkippingReader.scala @@ -20,6 +20,7 @@ package org.apache.spark.sql.delta.stats import java.io.Closeable import scala.collection.mutable.ArrayBuffer +import scala.util.control.NonFatal import org.apache.spark.sql.delta.skipping.clustering.{ClusteredTableUtils, ClusteringColumnInfo} import org.apache.spark.sql.delta.ClassicColumnConversions._ @@ -684,36 +685,73 @@ trait DataSkippingReaderBase * is unavailable) are kept. * * `dataFilters` must come from a single logical read: they are AND-combined via - * [[buildDataSkippingPredicate]]. Callers with multiple independent reads must invoke this per - * read and union the survivors (OR semantics). - * - * One-way safe: it applies `expr || !verifyStatsForFilter(...)` exactly as - * [[getDataSkippedFiles]], so a file is dropped only when its stats *prove* it cannot match -- a - * real conflict is never turned into a false negative. Returns the original [[AddFile]]s (matched - * by path), stats untouched. + * [[buildDataSkippingPredicate]]. This is the one-read case of + * [[filterFilesMatchingAnyReadPredicate]]; callers with multiple independent reads should use + * that method so all reads are evaluated in a single Spark job. */ private[delta] def filterFilesByDataSkipping( files: Seq[AddFile], - dataFilters: Seq[Expression]): Seq[AddFile] = { + dataFilters: Seq[Expression]): Seq[AddFile] = + filterFilesMatchingAnyReadPredicate(files, Seq(dataFilters)) + + /** + * Conflict-detection helper (reader-side data skipping) over several INDEPENDENT reads: returns + * the subset of `files` that could still match ANY read and therefore must be treated as + * conflicts. Each inner `Seq[Expression]` is one logical read's data filters (AND-combined via + * [[buildDataSkippingPredicate]]); the reads are OR-combined, matching read semantics -- a file + * is a candidate if it could match any one of them. + * + * All reads are evaluated in a SINGLE Spark job: the per-read skipping predicates are OR-ed + * together into one `where` clause, rather than filtering per read and unioning the survivors + * (which launched one job per read). Note we cannot instead flatten every read's filters into one + * predicate -- that would AND them (read1 AND read2), the opposite of the OR we need. + * + * One-way safe: each read contributes `expr || !verifyStatsForFilter(...)` exactly as + * [[getDataSkippedFiles]], so a file is dropped only when its stats *prove* it fails EVERY read + * -- a real conflict is never a false negative. Files with missing/insufficient stats, a + * read with no usable skipping predicate (empty / ineligible filters -> matches everything), or a + * table without stats are all kept. Returns the original [[AddFile]]s (matched by path). + * + * Fail-safe: skipping here is a pure optimization over correct (conservative) conflict detection, + * so if building or evaluating the predicate throws we fall back to the default behavior of + * keeping all `files` as conflict candidates rather than failing the commit. + */ + private[delta] def filterFilesMatchingAnyReadPredicate( + files: Seq[AddFile], + dataFiltersPerRead: Seq[Seq[Expression]]): Seq[AddFile] = { import org.apache.spark.sql.delta.implicits._ - if (files.isEmpty || dataFilters.isEmpty || schema.isEmpty) return files - buildDataSkippingPredicate(dataFilters) match { - // No usable data-skipping predicate -> keep all files (conservative). - case None => files - case Some(pred) => - val survivingPaths = recordFrameProfile( - "Delta", "DataSkippingReader.filterFilesByDataSkipping") { - // `expr || !verifyStatsForFilter(...)` keeps any file whose referenced stats are - // missing/NULL (mirrors getDataSkippedFiles): only skip when stats prove no match. - files.toDF(spark) - .withColumn("stats", parseAndDecodeStats(col("stats"))) - .where(pred.expr || !verifyStatsForFilter(pred.referencedStats)) - .select("path") - .collect() - .map(_.getString(0)) - .toSet - } - files.filter(f => survivingPaths.contains(f.path)) + if (files.isEmpty || dataFiltersPerRead.isEmpty || schema.isEmpty) return files + try { + // One skipping predicate per read. A read with no usable predicate (empty or ineligible + // filters) matches every file -> nothing can be skipped, so keep all files without a job. + val perReadPredicates = dataFiltersPerRead.map { dataFilters => + if (dataFilters.isEmpty) None else buildDataSkippingPredicate(dataFilters) + } + if (perReadPredicates.exists(_.isEmpty)) return files + // Survive if the file could match ANY read. Per read, `expr || !verifyStatsForFilter(...)` + // keeps any file whose referenced stats are missing/NULL (mirrors getDataSkippedFiles): only + // skip when stats prove no match. OR the reads so a match against any one keeps the file. + val survivorCondition = perReadPredicates.flatten + .map(pred => pred.expr || !verifyStatsForFilter(pred.referencedStats)) + .reduce(_ || _) + val survivingPaths = recordFrameProfile( + "Delta", "DataSkippingReader.filterFilesMatchingAnyReadPredicate") { + files.toDF(spark) + .withColumn("stats", parseAndDecodeStats(col("stats"))) + .where(survivorCondition) + .select("path") + .collect() + .map(_.getString(0)) + .toSet + } + files.filter(f => survivingPaths.contains(f.path)) + } catch { + case NonFatal(e) => + // Optimization only: never let a skipping failure abort a commit. Fall back to the default + // (feature-off) behavior of treating every added file as a conflict candidate. + logWarning(log"Conflict-time data skipping failed to evaluate; falling back to treating " + + log"all added files as conflict candidates", e) + files } } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/ConflictDataSkippingSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/ConflictDataSkippingSuite.scala index b003835bd13..bf5648ce97e 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/ConflictDataSkippingSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/ConflictDataSkippingSuite.scala @@ -28,7 +28,7 @@ import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.SparkException import org.apache.spark.sql.{QueryTest, Row, SaveMode} -import org.apache.spark.sql.catalyst.expressions.{AttributeReference, GreaterThanOrEqual, LessThan, Literal} +import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression, GreaterThanOrEqual, LessThan, Literal} import org.apache.spark.sql.functions.lit import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types.LongType @@ -249,4 +249,32 @@ class ConflictDataSkippingSuite extends QueryTest assert(geOnly.size == 8, s"expected 8 files with id >= 200, got ${geOnly.size}") } } + + test("filterFilesMatchingAnyReadPredicate OR-combines independent reads in one call") { + // Independent reads have OR semantics: a file survives if it could match ANY read. This is the + // multi-read path ConflictChecker uses -- all reads are evaluated together in a single job. + withTempDir { dir => + createTable(dir) + val log = DeltaLog.forTable(spark, dir.getCanonicalPath) + val snapshot = log.update() + val allFiles = snapshot.allFiles.collect().toSeq + assert(allFiles.size == 10) + + val id = AttributeReference("id", LongType)() + // Read 1: id in [200, 300) -> the single [200,300) file. Read 2: id >= 800 -> the [800,900) + // and [900,1000) files. Their union (OR across reads) is exactly 3 files. + val read1 = Seq[Expression]( + GreaterThanOrEqual(id, Literal(200L)), LessThan(id, Literal(300L))) + val read2 = Seq[Expression](GreaterThanOrEqual(id, Literal(800L))) + val survivors = + snapshot.filterFilesMatchingAnyReadPredicate(allFiles, Seq(read1, read2)) + assert(survivors.size == 3, + s"expected [200,300) + [800,1000), got ${survivors.map(_.path).sorted}") + + // A read with no usable predicate matches everything, so nothing can be skipped -> all kept. + val allKept = snapshot.filterFilesMatchingAnyReadPredicate(allFiles, Seq(read1, Seq.empty)) + assert(allKept.size == allFiles.size, + s"a read with no predicate must keep all files, got ${allKept.size}") + } + } } From e4ac0dab359c229a0b4a3f0700051ff8e20092a7 Mon Sep 17 00:00:00 2001 From: Eunjin Song Date: Wed, 12 Aug 2026 22:59:17 -0700 Subject: [PATCH 04/29] Address review: isolate conflict-time skipping in a self-typed trait Move buildDataSkippingPredicate / filterFilesByDataSkipping / filterFilesMatchingAnyReadPredicate out of DataSkippingReaderBase into a new self-typed trait ConflictDataSkippingReader (self: DataSkippingReaderBase), mixed into the base. Existing data-skipping code is left untouched: withStatsInternal0 is reverted to its original inline form, so the only change to DataSkippingReader.scala is the one-line trait mixin. parseAndDecodeStats is duplicated (trait-private) rather than shared with withStatsInternal0, keeping the feature a fully isolated additive unit per review feedback. Co-Authored-By: Claude Opus 4.8 --- .../stats/ConflictDataSkippingReader.scala | 172 ++++++++++++++++++ .../sql/delta/stats/DataSkippingReader.scala | 125 +------------ 2 files changed, 178 insertions(+), 119 deletions(-) create mode 100644 spark/src/main/scala/org/apache/spark/sql/delta/stats/ConflictDataSkippingReader.scala diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/stats/ConflictDataSkippingReader.scala b/spark/src/main/scala/org/apache/spark/sql/delta/stats/ConflictDataSkippingReader.scala new file mode 100644 index 00000000000..2305397b7f5 --- /dev/null +++ b/spark/src/main/scala/org/apache/spark/sql/delta/stats/ConflictDataSkippingReader.scala @@ -0,0 +1,172 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.stats + +import scala.util.control.NonFatal + +import org.apache.spark.sql.delta.ClassicColumnConversions._ +import org.apache.spark.sql.delta.DeltaTableUtils +import org.apache.spark.sql.delta.actions.AddFile +import org.apache.spark.sql.delta.expressions.DecodeNestedZ85EncodedVariant +import org.apache.spark.sql.delta.metering.DeltaLogging +import org.apache.spark.sql.delta.schema.SchemaUtils +import org.apache.spark.sql.delta.sources.DeltaSQLConf + +import org.apache.spark.sql.Column +import org.apache.spark.sql.catalyst.expressions._ +import org.apache.spark.sql.functions.{col, from_json} + +/** + * Reader-side data skipping used during conflict detection (row-level concurrency Case 1: writers + * touching disjoint data ranges). + * + * Mixed into [[DataSkippingReaderBase]] as a self-typed trait so this feature is a single isolated + * unit that only ADDS methods -- it does not modify any existing data-skipping code. It is opt-in: + * the caller ([[org.apache.spark.sql.delta.ConflictChecker]]) invokes these methods only behind + * `DeltaSQLConf.DELTA_CONFLICT_DETECTION_DATA_SKIPPING_ENABLED`, and each is one-way safe -- a file + * is dropped only when its stats *prove* it cannot match, so a real conflict is never a false + * negative. + */ +trait ConflictDataSkippingReader extends DeltaLogging { self: DataSkippingReaderBase => + + /** + * Parses (and, when the schema contains VariantType, Z85-decodes) the JSON `statsCol` into the + * `statsSchema` struct, mirroring [[withStatsInternal0]]. Deliberately kept private to this trait + * rather than shared with `withStatsInternal0`, so the existing stats path stays untouched. + */ + private def parseAndDecodeStats(statsCol: Column): Column = { + val parsedStats = from_json(statsCol, statsSchema) + // Only use DecodeNestedZ85EncodedVariant if the schema contains VariantType. + // This avoids performance overhead for tables without variant columns. + // `DecodeNestedZ85EncodedVariant` is a temporary workaround since the Spark 4.1 from_json + // expression has no way to decode a VariantVal from an encoded Z85 string. + // TODO: Add Z85 decoding to Variant in Spark 4.2 and use that from_json option here. + if (SchemaUtils.checkForVariantTypeColumnsRecursively(statsSchema)) { + Column(DecodeNestedZ85EncodedVariant(parsedStats.expr)) + } else { + parsedStats + } + } + + /** + * Builds a single [[DataSkippingPredicate]] (skipping expression + the stats it references) from + * `dataFilters`, mirroring [[filesForScan]]'s eligibility filtering, per-filter construction and + * conjunction fold. Returns None when stats skipping is unavailable or no eligible filter yields + * a predicate. The filters are AND-combined, so callers must pass filters from a single logical + * read (predicates from independent reads have OR, not AND, semantics). + */ + private[delta] def buildDataSkippingPredicate( + dataFilters: Seq[Expression]): Option[DataSkippingPredicate] = { + import DeltaTableUtils._ + // Stats-skipping conf, inlined from DataSkippingReaderBase.useStats (file-private, so not + // visible here). Reading it inline avoids widening that member's visibility. + if (!spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_STATS_SKIPPING)) return None + // Mirror filesForScan eligibility: drop subquery / non-deterministic / metadata filters, so we + // never build a skipping predicate that could wrongly exclude a matching file. + val eligibleFilters = dataFilters.filterNot { f => + containsSubquery(f) || !f.deterministic || f.exists { + case MetadataAttribute(_) => true + case _ => false + } + } + val constructDataFilters = new DataFiltersBuilder( + spark = spark, + dataSkippingType = DeltaDataSkippingType.dataSkippingOnlyV1, + getStatsColumnOpt = (s: StatsColumn) => getStatsColumnOpt(s)) + eligibleFilters + .flatMap(f => constructDataFilters(f)) + .reduceOption((skip1, skip2) => DataSkippingPredicate( + skip1.expr && skip2.expr, skip1.referencedStats ++ skip2.referencedStats)) + } + + /** + * Conflict-detection helper (reader-side data skipping): returns the subset of `files` whose + * statistics do NOT prove they fail `dataFilters` -- i.e. the files that could still match and + * therefore must be treated as conflicts. Files with missing/insufficient stats (or when skipping + * is unavailable) are kept. + * + * `dataFilters` must come from a single logical read: they are AND-combined via + * [[buildDataSkippingPredicate]]. This is the one-read case of + * [[filterFilesMatchingAnyReadPredicate]]; callers with multiple independent reads should use + * that method so all reads are evaluated in a single Spark job. + */ + private[delta] def filterFilesByDataSkipping( + files: Seq[AddFile], + dataFilters: Seq[Expression]): Seq[AddFile] = + filterFilesMatchingAnyReadPredicate(files, Seq(dataFilters)) + + /** + * Conflict-detection helper (reader-side data skipping) over several INDEPENDENT reads: returns + * the subset of `files` that could still match ANY read and therefore must be treated as + * conflicts. Each inner `Seq[Expression]` is one logical read's data filters (AND-combined via + * [[buildDataSkippingPredicate]]); the reads are OR-combined, matching read semantics -- a file + * is a candidate if it could match any one of them. + * + * All reads are evaluated in a SINGLE Spark job: the per-read skipping predicates are OR-ed + * together into one `where` clause, rather than filtering per read and unioning the survivors + * (which launched one job per read). Note we cannot instead flatten every read's filters into one + * predicate -- that would AND them (read1 AND read2), the opposite of the OR we need. + * + * One-way safe: each read contributes `expr || !verifyStatsForFilter(...)` exactly as + * [[getDataSkippedFiles]], so a file is dropped only when its stats *prove* it fails EVERY read + * -- a real conflict is never a false negative. Files with missing/insufficient stats, a + * read with no usable skipping predicate (empty / ineligible filters -> matches everything), or a + * table without stats are all kept. Returns the original [[AddFile]]s (matched by path). + * + * Fail-safe: skipping here is a pure optimization over correct (conservative) conflict detection, + * so if building or evaluating the predicate throws we fall back to the default behavior of + * keeping all `files` as conflict candidates rather than failing the commit. + */ + private[delta] def filterFilesMatchingAnyReadPredicate( + files: Seq[AddFile], + dataFiltersPerRead: Seq[Seq[Expression]]): Seq[AddFile] = { + import org.apache.spark.sql.delta.implicits._ + if (files.isEmpty || dataFiltersPerRead.isEmpty || schema.isEmpty) return files + try { + // One skipping predicate per read. A read with no usable predicate (empty or ineligible + // filters) matches every file -> nothing can be skipped, so keep all files without a job. + val perReadPredicates = dataFiltersPerRead.map { dataFilters => + if (dataFilters.isEmpty) None else buildDataSkippingPredicate(dataFilters) + } + if (perReadPredicates.exists(_.isEmpty)) return files + // Survive if the file could match ANY read. Per read, `expr || !verifyStatsForFilter(...)` + // keeps any file whose referenced stats are missing/NULL (mirrors getDataSkippedFiles): only + // skip when stats prove no match. OR the reads so a match against any one keeps the file. + val survivorCondition = perReadPredicates.flatten + .map(pred => pred.expr || !verifyStatsForFilter(pred.referencedStats)) + .reduce(_ || _) + val survivingPaths = recordFrameProfile( + "Delta", "DataSkippingReader.filterFilesMatchingAnyReadPredicate") { + files.toDF(spark) + .withColumn("stats", parseAndDecodeStats(col("stats"))) + .where(survivorCondition) + .select("path") + .collect() + .map(_.getString(0)) + .toSet + } + files.filter(f => survivingPaths.contains(f.path)) + } catch { + case NonFatal(e) => + // Optimization only: never let a skipping failure abort a commit. Fall back to the default + // (feature-off) behavior of treating every added file as a conflict candidate. + logWarning(log"Conflict-time data skipping failed to evaluate; falling back to treating " + + log"all added files as conflict candidates", e) + files + } + } +} diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/stats/DataSkippingReader.scala b/spark/src/main/scala/org/apache/spark/sql/delta/stats/DataSkippingReader.scala index 6c9673c55a4..73999236511 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/stats/DataSkippingReader.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/stats/DataSkippingReader.scala @@ -20,7 +20,6 @@ package org.apache.spark.sql.delta.stats import java.io.Closeable import scala.collection.mutable.ArrayBuffer -import scala.util.control.NonFatal import org.apache.spark.sql.delta.skipping.clustering.{ClusteredTableUtils, ClusteringColumnInfo} import org.apache.spark.sql.delta.ClassicColumnConversions._ @@ -265,6 +264,7 @@ trait DataSkippingReaderBase with StatisticsCollection with ReadsMetadataFields with StateCache + with ConflictDataSkippingReader with DeltaLogging { import DataSkippingReader._ @@ -287,29 +287,22 @@ trait DataSkippingReaderBase DeltaSQLConf.DELTA_DATASKIPPING_PARTITION_LIKE_FILTERS_ADDITIONAL_SUPPORTED_EXPRESSIONS) .toSet.flatMap((exprs: String) => exprs.split(",")) - /** - * Parses (and, when the schema contains VariantType, Z85-decodes) the JSON `statsCol` into the - * `statsSchema` struct. Shared by [[withStatsInternal0]] and conflict-detection data skipping - * ([[filterFilesByDataSkipping]]) so both parse stats the same way. - */ - private def parseAndDecodeStats(statsCol: Column): Column = { - val parsedStats = from_json(statsCol, statsSchema) + /** Returns a DataFrame expression to obtain a list of files with parsed statistics. */ + private def withStatsInternal0: DataFrame = { + val parsedStats = from_json(col("stats"), statsSchema) // Only use DecodeNestedZ85EncodedVariant if the schema contains VariantType. // This avoids performance overhead for tables without variant columns. // `DecodeNestedZ85EncodedVariant` is a temporary workaround since the Spark 4.1 from_json // expression has no way to decode a VariantVal from an encoded Z85 string. // TODO: Add Z85 decoding to Variant in Spark 4.2 and use that from_json option here. - if (SchemaUtils.checkForVariantTypeColumnsRecursively(statsSchema)) { + val decodedStats = if (SchemaUtils.checkForVariantTypeColumnsRecursively(statsSchema)) { Column(DecodeNestedZ85EncodedVariant(parsedStats.expr)) } else { parsedStats } + allFiles.withColumn("stats", decodedStats) } - /** Returns a DataFrame expression to obtain a list of files with parsed statistics. */ - private def withStatsInternal0: DataFrame = - allFiles.withColumn("stats", parseAndDecodeStats(col("stats"))) - private lazy val withStatsCache = cacheDS(withStatsInternal0, s"Delta Table State with Stats #$version - $redactedPath") @@ -649,112 +642,6 @@ trait DataSkippingReaderBase files.toSeq -> Seq(DataSize(totalSize), DataSize(partitionSize), DataSize(scanSize)) } - /** - * Builds a single [[DataSkippingPredicate]] (skipping expression + the stats it references) from - * `dataFilters`, mirroring [[filesForScan]]'s eligibility filtering, per-filter construction and - * conjunction fold. Returns None when stats skipping is unavailable or no eligible filter yields - * a predicate. The filters are AND-combined, so callers must pass filters from a single logical - * read (predicates from independent reads have OR, not AND, semantics). - */ - private[delta] def buildDataSkippingPredicate( - dataFilters: Seq[Expression]): Option[DataSkippingPredicate] = { - import DeltaTableUtils._ - if (!useStats) return None - // Mirror filesForScan eligibility: drop subquery / non-deterministic / metadata filters, so we - // never build a skipping predicate that could wrongly exclude a matching file. - val eligibleFilters = dataFilters.filterNot { f => - containsSubquery(f) || !f.deterministic || f.exists { - case MetadataAttribute(_) => true - case _ => false - } - } - val constructDataFilters = new DataFiltersBuilder( - spark = spark, - dataSkippingType = DeltaDataSkippingType.dataSkippingOnlyV1, - getStatsColumnOpt = (s: StatsColumn) => getStatsColumnOpt(s)) - eligibleFilters - .flatMap(f => constructDataFilters(f)) - .reduceOption((skip1, skip2) => DataSkippingPredicate( - skip1.expr && skip2.expr, skip1.referencedStats ++ skip2.referencedStats)) - } - - /** - * Conflict-detection helper (reader-side data skipping): returns the subset of `files` whose - * statistics do NOT prove they fail `dataFilters` -- i.e. the files that could still match and - * therefore must be treated as conflicts. Files with missing/insufficient stats (or when skipping - * is unavailable) are kept. - * - * `dataFilters` must come from a single logical read: they are AND-combined via - * [[buildDataSkippingPredicate]]. This is the one-read case of - * [[filterFilesMatchingAnyReadPredicate]]; callers with multiple independent reads should use - * that method so all reads are evaluated in a single Spark job. - */ - private[delta] def filterFilesByDataSkipping( - files: Seq[AddFile], - dataFilters: Seq[Expression]): Seq[AddFile] = - filterFilesMatchingAnyReadPredicate(files, Seq(dataFilters)) - - /** - * Conflict-detection helper (reader-side data skipping) over several INDEPENDENT reads: returns - * the subset of `files` that could still match ANY read and therefore must be treated as - * conflicts. Each inner `Seq[Expression]` is one logical read's data filters (AND-combined via - * [[buildDataSkippingPredicate]]); the reads are OR-combined, matching read semantics -- a file - * is a candidate if it could match any one of them. - * - * All reads are evaluated in a SINGLE Spark job: the per-read skipping predicates are OR-ed - * together into one `where` clause, rather than filtering per read and unioning the survivors - * (which launched one job per read). Note we cannot instead flatten every read's filters into one - * predicate -- that would AND them (read1 AND read2), the opposite of the OR we need. - * - * One-way safe: each read contributes `expr || !verifyStatsForFilter(...)` exactly as - * [[getDataSkippedFiles]], so a file is dropped only when its stats *prove* it fails EVERY read - * -- a real conflict is never a false negative. Files with missing/insufficient stats, a - * read with no usable skipping predicate (empty / ineligible filters -> matches everything), or a - * table without stats are all kept. Returns the original [[AddFile]]s (matched by path). - * - * Fail-safe: skipping here is a pure optimization over correct (conservative) conflict detection, - * so if building or evaluating the predicate throws we fall back to the default behavior of - * keeping all `files` as conflict candidates rather than failing the commit. - */ - private[delta] def filterFilesMatchingAnyReadPredicate( - files: Seq[AddFile], - dataFiltersPerRead: Seq[Seq[Expression]]): Seq[AddFile] = { - import org.apache.spark.sql.delta.implicits._ - if (files.isEmpty || dataFiltersPerRead.isEmpty || schema.isEmpty) return files - try { - // One skipping predicate per read. A read with no usable predicate (empty or ineligible - // filters) matches every file -> nothing can be skipped, so keep all files without a job. - val perReadPredicates = dataFiltersPerRead.map { dataFilters => - if (dataFilters.isEmpty) None else buildDataSkippingPredicate(dataFilters) - } - if (perReadPredicates.exists(_.isEmpty)) return files - // Survive if the file could match ANY read. Per read, `expr || !verifyStatsForFilter(...)` - // keeps any file whose referenced stats are missing/NULL (mirrors getDataSkippedFiles): only - // skip when stats prove no match. OR the reads so a match against any one keeps the file. - val survivorCondition = perReadPredicates.flatten - .map(pred => pred.expr || !verifyStatsForFilter(pred.referencedStats)) - .reduce(_ || _) - val survivingPaths = recordFrameProfile( - "Delta", "DataSkippingReader.filterFilesMatchingAnyReadPredicate") { - files.toDF(spark) - .withColumn("stats", parseAndDecodeStats(col("stats"))) - .where(survivorCondition) - .select("path") - .collect() - .map(_.getString(0)) - .toSet - } - files.filter(f => survivingPaths.contains(f.path)) - } catch { - case NonFatal(e) => - // Optimization only: never let a skipping failure abort a commit. Fall back to the default - // (feature-off) behavior of treating every added file as a conflict candidate. - logWarning(log"Conflict-time data skipping failed to evaluate; falling back to treating " + - log"all added files as conflict candidates", e) - files - } - } - private def getCorrectDataSkippingType( dataSkippingType: DeltaDataSkippingType): DeltaDataSkippingType = { dataSkippingType From 0e1c865d0a033859b7dd3d4a311d5afbc10f0167 Mon Sep 17 00:00:00 2001 From: seewishnew Date: Fri, 9 Oct 2026 19:25:39 -0700 Subject: [PATCH 05/29] [Spark] Preserve KernelContext when wrapping Kernel snapshots (#7818) #### Which Delta project/connector is this regarding? - [x] Spark ## Description Retain the loading `KernelContext` when wrapping a Kernel snapshot in `DeltaV2Snapshot`, and use its default engine for subsequent snapshot operations. This keeps snapshot operations associated with the table's context and configured filesystem options instead of constructing an engine from session defaults alone. Thread the context through snapshot managers, transactions, and snapshot copies, and update the affected callers and test fixtures. The snapshot managers, table, and transaction adapter retain context instead of a default-engine field. The context creates a default engine when requested, materializing Hadoop configuration from the active Spark session plus the table's invariant filesystem options. No per-session engine cache or credential-provider refresh policy is introduced. ## How was this patch tested? - `spark/testOnly org.apache.spark.sql.delta.v2.interop.DeltaV2SnapshotSuite`: all 23 tests passed. - `sparkV2/Test/compile` passed, including Java formatting and style checks. - Negative control: restoring the old lazy getter makes the new filesystem regression fail with stale session properties; restoring the fix makes it pass. - Added regression coverage for context-engine delegation on latest and historical snapshots and context retention in catalog copies. - Added a real recording-filesystem regression across sessions A/B/A and inherited child-thread sessions, exercising latest/historical snapshots, allFiles, and transaction-version lookup while checking session properties and table-option precedence. ## Does this PR introduce _any_ user-facing changes? No changes to released APIs. This fixes engine selection in the unreleased Spark connector's snapshot facade. --------- Signed-off-by: Vishnu Chandrashekhar Co-authored-by: Vishnu Chandrashekhar --- .../spark/internal/v2/DeltaV2TestBase.java | 21 +- ...eltaV2ChangelogCatalogIntegrationTest.java | 2 +- ...taV2ChangelogDirectBatchExecutionTest.java | 12 +- .../changelog/DeltaV2ChangelogTestBase.java | 4 +- .../internal/ApplyV2ReadOptionsSuite.scala | 3 +- .../DeltaV2OptimisticTransactionSuite.scala | 29 ++- .../v2/interop/DeltaV2SnapshotSuite.scala | 182 +++++++++++++++--- .../v2/interop/KernelSnapshotUtilsSuite.scala | 12 +- .../internal/v2/catalog/DeltaV2Table.java | 16 +- .../v2/snapshot/PathBasedSnapshotManager.java | 32 ++- .../v2/snapshot/SnapshotManagerFactory.java | 14 +- .../UCManagedTableSnapshotManager.java | 27 ++- .../spark/internal/v2/write/DeltaV2Write.java | 3 +- .../internal/v2/kernel/KernelContext.scala | 19 +- .../tablemanager/CachedSnapshotManager.scala | 2 +- .../DeltaV2OptimisticTransaction.scala | 11 +- .../delta/v2/interop/DeltaV2Snapshot.scala | 15 +- .../v2/interop/DeltaV2SnapshotManager.scala | 10 +- .../spark/internal/v2/DeltaV2TestBase.java | 21 +- .../internal/v2/catalog/DeltaV2TableTest.java | 8 +- .../read/DeltaV2MicroBatchStreamCDCTest.java | 27 ++- .../v2/read/DeltaV2MicroBatchStreamTest.java | 73 +++---- .../v2/read/DeltaV2ScanBuilderTest.java | 2 +- .../v2/read/MetadataEvolutionHandlerTest.java | 14 +- .../internal/v2/read/ScanFileRDDTest.java | 6 +- .../PathBasedSnapshotManagerTest.java | 30 +-- .../SerializableReadOnlySnapshotTest.java | 6 +- .../v2/utils/StreamingHelperTest.java | 39 ++-- .../DeltaRowLevelOperationBuilderTest.java | 13 +- .../v2/write/DeltaV2BatchWriteTest.java | 13 +- .../v2/write/DeltaV2StreamingWriteTest.java | 6 +- .../v2/write/DeltaV2WriteContextTest.java | 27 +-- .../internal/v2/write/DeltaV2WriteTest.java | 8 +- .../write/DeltaV2WriterCommitMessageTest.java | 2 +- .../UCManagedTableSnapshotManagerSuite.scala | 16 +- .../CachedSnapshotManagerSuite.scala | 4 +- .../DeltaV2TableManagerImplSuite.scala | 8 +- 37 files changed, 458 insertions(+), 279 deletions(-) diff --git a/spark-unified/src/test/java/io/delta/spark/internal/v2/DeltaV2TestBase.java b/spark-unified/src/test/java/io/delta/spark/internal/v2/DeltaV2TestBase.java index b00222ee3f2..f547f3c75fe 100644 --- a/spark-unified/src/test/java/io/delta/spark/internal/v2/DeltaV2TestBase.java +++ b/spark-unified/src/test/java/io/delta/spark/internal/v2/DeltaV2TestBase.java @@ -15,9 +15,14 @@ */ package io.delta.spark.internal.v2; -import io.delta.kernel.defaults.engine.DefaultEngine; import io.delta.kernel.engine.Engine; +import io.delta.spark.internal.v2.kernel.KernelContext; +import io.delta.spark.internal.v2.utils.ScalaUtils; +import java.util.HashMap; +import java.util.Map; +import org.apache.hadoop.conf.Configuration; import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.delta.storage.LogStore; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -25,6 +30,7 @@ public abstract class DeltaV2TestBase { protected static SparkSession spark; protected static Engine defaultEngine; + protected static KernelContext defaultKernelContext; @BeforeAll public static void setUpSparkAndEngine() { @@ -39,7 +45,18 @@ public static void setUpSparkAndEngine() { .config("spark.sql.catalog.dsv2", "io.delta.spark.internal.v2.catalog.TestCatalog") .config("spark.sql.catalog.dsv2.base_path", System.getProperty("java.io.tmpdir")) .getOrCreate(); - defaultEngine = DefaultEngine.create(spark.sessionState().newHadoopConf()); + defaultKernelContext = createKernelContext(); + defaultEngine = defaultKernelContext.getDefaultEngine(); + } + + protected static KernelContext createKernelContext() { + return createKernelContext(spark.sessionState().newHadoopConf()); + } + + protected static KernelContext createKernelContext(Configuration hadoopConf) { + Map options = new HashMap<>(); + hadoopConf.forEach(entry -> options.put(entry.getKey(), entry.getValue())); + return new KernelContext(ScalaUtils.toScalaMap(options), LogStore.apply(spark)); } @AfterAll diff --git a/spark-unified/src/test/scala-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaV2ChangelogCatalogIntegrationTest.java b/spark-unified/src/test/scala-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaV2ChangelogCatalogIntegrationTest.java index 6668babc765..20216fa2535 100644 --- a/spark-unified/src/test/scala-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaV2ChangelogCatalogIntegrationTest.java +++ b/spark-unified/src/test/scala-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaV2ChangelogCatalogIntegrationTest.java @@ -107,7 +107,7 @@ private java.sql.Timestamp commitTimestamp(String tableLocation, long version) { // Reads through the kernel snapshot manager, so it works inside the STRICT block the test body // runs in. tableLocation is resolved by withHistoryTable before STRICT is set. DeltaV2SnapshotManager snapshotManager = - SnapshotManagerFactory.create(tableLocation, defaultEngine, Optional.empty()); + SnapshotManagerFactory.create(tableLocation, defaultKernelContext, Optional.empty()); long millis = snapshotManager.loadSnapshotAt(version).timestamp(); return new java.sql.Timestamp(millis); } diff --git a/spark-unified/src/test/scala-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaV2ChangelogDirectBatchExecutionTest.java b/spark-unified/src/test/scala-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaV2ChangelogDirectBatchExecutionTest.java index 547959b7d7a..0f8208ff647 100644 --- a/spark-unified/src/test/scala-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaV2ChangelogDirectBatchExecutionTest.java +++ b/spark-unified/src/test/scala-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaV2ChangelogDirectBatchExecutionTest.java @@ -77,7 +77,7 @@ public void testDirectBatchExecutionWithExplicitExpectedRows(boolean enableDvs) spark.sql(String.format("DELETE FROM %s WHERE id = 1", tableName)); DeltaV2SnapshotManager snapshotManager = - SnapshotManagerFactory.create(tablePath, defaultEngine, Optional.empty()); + SnapshotManagerFactory.create(tablePath, defaultKernelContext, Optional.empty()); StructType dataSchema = spark.table(tableName).schema(); long latestVersion = snapshotManager.loadLatestSnapshot().version(); Map commitTimestampsMicros = loadCommitTimestampsMicros(tableName); @@ -149,7 +149,7 @@ public void testRowTrackingMetadataReferences() throws Exception { spark.sql(String.format("INSERT INTO %s VALUES (1)", tableName)); DeltaV2SnapshotManager snapshotManager = - SnapshotManagerFactory.create(tablePath, defaultEngine, Optional.empty()); + SnapshotManagerFactory.create(tablePath, defaultKernelContext, Optional.empty()); StructType dataSchema = spark.table(tableName).schema(); long latestVersion = snapshotManager.loadLatestSnapshot().version(); @@ -200,7 +200,7 @@ public void testUpdateProducesPairedDeleteAndInsert(boolean enableDvs) throws Ex spark.sql(String.format("UPDATE %s SET name = 'AliceX' WHERE id = 1", tableName)); DeltaV2SnapshotManager snapshotManager = - SnapshotManagerFactory.create(tablePath, defaultEngine, Optional.empty()); + SnapshotManagerFactory.create(tablePath, defaultKernelContext, Optional.empty()); StructType dataSchema = spark.table(tableName).schema(); long latestVersion = snapshotManager.loadLatestSnapshot().version(); Map commitTimestampsMicros = loadCommitTimestampsMicros(tableName); @@ -273,7 +273,7 @@ public void testRangeSlicingNonZeroStart() throws Exception { spark.sql(String.format("INSERT INTO %s VALUES (3, 'Charlie')", tableName)); DeltaV2SnapshotManager snapshotManager = - SnapshotManagerFactory.create(tablePath, defaultEngine, Optional.empty()); + SnapshotManagerFactory.create(tablePath, defaultKernelContext, Optional.empty()); StructType dataSchema = spark.table(tableName).schema(); Map commitTimestampsMicros = loadCommitTimestampsMicros(tableName); @@ -342,7 +342,7 @@ public void testChangelogWithReservedCharsInPath() throws Exception { spark.sql(String.format("INSERT INTO %s VALUES (1, 'Alice'), (2, 'Bob')", tableName)); DeltaV2SnapshotManager snapshotManager = - SnapshotManagerFactory.create(tablePath, defaultEngine, Optional.empty()); + SnapshotManagerFactory.create(tablePath, defaultKernelContext, Optional.empty()); long latestVersion = snapshotManager.loadLatestSnapshot().version(); Map commitTimestampsMicros = loadCommitTimestampsMicros(tableName); @@ -407,7 +407,7 @@ public void testChangelogWithReservedCharsInDataFileNames() throws Exception { spark.sql(String.format("INSERT INTO %s VALUES (1, 'Alice'), (2, 'Bob')", tableName)); DeltaV2SnapshotManager snapshotManager = - SnapshotManagerFactory.create(tablePath, defaultEngine, Optional.empty()); + SnapshotManagerFactory.create(tablePath, defaultKernelContext, Optional.empty()); long latestVersion = snapshotManager.loadLatestSnapshot().version(); Map commitTimestampsMicros = loadCommitTimestampsMicros(tableName); diff --git a/spark-unified/src/test/scala-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaV2ChangelogTestBase.java b/spark-unified/src/test/scala-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaV2ChangelogTestBase.java index d7e43880ded..36650027f83 100644 --- a/spark-unified/src/test/scala-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaV2ChangelogTestBase.java +++ b/spark-unified/src/test/scala-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaV2ChangelogTestBase.java @@ -15,7 +15,6 @@ */ package io.delta.spark.internal.v2.read.changelog; -import io.delta.kernel.defaults.engine.DefaultEngine; import io.delta.spark.internal.v2.DeltaV2TestBase; import org.apache.spark.sql.SparkSession; import org.junit.jupiter.api.BeforeAll; @@ -57,6 +56,7 @@ public static void setUpChangelogSparkAndEngine() { .config("spark.sql.catalog.dsv2.base_path", System.getProperty("java.io.tmpdir")) .config("spark.databricks.delta.changelogV2.enabled", "true") .getOrCreate(); - defaultEngine = DefaultEngine.create(spark.sessionState().newHadoopConf()); + defaultKernelContext = createKernelContext(); + defaultEngine = defaultKernelContext.getDefaultEngine(); } } diff --git a/spark-unified/src/test/scala/io/delta/internal/ApplyV2ReadOptionsSuite.scala b/spark-unified/src/test/scala/io/delta/internal/ApplyV2ReadOptionsSuite.scala index 23c76a69f8a..bc15c913876 100644 --- a/spark-unified/src/test/scala/io/delta/internal/ApplyV2ReadOptionsSuite.scala +++ b/spark-unified/src/test/scala/io/delta/internal/ApplyV2ReadOptionsSuite.scala @@ -34,6 +34,7 @@ import org.apache.spark.sql.delta.sources.{DeltaSourceMetadataTrackingLog, Delta import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.util.CaseInsensitiveStringMap import io.delta.spark.internal.v2.catalog.DeltaV2Table +import io.delta.spark.internal.v2.kernel.KernelContext import io.delta.spark.internal.v2.snapshot.PathBasedSnapshotManager import io.delta.storage.commit.uccommitcoordinator.UCCommitCoordinatorClient @@ -270,7 +271,7 @@ class ApplyV2ReadOptionsSuite extends DeltaSQLCommandTest { private def seedSchemaLogWithExtraColumn(tablePath: String, schemaLogPath: String): Unit = { val deltaLog = DeltaLog.forTable(spark, tablePath) val snapshotManager = - new PathBasedSnapshotManager(tablePath, deltaLog.newDeltaHadoopConf()) + new PathBasedSnapshotManager(tablePath, new KernelContext(deltaLog.options, deltaLog.store)) val tableId = snapshotManager.loadLatestSnapshot.metadata.getId val trackingLog = DeltaSourceMetadataTrackingLog.create( spark, schemaLogPath, tableId, tablePath, parameters = Map.empty[String, String]) diff --git a/spark-unified/src/test/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2OptimisticTransactionSuite.scala b/spark-unified/src/test/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2OptimisticTransactionSuite.scala index 5a16976f295..5c03a02709d 100644 --- a/spark-unified/src/test/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2OptimisticTransactionSuite.scala +++ b/spark-unified/src/test/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2OptimisticTransactionSuite.scala @@ -21,8 +21,9 @@ import java.nio.file.{Files, StandardCopyOption} import org.apache.spark.sql.delta.{DeltaColumnMapping, DeltaOperations, OptimisticTransaction, OptimisticTransactionSuite} import org.apache.spark.sql.delta.actions.{AddFile, Metadata, Protocol, SetTransaction} +import org.apache.spark.sql.delta.storage.LogStore import org.apache.spark.sql.delta.test.V2ForceTest -import io.delta.spark.internal.v2.kernel.KernelEngineFactory +import io.delta.spark.internal.v2.kernel.KernelContext import org.apache.hadoop.fs.Path import io.delta.kernel.{Table => KernelTable} import io.delta.kernel.internal.{SnapshotImpl => KernelSnapshotImpl} @@ -48,15 +49,14 @@ class DeltaV2OptimisticTransactionSuite * inherited V1 conflict test cases. */ override protected def startTestTransaction(dataPath: Path): OptimisticTransaction = { - // scalastyle:off deltahadoopconfiguration - val kernelEngine = KernelEngineFactory.createDefaultEngine(spark.sessionState.newHadoopConf()) - // scalastyle:on deltahadoopconfiguration + val kernelContext = new KernelContext(Map.empty, LogStore(spark)) + val kernelEngine = kernelContext.getDefaultEngine() val kernelSnap = KernelTable .forPath(kernelEngine, dataPath.toString) .getLatestSnapshot(kernelEngine) .asInstanceOf[KernelSnapshotImpl] - val deltaV2Snapshot = new DeltaV2Snapshot(kernelSnap) - new DeltaV2OptimisticTransaction(catalogTable = None, deltaV2Snapshot, kernelEngine) + val deltaV2Snapshot = new DeltaV2Snapshot(kernelSnap, kernelContext) + new DeltaV2OptimisticTransaction(catalogTable = None, deltaV2Snapshot, kernelContext) } override protected def testDefaultMetadata(): Metadata = @@ -93,27 +93,24 @@ class DeltaV2OptimisticTransactionSuite /** Builds a Kernel-backed transaction over the latest snapshot of the table at `dir`. */ private def startKernelTxn(dir: File): DeltaV2OptimisticTransaction = { - // scalastyle:off deltahadoopconfiguration - // No DeltaLog here (the snapshot is loaded via Kernel), so use the session Hadoop conf. - val kernelEngine = KernelEngineFactory.createDefaultEngine(spark.sessionState.newHadoopConf()) - // scalastyle:on deltahadoopconfiguration + val kernelContext = new KernelContext(Map.empty, LogStore(spark)) + val kernelEngine = kernelContext.getDefaultEngine() val kernelSnap = KernelTable .forPath(kernelEngine, dir.getCanonicalPath) .getLatestSnapshot(kernelEngine) .asInstanceOf[KernelSnapshotImpl] - val deltaV2Snapshot = new DeltaV2Snapshot(kernelSnap) - new DeltaV2OptimisticTransaction(catalogTable = None, deltaV2Snapshot, kernelEngine) + val deltaV2Snapshot = new DeltaV2Snapshot(kernelSnap, kernelContext) + new DeltaV2OptimisticTransaction(catalogTable = None, deltaV2Snapshot, kernelContext) } private def latestKernelSnapshot(dir: File): DeltaV2Snapshot = { - // scalastyle:off deltahadoopconfiguration - val kernelEngine = KernelEngineFactory.createDefaultEngine(spark.sessionState.newHadoopConf()) - // scalastyle:on deltahadoopconfiguration + val kernelContext = new KernelContext(Map.empty, LogStore(spark)) + val kernelEngine = kernelContext.getDefaultEngine() val kernelSnap = KernelTable .forPath(kernelEngine, dir.getCanonicalPath) .getLatestSnapshot(kernelEngine) .asInstanceOf[KernelSnapshotImpl] - new DeltaV2Snapshot(kernelSnap) + new DeltaV2Snapshot(kernelSnap, kernelContext) } /** Seeds a simple (unpartitioned) V1 Delta table at `dir`. */ diff --git a/spark-unified/src/test/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2SnapshotSuite.scala b/spark-unified/src/test/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2SnapshotSuite.scala index 72039e14338..5d3e228e7ea 100644 --- a/spark-unified/src/test/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2SnapshotSuite.scala +++ b/spark-unified/src/test/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2SnapshotSuite.scala @@ -16,17 +16,27 @@ package org.apache.spark.sql.delta.v2.interop +import java.net.URI +import java.util.concurrent.{CopyOnWriteArrayList, FutureTask} + +import scala.collection.JavaConverters._ + import org.apache.spark.sql.delta.{DeltaLog, DeltaOperations, NoMapping, Snapshot} import org.apache.spark.sql.delta.actions.DomainMetadata import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.stats.FileSizeHistogramUtils +import org.apache.spark.sql.delta.storage.LogStore import org.apache.spark.sql.delta.test.DeltaSQLCommandTest -import io.delta.spark.internal.v2.kernel.KernelEngineFactory -import org.apache.hadoop.fs.Path +import io.delta.spark.internal.v2.kernel.KernelContext +import io.delta.spark.internal.v2.snapshot.PathBasedSnapshotManager +import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.fs.{FileStatus, FSDataInputStream, Path, RawLocalFileSystem} +import org.mockito.Mockito.{atLeastOnce, clearInvocations, spy, verify, verifyNoInteractions} import io.delta.kernel.TableManager import io.delta.kernel.engine.Engine import io.delta.kernel.internal.{SnapshotImpl => KernelSnapshot} +import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable, CatalogTableType} @@ -38,23 +48,115 @@ import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable */ class DeltaV2SnapshotSuite extends DeltaSQLCommandTest { - // scalastyle:off deltahadoopconfiguration - // No DeltaLog in this test (the snapshot is loaded via Kernel), so use the session Hadoop conf. - private def engine: Engine = - KernelEngineFactory.createDefaultEngine(spark.sessionState.newHadoopConf()) - // scalastyle:on deltahadoopconfiguration + private lazy val kernelContext = new KernelContext(Map.empty, LogStore(spark)) + + private def engine: Engine = kernelContext.getDefaultEngine() /** Load the latest snapshot of the Delta table at `path` via Kernel. */ private def loadKernelSnapshot(path: String): KernelSnapshot = TableManager.loadSnapshot(path).build(engine).asInstanceOf[KernelSnapshot] + test("snapshot operations use the loading context's engine") { + withTempDir { dir => + val path = dir.getCanonicalPath + val appId = "snapshot-context-test" + spark.range(2).write.format("delta") + .option("txnAppId", appId).option("txnVersion", 0).save(path) + spark.range(2, 4).write.format("delta").mode("append") + .option("txnAppId", appId).option("txnVersion", 1).save(path) + + val context = spy(new KernelContext(Map.empty, LogStore(spark))) + val manager = new PathBasedSnapshotManager(path, context) + verifyNoInteractions(context) + val snapshots = Seq(manager.loadLatestSnapshot(), manager.loadSnapshotAt(0L)) + + snapshots.foreach { snapshot => + val v2 = snapshot.asInstanceOf[DeltaV2Snapshot] + val v1 = DeltaLog.forTable(spark, path).getSnapshotAt(v2.version) + val expectedFiles = v1.allFiles.collect().map(_.path).toSet + assert(v2.kernelContext eq context) + try { + clearInvocations(context) + assert(v2.allFiles.collect().map(_.path).toSet === expectedFiles) + verify(context, atLeastOnce()).getDefaultEngine() + clearInvocations(context) + assert(v2.timestamp === v1.timestamp) + verify(context, atLeastOnce()).getDefaultEngine() + clearInvocations(context) + assert(v2.getLatestTransactionVersion(appId).getAsLong === v2.version) + verify(context, atLeastOnce()).getDefaultEngine() + } finally { + v2.uncache() + } + } + } + } + + test("snapshot filesystem access follows the active session and child thread") { + withTempDir { dir => + val path = dir.getCanonicalPath + val appId = "snapshot-session-test" + spark.range(2).coalesce(1).write.format("delta") + .option("txnAppId", appId).option("txnVersion", 0).save(path) + val expectedFiles = + DeltaLog.forTable(spark, path).update().allFiles.collect().map(_.path).toSet + val invariantOptions = Map( + "fs.file.impl" -> classOf[SnapshotContextRecordingFileSystem].getName, + "fs.file.impl.disable.cache" -> "true", + SnapshotContextRecordingFileSystem.TableMarkerKey -> "table-option") + val context = new KernelContext(invariantOptions, LogStore(spark)) + val manager = new PathBasedSnapshotManager(path, context) + val originalSession = SparkSession.active + val sessionA = spark.newSession() + val sessionB = spark.newSession() + + def checkAccess(session: SparkSession, marker: String): Unit = { + assert(SparkSession.active eq session) + session.conf.set(SnapshotContextRecordingFileSystem.SessionMarkerKey, marker) + session.conf.set(SnapshotContextRecordingFileSystem.TableMarkerKey, "session-option") + SnapshotContextRecordingFileSystem.clear() + val snapshots = Seq(manager.loadLatestSnapshot(), manager.loadSnapshotAt(0L)) + try { + snapshots.foreach { snapshot => + val v2 = snapshot.asInstanceOf[DeltaV2Snapshot] + assert(v2.kernelContext eq context) + assert(v2.allFiles.collect().map(_.path).toSet === expectedFiles) + assert(v2.getLatestTransactionVersion(appId).getAsLong === 0L) + } + val observations = SnapshotContextRecordingFileSystem.observations + assert(observations.exists(_._1 == "listStatus"), observations.toString) + assert(observations.exists(_._1 == "open"), observations.toString) + assert(observations.forall { case (_, sessionMarker, tableMarker) => + sessionMarker == marker && tableMarker == "table-option" + }, observations.toString) + } finally { + snapshots.foreach(_.uncache()) + } + } + + try { + Seq(sessionA -> "A", sessionB -> "B", sessionA -> "A-again").foreach { + case (session, marker) => + SparkSession.setActiveSession(session) + checkAccess(session, marker) + val child = new FutureTask[Unit](() => checkAccess(session, s"$marker-child")) + new Thread(child, "snapshot-context-session-test").start() + child.get() + } + } finally { + SparkSession.setActiveSession(originalSession) + SnapshotContextRecordingFileSystem.clear() + } + } + } + test("path and version are derived from the wrapped Kernel snapshot") { withTempDir { dir => val path = dir.getCanonicalPath spark.range(5).write.format("delta").save(path) // version 0 val kernelSnapshot0 = loadKernelSnapshot(path) - val snapshot0 = new DeltaV2Snapshot(kernelSnapshot0) + val snapshot0 = new DeltaV2Snapshot(kernelSnapshot0, kernelContext) assert(snapshot0.version === kernelSnapshot0.getVersion) assert(snapshot0.version === 0L) assert(snapshot0.path === new Path(kernelSnapshot0.getDataPath.toString)) @@ -64,7 +166,7 @@ class DeltaV2SnapshotSuite extends DeltaSQLCommandTest { spark.range(1).write.format("delta").mode("append").save(path) // version 2 val kernelSnapshot2 = loadKernelSnapshot(path) - val snapshot2 = new DeltaV2Snapshot(kernelSnapshot2) + val snapshot2 = new DeltaV2Snapshot(kernelSnapshot2, kernelContext) assert(snapshot2.version === 2L) assert(snapshot2.version === kernelSnapshot2.getVersion) } @@ -81,9 +183,10 @@ class DeltaV2SnapshotSuite extends DeltaSQLCommandTest { storage = CatalogStorageFormat.empty.copy(locationUri = Some(dir.toURI)), schema = spark.range(0).schema) - assert(new DeltaV2Snapshot(kernelSnapshot).catalogTable.isEmpty) - assert(new DeltaV2Snapshot(kernelSnapshot, Some(catalogTable)).catalogTable.contains( - catalogTable)) + assert(new DeltaV2Snapshot(kernelSnapshot, kernelContext).catalogTable.isEmpty) + val snapshot = new DeltaV2Snapshot(kernelSnapshot, kernelContext, Some(catalogTable)) + assert(snapshot.catalogTable.contains(catalogTable)) + assert(snapshot.kernelContext eq kernelContext) } } @@ -93,7 +196,7 @@ class DeltaV2SnapshotSuite extends DeltaSQLCommandTest { */ private def assertMatchesV1(path: String): Unit = { val v1 = DeltaLog.forTable(spark, path).update() - val v2 = new DeltaV2Snapshot(loadKernelSnapshot(path)) + val v2 = new DeltaV2Snapshot(loadKernelSnapshot(path), kernelContext) assert(v2.version === v1.version) assert(v2.metadata.id === v1.metadata.id) @@ -180,7 +283,7 @@ class DeltaV2SnapshotSuite extends DeltaSQLCommandTest { val path = dir.getCanonicalPath spark.range(1).write.format("delta").save(path) - val snapshot = new DeltaV2Snapshot(loadKernelSnapshot(path)) + val snapshot = new DeltaV2Snapshot(loadKernelSnapshot(path), kernelContext) // These still fail loudly so an unmigrated caller cannot silently read empty V1 state. intercept[UnsupportedOperationException](snapshot.stateDF) @@ -207,7 +310,7 @@ class DeltaV2SnapshotSuite extends DeltaSQLCommandTest { DomainMetadata("test.userDomain", """{"key":"value"}""", removed = false) :: Nil, DeltaOperations.ManualUpdate) - val snapshot = new DeltaV2Snapshot(loadKernelSnapshot(path)) + val snapshot = new DeltaV2Snapshot(loadKernelSnapshot(path), kernelContext) val userDomain = snapshot.domainMetadata.find(_.domain == "test.userDomain") .getOrElse(fail("expected the test.userDomain domain")) @@ -225,7 +328,7 @@ class DeltaV2SnapshotSuite extends DeltaSQLCommandTest { spark.range(0, 5).write.format("delta").mode("append").save(path) val kernelSnapshot = loadKernelSnapshot(path) assert(kernelSnapshot.getCurrentCrcInfo.get().getFileSizeHistogram.isPresent) - val snapshot = new DeltaV2Snapshot(kernelSnapshot) + val snapshot = new DeltaV2Snapshot(kernelSnapshot, kernelContext) assert(snapshot.fileSizeHistogram.nonEmpty) assert( @@ -246,7 +349,7 @@ class DeltaV2SnapshotSuite extends DeltaSQLCommandTest { withSQLConf(DeltaSQLConf.DELTA_WRITE_CHECKSUM_ENABLED.key -> "false") { spark.range(0, 1000).write.format("delta").save(path) } - val snapshot = new DeltaV2Snapshot(loadKernelSnapshot(path)) + val snapshot = new DeltaV2Snapshot(loadKernelSnapshot(path), kernelContext) assert(snapshot.fileSizeHistogram.isEmpty) } } @@ -259,7 +362,7 @@ class DeltaV2SnapshotSuite extends DeltaSQLCommandTest { val kernelSnapshot = loadKernelSnapshot(path) assert(kernelSnapshot.getCurrentCrcInfo.isPresent) - val snapshot = new DeltaV2Snapshot(kernelSnapshot) + val snapshot = new DeltaV2Snapshot(kernelSnapshot, kernelContext) assert(snapshot.sizeInBytesIfKnown.nonEmpty) assert( snapshot.sizeInBytes === snapshot.allFiles.collect().map(_.size).sum @@ -274,7 +377,7 @@ class DeltaV2SnapshotSuite extends DeltaSQLCommandTest { spark.range(0, 1000).write.format("delta").save(path) } - val snapshot = new DeltaV2Snapshot(loadKernelSnapshot(path)) + val snapshot = new DeltaV2Snapshot(loadKernelSnapshot(path), kernelContext) assert(snapshot.sizeInBytesIfKnown.isEmpty) assert(snapshot.sizeInBytes === -1L) } @@ -286,7 +389,8 @@ class DeltaV2SnapshotSuite extends DeltaSQLCommandTest { spark.range(5).write.format("delta").save(path) withSQLConf(DeltaSQLConf.DELTA_FILE_SIZE_HISTOGRAM_ENABLED.key -> "false") { - assert(new DeltaV2Snapshot(loadKernelSnapshot(path)).fileSizeHistogram.isEmpty) + val snapshot = new DeltaV2Snapshot(loadKernelSnapshot(path), kernelContext) + assert(snapshot.fileSizeHistogram.isEmpty) } } } @@ -296,7 +400,7 @@ class DeltaV2SnapshotSuite extends DeltaSQLCommandTest { val path = dir.getCanonicalPath spark.range(1).write.format("delta").save(path) - val snapshot = new DeltaV2Snapshot(loadKernelSnapshot(path)) + val snapshot = new DeltaV2Snapshot(loadKernelSnapshot(path), kernelContext) assert(snapshot.domainMetadata.isEmpty) } } @@ -305,7 +409,7 @@ class DeltaV2SnapshotSuite extends DeltaSQLCommandTest { withTempDir { dir => val path = dir.getCanonicalPath spark.range(1).write.format("delta").save(path) - val snapshot = new DeltaV2Snapshot(loadKernelSnapshot(path)) + val snapshot = new DeltaV2Snapshot(loadKernelSnapshot(path), kernelContext) val activeSession = spark.newSession() activeSession.withActive { @@ -320,7 +424,7 @@ class DeltaV2SnapshotSuite extends DeltaSQLCommandTest { val path = dir.getCanonicalPath spark.range(1).write.format("delta").save(path) val kernelSnapshot = loadKernelSnapshot(path) - val snapshot = new DeltaV2Snapshot(kernelSnapshot) + val snapshot = new DeltaV2Snapshot(kernelSnapshot, kernelContext) assert(DeltaV2Snapshot.getKernelSnapshot(snapshot) eq kernelSnapshot) } @@ -348,3 +452,35 @@ class DeltaV2SnapshotSuite extends DeltaSQLCommandTest { } } + +private[interop] object SnapshotContextRecordingFileSystem { + val SessionMarkerKey = "fs.snapshot-context-test.session" + val TableMarkerKey = "fs.snapshot-context-test.table" + private val accesses = new CopyOnWriteArrayList[(String, String, String)]() + + def clear(): Unit = accesses.clear() + def observations: Seq[(String, String, String)] = accesses.asScala.toVector + def record(operation: String, sessionMarker: String, tableMarker: String): Unit = + accesses.add((operation, sessionMarker, tableMarker)) +} + +private[interop] class SnapshotContextRecordingFileSystem extends RawLocalFileSystem { + private var sessionMarker: String = _ + private var tableMarker: String = _ + + override def initialize(uri: URI, conf: Configuration): Unit = { + sessionMarker = conf.get(SnapshotContextRecordingFileSystem.SessionMarkerKey) + tableMarker = conf.get(SnapshotContextRecordingFileSystem.TableMarkerKey) + super.initialize(uri, conf) + } + + override def open(path: Path, bufferSize: Int): FSDataInputStream = { + SnapshotContextRecordingFileSystem.record("open", sessionMarker, tableMarker) + super.open(path, bufferSize) + } + + override def listStatus(path: Path): Array[FileStatus] = { + SnapshotContextRecordingFileSystem.record("listStatus", sessionMarker, tableMarker) + super.listStatus(path) + } +} diff --git a/spark-unified/src/test/scala/org/apache/spark/sql/delta/v2/interop/KernelSnapshotUtilsSuite.scala b/spark-unified/src/test/scala/org/apache/spark/sql/delta/v2/interop/KernelSnapshotUtilsSuite.scala index 634f6c7fe66..858e820954b 100644 --- a/spark-unified/src/test/scala/org/apache/spark/sql/delta/v2/interop/KernelSnapshotUtilsSuite.scala +++ b/spark-unified/src/test/scala/org/apache/spark/sql/delta/v2/interop/KernelSnapshotUtilsSuite.scala @@ -23,18 +23,18 @@ import java.nio.file.Files import org.apache.spark.sql.delta.{DeltaConfigs, DeltaLog} import org.apache.spark.sql.delta.actions.AddFile import org.apache.spark.sql.delta.sources.DeltaSQLConf +import org.apache.spark.sql.delta.storage.LogStore import org.apache.spark.sql.delta.test.DeltaSQLCommandTest -import io.delta.spark.internal.v2.kernel.KernelEngineFactory +import io.delta.spark.internal.v2.kernel.KernelContext import io.delta.kernel.TableManager import io.delta.kernel.engine.Engine import io.delta.kernel.internal.SnapshotImpl class KernelSnapshotUtilsSuite extends DeltaSQLCommandTest { - // scalastyle:off deltahadoopconfiguration - private def engine: Engine = - KernelEngineFactory.createDefaultEngine(spark.sessionState.newHadoopConf()) - // scalastyle:on deltahadoopconfiguration + private lazy val kernelContext = new KernelContext(Map.empty, LogStore(spark)) + + private def engine: Engine = kernelContext.getDefaultEngine() private def kernelSnapshotFor(path: String, engine: Engine): SnapshotImpl = TableManager.loadSnapshot(path).build(engine).asInstanceOf[SnapshotImpl] @@ -171,7 +171,7 @@ class KernelSnapshotUtilsSuite extends DeltaSQLCommandTest { KernelSnapshotUtils.buildAllFiles(kernelSnapshot, spark, kernelEngine).collect() assert(kernelFiles.find(_.path == taggedPath).map(_.tags).contains(expectedTags)) - val snapshot = new DeltaV2Snapshot(kernelSnapshot) + val snapshot = new DeltaV2Snapshot(kernelSnapshot, kernelContext) val selectedFiles = snapshot.filesForScan(Nil).files assert(selectedFiles.find(_.path == taggedPath).map(_.tags).contains(expectedTags)) } diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/catalog/DeltaV2Table.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/catalog/DeltaV2Table.java index 2f216973ee1..dc02ee4d6f0 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/catalog/DeltaV2Table.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/catalog/DeltaV2Table.java @@ -26,6 +26,7 @@ import io.delta.spark.internal.v2.exception.NoRecreatableHistoryException; import io.delta.spark.internal.v2.exception.TableNotFoundException; import io.delta.spark.internal.v2.exception.TimestampOutOfRangeException; +import io.delta.spark.internal.v2.kernel.KernelContext; import io.delta.spark.internal.v2.read.DeltaV2ScanUtils; import io.delta.spark.internal.v2.read.MetadataEvolutionHandler; import io.delta.spark.internal.v2.read.cdc.CDCSchemaContext; @@ -131,7 +132,7 @@ private static Set buildReadOnlyCapabilities() { private final Snapshot initialSnapshot; private final Configuration hadoopConf; - private final Engine kernelEngine; + private final KernelContext kernelContext; private final SchemaProvider schemaProvider; private final Optional catalogTable; @@ -251,7 +252,7 @@ private DeltaV2Table( DeltaV2TableManager tableManager = DeltaV2TableManagerCache$.MODULE$.forTable( activeSession, tablePath, options, catalogTableOpt); - this.kernelEngine = tableManager.kernelContext().getDefaultEngine(); + this.kernelContext = tableManager.kernelContext(); this.snapshotManager = tableManager.snapshotManager(catalogTableOpt); try { if (timeTravelVersion.isPresent()) { @@ -279,7 +280,7 @@ private DeltaV2Table( Optional persistedMetadata = MetadataEvolutionHandler.getPersistedMetadataForMicroBatchStream( - SparkSession.active(), initialSnapshot, options, snapshotManager, kernelEngine); + SparkSession.active(), initialSnapshot, options, snapshotManager, kernelEngine()); StructType rawSchema; List partitionColumnNames; @@ -371,7 +372,7 @@ protected AbstractMetadata metadata() { /** Inputs exposed to the Spark-version shim for metadata-only DELETE. */ protected Engine kernelEngine() { - return kernelEngine; + return kernelContext.getDefaultEngine(); } protected Snapshot initialSnapshot() { @@ -548,7 +549,7 @@ public ScanBuilder newScanBuilder(CaseInsensitiveStringMap scanOptions) { return DeltaV2ScanUtils.newScanBuilder( name(), initialSnapshot, - kernelEngine, + kernelEngine(), catalogTable, snapshotManager, schemaProvider.getDataSchema(), @@ -573,7 +574,7 @@ private Snapshot loadSnapshotAtCheckedVersion(DeltaV2SnapshotManager manager, lo public WriteBuilder newWriteBuilder(LogicalWriteInfo info) { requireNonNull(info, "write info is null"); return new DeltaV2WriteBuilder( - kernelEngine, + kernelEngine(), tablePath, hadoopConf, initialSnapshot, @@ -592,7 +593,8 @@ public WriteBuilder newWriteBuilder(LogicalWriteInfo info) { @Override public RowLevelOperationBuilder newRowLevelOperationBuilder(RowLevelOperationInfo info) { requireNonNull(info, "row-level operation info is null"); - return new DeltaRowLevelOperationBuilder(this, kernelEngine, hadoopConf, initialSnapshot, info); + return new DeltaRowLevelOperationBuilder( + this, kernelEngine(), hadoopConf, initialSnapshot, info); } @Override diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/snapshot/PathBasedSnapshotManager.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/snapshot/PathBasedSnapshotManager.java index 249f47f031d..56d3913f60f 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/snapshot/PathBasedSnapshotManager.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/snapshot/PathBasedSnapshotManager.java @@ -24,10 +24,9 @@ import io.delta.kernel.internal.DeltaHistoryManager; import io.delta.kernel.internal.SnapshotImpl; import io.delta.spark.internal.v2.exception.VersionNotFoundException; -import io.delta.spark.internal.v2.kernel.KernelEngineFactory; +import io.delta.spark.internal.v2.kernel.KernelContext; import java.util.ArrayList; import java.util.Optional; -import org.apache.hadoop.conf.Configuration; import org.apache.spark.annotation.Experimental; import org.apache.spark.sql.delta.Snapshot; import org.apache.spark.sql.delta.v2.interop.DeltaV2SnapshotManager; @@ -38,17 +37,11 @@ public class PathBasedSnapshotManager implements DeltaV2SnapshotManager { private final String tablePath; - private final Engine kernelEngine; + private final KernelContext kernelContext; - public PathBasedSnapshotManager(String tablePath, Configuration hadoopConf) { - this( - tablePath, - KernelEngineFactory.createDefaultEngine(requireNonNull(hadoopConf, "hadoopConf is null"))); - } - - public PathBasedSnapshotManager(String tablePath, Engine kernelEngine) { + public PathBasedSnapshotManager(String tablePath, KernelContext kernelContext) { this.tablePath = requireNonNull(tablePath, "tablePath is null"); - this.kernelEngine = requireNonNull(kernelEngine, "kernelEngine is null"); + this.kernelContext = requireNonNull(kernelContext, "kernelContext is null"); } /** @@ -59,7 +52,7 @@ public PathBasedSnapshotManager(String tablePath, Engine kernelEngine) { @Override public Snapshot loadLatestSnapshot() { return DeltaV2SnapshotManager$.MODULE$.wrapKernelSnapshot( - loadLatestKernelSnapshot(), tablePath); + loadLatestKernelSnapshot(), kernelContext, tablePath); } /** @@ -71,16 +64,19 @@ public Snapshot loadLatestSnapshot() { @Override public Snapshot loadSnapshotAt(long version) { return DeltaV2SnapshotManager$.MODULE$.wrapKernelSnapshot( - loadKernelSnapshotAt(version), tablePath); + loadKernelSnapshotAt(version), kernelContext, tablePath); } private SnapshotImpl loadLatestKernelSnapshot() { - return (SnapshotImpl) TableManager.loadSnapshot(tablePath).build(kernelEngine); + return (SnapshotImpl) + TableManager.loadSnapshot(tablePath).build(kernelContext.getDefaultEngine()); } private SnapshotImpl loadKernelSnapshotAt(long version) { return (SnapshotImpl) - TableManager.loadSnapshot(tablePath).atVersion(version).build(kernelEngine); + TableManager.loadSnapshot(tablePath) + .atVersion(version) + .build(kernelContext.getDefaultEngine()); } /** @@ -106,7 +102,7 @@ public DeltaHistoryManager.Commit getActiveCommitAtTime( boolean canReturnEarliestCommit) { SnapshotImpl snapshot = loadLatestKernelSnapshot(); return DeltaHistoryManager.getActiveCommitAtTimestamp( - kernelEngine, + kernelContext.getDefaultEngine(), snapshot, snapshot.getLogPath(), timestampMillis, @@ -133,11 +129,11 @@ public void checkVersionExists(long version, boolean mustBeRecreatable, boolean long earliest = mustBeRecreatable ? DeltaHistoryManager.getEarliestRecreatableCommit( - kernelEngine, + kernelContext.getDefaultEngine(), snapshot.getLogPath(), Optional.empty() /*earliestRatifiedCommitVersion*/) : DeltaHistoryManager.getEarliestDeltaFile( - kernelEngine, + kernelContext.getDefaultEngine(), snapshot.getLogPath(), Optional.empty() /*earliestRatifiedCommitVersion*/); diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/snapshot/SnapshotManagerFactory.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/snapshot/SnapshotManagerFactory.java index acff35ec2bf..4a3fbca5cb8 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/snapshot/SnapshotManagerFactory.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/snapshot/SnapshotManagerFactory.java @@ -16,8 +16,8 @@ package io.delta.spark.internal.v2.snapshot; import io.delta.kernel.Meta; -import io.delta.kernel.engine.Engine; import io.delta.kernel.unitycatalog.UCCatalogManagedClient; +import io.delta.spark.internal.v2.kernel.KernelContext; import io.delta.spark.internal.v2.snapshot.unitycatalog.UCManagedTableSnapshotManager; import io.delta.spark.internal.v2.snapshot.unitycatalog.UCTableInfo; import io.delta.spark.internal.v2.snapshot.unitycatalog.UCUtils; @@ -51,33 +51,33 @@ private SnapshotManagerFactory() {} * Creates a snapshot manager for the given table. * * @param tablePath the filesystem path to the Delta table - * @param kernelEngine the pre-configured Kernel {@link Engine} to use for table operations + * @param kernelContext the configured context to use for table operations * @param catalogTable optional Spark catalog table metadata * @return a {@link DeltaV2SnapshotManager} appropriate for the table type */ public static DeltaV2SnapshotManager create( - String tablePath, Engine kernelEngine, Optional catalogTable) { + String tablePath, KernelContext kernelContext, Optional catalogTable) { if (catalogTable.isPresent()) { Optional ucTableInfo = UCUtils.extractTableInfo(catalogTable.get(), SparkSession.active()); if (ucTableInfo.isPresent()) { - return createUCManagedSnapshotManager(ucTableInfo.get(), kernelEngine); + return createUCManagedSnapshotManager(ucTableInfo.get(), kernelContext); } // Catalog table without UC metadata falls back to path-based handling. } // Default: path-based snapshot manager for non-UC tables - return new PathBasedSnapshotManager(tablePath, kernelEngine); + return new PathBasedSnapshotManager(tablePath, kernelContext); } private static UCManagedTableSnapshotManager createUCManagedSnapshotManager( - UCTableInfo tableInfo, Engine kernelEngine) { + UCTableInfo tableInfo, KernelContext kernelContext) { Map ucConfig = new HashMap<>(tableInfo.toUcConfig()); ucConfig.put("appVersions.Kernel", Meta.KERNEL_VERSION); ucConfig.put("appVersions.Delta V2 connector", "true"); UCClient ucClient = UCTokenBasedRestClientFactory$.MODULE$.createUCClient(ucConfig); UCCatalogManagedClient ucCatalogClient = new UCCatalogManagedClient(ucClient); - return new UCManagedTableSnapshotManager(ucCatalogClient, tableInfo, kernelEngine); + return new UCManagedTableSnapshotManager(ucCatalogClient, tableInfo, kernelContext); } } diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/snapshot/unitycatalog/UCManagedTableSnapshotManager.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/snapshot/unitycatalog/UCManagedTableSnapshotManager.java index f1393a9521e..65499c9051b 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/snapshot/unitycatalog/UCManagedTableSnapshotManager.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/snapshot/unitycatalog/UCManagedTableSnapshotManager.java @@ -25,6 +25,7 @@ import io.delta.kernel.unitycatalog.UCCatalogManagedClient; import io.delta.kernel.unitycatalog.UCTableIdentifier; import io.delta.spark.internal.v2.exception.VersionNotFoundException; +import io.delta.spark.internal.v2.kernel.KernelContext; import java.util.List; import java.util.Optional; import org.apache.spark.sql.delta.Snapshot; @@ -43,24 +44,26 @@ public class UCManagedTableSnapshotManager implements DeltaV2SnapshotManager { private final String tableId; private final String tablePath; private final UCTableIdentifier tableIdentifier; - private final Engine engine; + private final KernelContext kernelContext; /** * Creates a new UCManagedTableSnapshotManager. * * @param ucCatalogManagedClient the UC client for catalog-managed operations * @param tableInfo the UC table information (tableId, tablePath, etc.) - * @param engine the Kernel engine for table operations + * @param kernelContext the configured context for table operations */ public UCManagedTableSnapshotManager( - UCCatalogManagedClient ucCatalogManagedClient, UCTableInfo tableInfo, Engine engine) { + UCCatalogManagedClient ucCatalogManagedClient, + UCTableInfo tableInfo, + KernelContext kernelContext) { this.ucCatalogManagedClient = requireNonNull(ucCatalogManagedClient, "ucCatalogManagedClient is null"); requireNonNull(tableInfo, "tableInfo is null"); this.tableId = tableInfo.getTableId(); this.tablePath = tableInfo.getTablePath(); this.tableIdentifier = tableInfo.getTableIdentifier(); - this.engine = requireNonNull(engine, "engine is null"); + this.kernelContext = requireNonNull(kernelContext, "kernelContext is null"); } /** @@ -71,19 +74,19 @@ public UCManagedTableSnapshotManager( @Override public Snapshot loadLatestSnapshot() { return DeltaV2SnapshotManager$.MODULE$.wrapKernelSnapshot( - loadKernelSnapshot(Optional.empty()), tablePath); + loadKernelSnapshot(Optional.empty()), kernelContext, tablePath); } @Override public Snapshot loadSnapshotAt(long version) { return DeltaV2SnapshotManager$.MODULE$.wrapKernelSnapshot( - loadKernelSnapshot(Optional.of(version)), tablePath); + loadKernelSnapshot(Optional.of(version)), kernelContext, tablePath); } private SnapshotImpl loadKernelSnapshot(Optional versionOpt) { return (SnapshotImpl) ucCatalogManagedClient.loadSnapshot( - engine, + kernelContext.getDefaultEngine(), tableId, tablePath, tableIdentifier, @@ -115,7 +118,7 @@ public DeltaHistoryManager.Commit getActiveCommitAtTime( SnapshotImpl snapshot = loadKernelSnapshot(Optional.empty()); List catalogCommits = snapshot.getLogSegment().getAllCatalogCommits(); return DeltaHistoryManager.getActiveCommitAtTimestamp( - engine, + kernelContext.getDefaultEngine(), snapshot, snapshot.getLogPath(), timestampMillis, @@ -160,9 +163,13 @@ public void checkVersionExists(long version, boolean mustBeRecreatable, boolean long earliestVersion = mustBeRecreatable ? DeltaHistoryManager.getEarliestRecreatableCommit( - engine, snapshot.getLogPath(), earliestCatalogCommitVersion) + kernelContext.getDefaultEngine(), + snapshot.getLogPath(), + earliestCatalogCommitVersion) : DeltaHistoryManager.getEarliestDeltaFile( - engine, snapshot.getLogPath(), earliestCatalogCommitVersion); + kernelContext.getDefaultEngine(), + snapshot.getLogPath(), + earliestCatalogCommitVersion); if (version < earliestVersion) { throw new VersionNotFoundException(version, earliestVersion, latestSnapshotVersion); diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/write/DeltaV2Write.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/write/DeltaV2Write.java index b19af11531a..4a3ac255d46 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/write/DeltaV2Write.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/write/DeltaV2Write.java @@ -140,9 +140,10 @@ static String getEngineInfo() { @Override public BatchWrite toBatch() { + DeltaV2Snapshot snapshot = (DeltaV2Snapshot) initialSnapshot; DeltaV2OptimisticTransaction optimisticTransaction = new DeltaV2OptimisticTransaction( - ScalaUtils.toScalaOption(catalogTable), (DeltaV2Snapshot) initialSnapshot, engine); + ScalaUtils.toScalaOption(catalogTable), snapshot, snapshot.kernelContext()); DeltaV2WriteContext context = DeltaV2WriteContext.create( engine, diff --git a/spark/v2/src/main/scala/io/delta/spark/internal/v2/kernel/KernelContext.scala b/spark/v2/src/main/scala/io/delta/spark/internal/v2/kernel/KernelContext.scala index 1e0c5d3778d..f6a14a8e0b4 100644 --- a/spark/v2/src/main/scala/io/delta/spark/internal/v2/kernel/KernelContext.scala +++ b/spark/v2/src/main/scala/io/delta/spark/internal/v2/kernel/KernelContext.scala @@ -25,14 +25,13 @@ import org.apache.spark.sql.SparkSession /** * Table-scoped owner of Kernel resources and deferred Hadoop configuration materialization. * - * It retains session-invariant filesystem options and the table manager's LogStore. When - * filesystem I/O begins, it binds those options to the active Spark session so session-derived - * settings and credentials are not retained by reusable connector state. Its LogStore and - * lazily-created Engine remain stable across Spark sessions. + * It retains session-invariant filesystem options and the table manager's LogStore. Hadoop + * configuration materialization combines those options with the active Spark session's settings. + * Callers look up an engine when an operation needs it instead of retaining their own engine. */ -private[v2] final class KernelContext( - val sessionInvariantFsOptions: Map[String, String], - val logStore: LogStore) { +final class KernelContext( + private[v2] val sessionInvariantFsOptions: Map[String, String], + private[v2] val logStore: LogStore) { require(sessionInvariantFsOptions != null, "sessionInvariantFsOptions must not be null") require(logStore != null, "logStore must not be null") @@ -44,9 +43,11 @@ private[v2] final class KernelContext( KernelDefaultEngine.create(hadoopConf) } - private lazy val kernelDefaultEngine = createDefaultEngine() - private[v2] def getDefaultEngine(): KernelEngine = kernelDefaultEngine + // The snapshot facade lives in a separate internal package in the standalone connector. + def getDefaultEngine(): KernelEngine = { + createDefaultEngine() + } } private[v2] object KernelContext { diff --git a/spark/v2/src/main/scala/io/delta/spark/internal/v2/tablemanager/CachedSnapshotManager.scala b/spark/v2/src/main/scala/io/delta/spark/internal/v2/tablemanager/CachedSnapshotManager.scala index 458bfb1caf8..1ecf704c07a 100644 --- a/spark/v2/src/main/scala/io/delta/spark/internal/v2/tablemanager/CachedSnapshotManager.scala +++ b/spark/v2/src/main/scala/io/delta/spark/internal/v2/tablemanager/CachedSnapshotManager.scala @@ -265,7 +265,7 @@ private[tablemanager] class CachedSnapshotManager( f: DeltaV2SnapshotManager => T): T = { f(SnapshotManagerFactory.create( tablePath.toString, - kernelContext.getDefaultEngine(), + kernelContext, catalogTableOpt.toJava)) } diff --git a/spark/v2/src/main/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2OptimisticTransaction.scala b/spark/v2/src/main/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2OptimisticTransaction.scala index 09a24efa19e..4d93c5a59c2 100644 --- a/spark/v2/src/main/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2OptimisticTransaction.scala +++ b/spark/v2/src/main/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2OptimisticTransaction.scala @@ -33,6 +33,7 @@ import org.apache.spark.sql.delta.amt.AMTCheckpointProvider import org.apache.spark.sql.delta.hooks.{CheckpointHook, ChecksumHook, HudiConverterHook, IcebergConverterHook, PostCommitHook} import org.apache.spark.sql.delta.util.{DeltaFileOperations, FileNames} import org.apache.spark.sql.delta.v2.kernel.KernelActionUtils +import io.delta.spark.internal.v2.kernel.KernelContext import io.delta.spark.internal.v2.snapshot.SnapshotManagerFactory import io.delta.storage.commit.Commit import org.apache.hadoop.conf.Configuration @@ -63,19 +64,21 @@ import org.apache.spark.util.{Clock, SystemClock} */ private[v2] class DeltaV2OptimisticTransaction( catalogTable: Option[CatalogTable], - // `deltaV2Snapshot` and `kernelEngine` are intentionally private, which is internal + // `deltaV2Snapshot` and `kernelContext` are intentionally private, which is internal // implementation details we do not expose in the public API. private val deltaV2Snapshot: DeltaV2Snapshot, - private val kernelEngine: KernelEngine) + private val kernelContext: KernelContext) extends OptimisticTransaction( null.asInstanceOf[DeltaLog], catalogTable, deltaV2Snapshot) { + private def kernelEngine: KernelEngine = kernelContext.getDefaultEngine() + private lazy val deltaV2SnapshotManager: DeltaV2SnapshotManager = SnapshotManagerFactory.create( dataPath.toString, - kernelEngine, + kernelContext, catalogTable.toJava ) @@ -189,7 +192,7 @@ private[v2] class DeltaV2OptimisticTransaction( kernelPostCommitSnapshot.getVersion >= committedVersion, s"Kernel reload returned version ${kernelPostCommitSnapshot.getVersion}, older than the " + s"just-committed version $committedVersion") - new DeltaV2Snapshot(kernelPostCommitSnapshot) + new DeltaV2Snapshot(kernelPostCommitSnapshot, kernelContext) } /** diff --git a/spark/v2/src/main/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2Snapshot.scala b/spark/v2/src/main/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2Snapshot.scala index 054a0c8005b..2c85b3a7e7d 100644 --- a/spark/v2/src/main/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2Snapshot.scala +++ b/spark/v2/src/main/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2Snapshot.scala @@ -21,7 +21,7 @@ import scala.jdk.OptionConverters._ import io.delta.kernel.engine.Engine import io.delta.kernel.internal.{SnapshotImpl => KernelSnapshot} -import io.delta.spark.internal.v2.kernel.KernelEngineFactory +import io.delta.spark.internal.v2.kernel.KernelContext import org.apache.spark.sql.delta.{CheckpointProvider, DeltaColumnMappingMode, DeltaLogFileIndex, Snapshot, VersionChecksum} import org.apache.spark.sql.delta.actions.{AddFile, DomainMetadata, Metadata, Protocol, RemoveFile, SingleAction} @@ -50,7 +50,8 @@ class DeltaV2Snapshot( // `Snapshot` interface: the decoded data path is only exposed on SnapshotImpl in the // OSS-published Kernel. The public io.delta.kernel.Snapshot has no `getDataPath`, and its // `getPath` is URL-encoded, which breaks Hadoop `Path` for table roots containing spaces. - private val kernelSnapshot: KernelSnapshot) + private val kernelSnapshot: KernelSnapshot, + private[v2] val kernelContext: KernelContext) extends Snapshot( // toString keeps this compiling across Kernel versions: getDataPath returns String in the // currently pinned Kernel and io.delta.kernel.internal.fs.Path in newer Kernels. @@ -62,20 +63,20 @@ class DeltaV2Snapshot( logSegment = null, deltaLog = null, checksumOpt = None) { + require(kernelContext != null, "kernelContext must not be null") + private[this] var resolvedCatalogTableOpt: Option[CatalogTable] = None def this( kernelSnapshot: KernelSnapshot, + kernelContext: KernelContext, catalogTableOpt: Option[CatalogTable]) = { - this(kernelSnapshot) + this(kernelSnapshot, kernelContext) resolvedCatalogTableOpt = catalogTableOpt } - // scalastyle:off deltahadoopconfiguration - private def kernelEngine: Engine = - KernelEngineFactory.createDefaultEngine(spark.sessionState.newHadoopConf()) - // scalastyle:on deltahadoopconfiguration + private def kernelEngine: Engine = kernelContext.getDefaultEngine() // --- logSegment/deltaLog = null guardrail: construction-path overrides ---------------------- // These are the members dereferenced while the super `Snapshot` constructor runs (init() and the diff --git a/spark/v2/src/main/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2SnapshotManager.scala b/spark/v2/src/main/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2SnapshotManager.scala index 6f277205716..e151affadb0 100644 --- a/spark/v2/src/main/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2SnapshotManager.scala +++ b/spark/v2/src/main/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2SnapshotManager.scala @@ -23,6 +23,7 @@ import io.delta.kernel.engine.Engine import io.delta.kernel.internal.DeltaHistoryManager import io.delta.kernel.internal.SnapshotImpl import io.delta.spark.internal.v2.exception.VersionNotFoundException +import io.delta.spark.internal.v2.kernel.KernelContext import org.apache.spark.sql.delta.Snapshot @@ -155,12 +156,17 @@ object DeltaV2SnapshotManager { * Wraps a Kernel snapshot in the V1 [[Snapshot]] facade returned by manager load APIs. * * @param kernelSnapshot the Kernel snapshot to wrap + * @param kernelContext the context used to load and access the snapshot * @param tablePath table path used in construction-error messages * @return the V1 snapshot facade */ - def wrapKernelSnapshot(kernelSnapshot: SnapshotImpl, tablePath: String): Snapshot = { + def wrapKernelSnapshot( + kernelSnapshot: SnapshotImpl, + kernelContext: KernelContext, + tablePath: String): Snapshot = { Objects.requireNonNull(kernelSnapshot, "kernelSnapshot is null") + Objects.requireNonNull(kernelContext, "kernelContext is null") Objects.requireNonNull(tablePath, "tablePath is null") - new DeltaV2Snapshot(kernelSnapshot) + new DeltaV2Snapshot(kernelSnapshot, kernelContext) } } diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/DeltaV2TestBase.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/DeltaV2TestBase.java index e25ed343d6c..aaee4f3326e 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/DeltaV2TestBase.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/DeltaV2TestBase.java @@ -15,16 +15,21 @@ */ package io.delta.spark.internal.v2; -import io.delta.kernel.defaults.engine.DefaultEngine; import io.delta.kernel.engine.Engine; +import io.delta.spark.internal.v2.kernel.KernelContext; import io.delta.spark.internal.v2.read.DeltaV2ScanBuilder; +import io.delta.spark.internal.v2.utils.ScalaUtils; import java.nio.file.Paths; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import org.apache.hadoop.conf.Configuration; import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.catalyst.expressions.AttributeReference; import org.apache.spark.sql.catalyst.expressions.Expression; import org.apache.spark.sql.catalyst.expressions.Literal; +import org.apache.spark.sql.delta.storage.LogStore; import org.apache.spark.sql.sources.And; import org.apache.spark.sql.sources.EqualTo; import org.apache.spark.sql.sources.Filter; @@ -42,6 +47,7 @@ public abstract class DeltaV2TestBase { protected static SparkSession spark; protected static Engine defaultEngine; + protected static KernelContext defaultKernelContext; @BeforeAll public static void setUpSparkAndEngine() { @@ -56,7 +62,18 @@ public static void setUpSparkAndEngine() { .config("spark.sql.catalog.dsv2", "io.delta.spark.internal.v2.catalog.TestCatalog") .config("spark.sql.catalog.dsv2.base_path", System.getProperty("java.io.tmpdir")) .getOrCreate(); - defaultEngine = DefaultEngine.create(spark.sessionState().newHadoopConf()); + defaultKernelContext = createKernelContext(); + defaultEngine = defaultKernelContext.getDefaultEngine(); + } + + protected static KernelContext createKernelContext() { + return createKernelContext(spark.sessionState().newHadoopConf()); + } + + protected static KernelContext createKernelContext(Configuration hadoopConf) { + Map options = new HashMap<>(); + hadoopConf.forEach(entry -> options.put(entry.getKey(), entry.getValue())); + return new KernelContext(ScalaUtils.toScalaMap(options), LogStore.apply(spark)); } @AfterAll diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/catalog/DeltaV2TableTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/catalog/DeltaV2TableTest.java index 053f95eded5..86e0531d1d4 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/catalog/DeltaV2TableTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/catalog/DeltaV2TableTest.java @@ -612,8 +612,7 @@ public void testUsesTableManagerKernelContextAndLoadsSnapshotsAcrossSessions( assertTrue(tableB.getSnapshotManager() instanceof CachedSnapshotManager); assertSame(managerA, managerB); assertSame(managerA.kernelContext(), managerB.kernelContext()); - assertSame(managerA.kernelContext().getDefaultEngine(), tableA.kernelEngine()); - assertSame(tableA.kernelEngine(), tableB.kernelEngine()); + assertSame(tableA.getSnapshotManager(), tableB.getSnapshotManager()); assertEquals("0", tableA.version()); assertEquals("0", tableB.version()); @@ -649,8 +648,7 @@ public void testNamedTableQueryLoadsSnapshotWithItsQueryContext(@TempDir File te TableIdentifier expectedIdentifier = catalogTable.identifier(); Snapshot realSnapshot = - new PathBasedSnapshotManager(path, spark.sessionState().newHadoopConf()) - .loadLatestSnapshot(); + new PathBasedSnapshotManager(path, createKernelContext()).loadLatestSnapshot(); AssertionError legacyApiError = new AssertionError("DeltaV2Table query used the legacy snapshot API"); DeltaV2TableManagerCache$.MODULE$.clearCache(); @@ -992,7 +990,7 @@ private void verifyPersistedEntryDrivesSparkTableSchema( // Capture v0 metadata BEFORE evolving the table. PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(tablePath, spark.sessionState().newHadoopConf()); + new PathBasedSnapshotManager(tablePath, createKernelContext()); SnapshotImpl snapshotV0 = DeltaV2Snapshot$.MODULE$.getKernelSnapshot( snapshotManager.loadSnapshotAt( diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/DeltaV2MicroBatchStreamCDCTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/DeltaV2MicroBatchStreamCDCTest.java index 7c32d61b6d2..0a0b5eb7b01 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/DeltaV2MicroBatchStreamCDCTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/DeltaV2MicroBatchStreamCDCTest.java @@ -71,7 +71,8 @@ public void testValidateCDFEnabled_throwsWhenNotEnabled(@TempDir File tempDir) t sql("INSERT INTO %s VALUES (1, 'User1')", tableName); Configuration hadoopConf = new Configuration(); - PathBasedSnapshotManager snapshotManager = new PathBasedSnapshotManager(tablePath, hadoopConf); + PathBasedSnapshotManager snapshotManager = + new PathBasedSnapshotManager(tablePath, createKernelContext(hadoopConf)); DeltaV2MicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); @@ -108,7 +109,8 @@ public void testValidateCDFEnabled_passesWhenEnabledAtStartVersionEqualToInit( sql("INSERT INTO %s VALUES (1, 'User1')", tableName); Configuration hadoopConf = new Configuration(); - PathBasedSnapshotManager snapshotManager = new PathBasedSnapshotManager(tablePath, hadoopConf); + PathBasedSnapshotManager snapshotManager = + new PathBasedSnapshotManager(tablePath, createKernelContext(hadoopConf)); DeltaV2MicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); @@ -128,7 +130,8 @@ public void testValidateCDFEnabled_throwsWhenCDFOffAtStartVersionButOnAtInit( sql("INSERT INTO %s VALUES (2, 'User2')", tableName); Configuration hadoopConf = new Configuration(); - PathBasedSnapshotManager snapshotManager = new PathBasedSnapshotManager(tablePath, hadoopConf); + PathBasedSnapshotManager snapshotManager = + new PathBasedSnapshotManager(tablePath, createKernelContext(hadoopConf)); DeltaV2MicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); @@ -150,7 +153,8 @@ public void testValidateCDFEnabled_swallowsForUnmaterializedStartVersion(@TempDi sql("INSERT INTO %s VALUES (1, 'User1')", tableName); Configuration hadoopConf = new Configuration(); - PathBasedSnapshotManager snapshotManager = new PathBasedSnapshotManager(tablePath, hadoopConf); + PathBasedSnapshotManager snapshotManager = + new PathBasedSnapshotManager(tablePath, createKernelContext(hadoopConf)); DeltaV2MicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); long latestVersion = snapshotManager.loadLatestSnapshot().version(); @@ -267,7 +271,8 @@ public void testGetFileChangesForCDC( // DSv2 Configuration hadoopConf = new Configuration(); - PathBasedSnapshotManager snapshotManager = new PathBasedSnapshotManager(tablePath, hadoopConf); + PathBasedSnapshotManager snapshotManager = + new PathBasedSnapshotManager(tablePath, createKernelContext(hadoopConf)); DeltaV2MicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); List dsv2Files = new ArrayList<>(); @@ -1026,7 +1031,8 @@ public void testProcessCommit_barrierSurvivesAdmissionLimit(@TempDir File tempDi private DeltaV2MicroBatchStream createStreamWithSeededTrackingLog( String tablePath, String schemaLogPath, long seededVersion) { Configuration hadoopConf = new Configuration(); - PathBasedSnapshotManager snapshotManager = new PathBasedSnapshotManager(tablePath, hadoopConf); + PathBasedSnapshotManager snapshotManager = + new PathBasedSnapshotManager(tablePath, createKernelContext(hadoopConf)); Engine engine = DefaultEngine.create(hadoopConf); Map javaOptions = new HashMap<>(); @@ -1123,7 +1129,8 @@ private List getDsv2FileChanges( Optional maxBytes) throws Exception { Configuration hadoopConf = new Configuration(); - PathBasedSnapshotManager snapshotManager = new PathBasedSnapshotManager(tablePath, hadoopConf); + PathBasedSnapshotManager snapshotManager = + new PathBasedSnapshotManager(tablePath, createKernelContext(hadoopConf)); DeltaV2MicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); List files = new ArrayList<>(); @@ -1203,14 +1210,16 @@ private DeltaV2MicroBatchStream createTestStreamWithDefaults( private DeltaV2MicroBatchStream createStream(String tablePath) { Configuration hadoopConf = new Configuration(); - PathBasedSnapshotManager snapshotManager = new PathBasedSnapshotManager(tablePath, hadoopConf); + PathBasedSnapshotManager snapshotManager = + new PathBasedSnapshotManager(tablePath, createKernelContext(hadoopConf)); return createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); } private CommitActions getCommitActions(String tablePath, long version) { Configuration hadoopConf = new Configuration(); Engine engine = DefaultEngine.create(hadoopConf); - PathBasedSnapshotManager snapshotManager = new PathBasedSnapshotManager(tablePath, hadoopConf); + PathBasedSnapshotManager snapshotManager = + new PathBasedSnapshotManager(tablePath, createKernelContext(hadoopConf)); CommitRange commitRange = snapshotManager.getTableChanges(engine, version, Optional.of(version)); diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/DeltaV2MicroBatchStreamTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/DeltaV2MicroBatchStreamTest.java index d48693e1c5b..ab1f2f424d4 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/DeltaV2MicroBatchStreamTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/DeltaV2MicroBatchStreamTest.java @@ -85,7 +85,8 @@ private DeltaV2MicroBatchStream createTestStream(File tempDir) { String tableName = "test_unsupported_" + System.nanoTime(); createEmptyTestTable(tablePath, tableName); Configuration hadoopConf = new Configuration(); - PathBasedSnapshotManager snapshotManager = new PathBasedSnapshotManager(tablePath, hadoopConf); + PathBasedSnapshotManager snapshotManager = + new PathBasedSnapshotManager(tablePath, createKernelContext(hadoopConf)); return createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); } @@ -116,7 +117,7 @@ public void testInitialOffset_withInitialSnapshot(@TempDir File tempDir) throws // Create stream without startingVersion (emptyDeltaOptions) Configuration hadoopConf = new Configuration(); PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(testTablePath, hadoopConf); + new PathBasedSnapshotManager(testTablePath, createKernelContext(hadoopConf)); DeltaV2MicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); @@ -155,7 +156,7 @@ public void testInitialOffset_doesNotResetFirstBatchState(@TempDir File tempDir) Configuration hadoopConf = new Configuration(); PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(testTablePath, hadoopConf); + new PathBasedSnapshotManager(testTablePath, createKernelContext(hadoopConf)); DeltaV2MicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); @@ -181,7 +182,7 @@ public void testInitialOffset_noDataRemainsFirstBatch(@TempDir File tempDir) thr DeltaOptions options = new DeltaOptions(scalaOptions, spark.sessionState().conf()); Configuration hadoopConf = new Configuration(); PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(testTablePath, hadoopConf); + new PathBasedSnapshotManager(testTablePath, createKernelContext(hadoopConf)); DeltaV2MicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, options); @@ -321,7 +322,7 @@ public void testInitialOffset_firstBatchParity( // DSv2 Configuration hadoopConf = new Configuration(); PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(testTablePath, hadoopConf); + new PathBasedSnapshotManager(testTablePath, createKernelContext(hadoopConf)); DeltaV2MicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, options); Offset initialOffset = stream.initialOffset(); @@ -417,7 +418,7 @@ public void testGetFileChanges( // Delta V2 micro-batch stream Configuration hadoopConf = new Configuration(); PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(testTablePath, hadoopConf); + new PathBasedSnapshotManager(testTablePath, createKernelContext(hadoopConf)); DeltaV2MicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); Optional endOffsetOption = ScalaUtils.toJavaOptional(scalaEndOffset); @@ -602,7 +603,7 @@ public void testGetFileChangesWithRateLimit( // Delta V2 micro-batch stream Configuration hadoopConf = new Configuration(); PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(testTablePath, hadoopConf); + new PathBasedSnapshotManager(testTablePath, createKernelContext(hadoopConf)); DeltaV2MicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); // We need a separate AdmissionLimits object for DSv2 because the method is stateful. @@ -785,7 +786,7 @@ public void testGetFileChanges_emptyVersions( // Test Delta V2 micro-batch stream Configuration hadoopConf = new Configuration(); PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(testTablePath, hadoopConf); + new PathBasedSnapshotManager(testTablePath, createKernelContext(hadoopConf)); DeltaV2MicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); try (CloseableIterator kernelChanges = @@ -841,7 +842,7 @@ public void testGetFileChanges_lazyLoading(@TempDir File tempDir) throws Excepti Configuration hadoopConf = new Configuration(); PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(testTablePath, hadoopConf); + new PathBasedSnapshotManager(testTablePath, createKernelContext(hadoopConf)); DeltaV2MicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); @@ -954,7 +955,7 @@ public void testGetFileChanges_onRemoveFile_throwError( // Test Delta V2 micro-batch stream Configuration hadoopConf = new Configuration(); PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(testTablePath, hadoopConf); + new PathBasedSnapshotManager(testTablePath, createKernelContext(hadoopConf)); DeltaV2MicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); @@ -1138,7 +1139,7 @@ private void runFileChangeParityTest( // DSv2 Configuration hadoopConf = new Configuration(); PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(testTablePath, hadoopConf); + new PathBasedSnapshotManager(testTablePath, createKernelContext(hadoopConf)); DeltaV2MicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, options); try (CloseableIterator kernelChanges = @@ -1205,7 +1206,7 @@ private void runFileChangeThrowsParityTest( // DSv2 Configuration hadoopConf = new Configuration(); PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(testTablePath, hadoopConf); + new PathBasedSnapshotManager(testTablePath, createKernelContext(hadoopConf)); DeltaV2MicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, options); @@ -1553,7 +1554,7 @@ public void testGetFileChanges_startingVersionAfterCheckpointAndLogCleanup(@Temp // Now test with startingVersion=1 Configuration hadoopConf = spark.sessionState().newHadoopConf(); PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(testTablePath, hadoopConf); + new PathBasedSnapshotManager(testTablePath, createKernelContext(hadoopConf)); DeltaV2MicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); @@ -1621,7 +1622,7 @@ public void testLatestOffset_notInitialSnapshot( // dsv2 Configuration hadoopConf = new Configuration(); PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(testTablePath, hadoopConf); + new PathBasedSnapshotManager(testTablePath, createKernelContext(hadoopConf)); DeltaV2MicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); Offset v2EndOffset = stream.latestOffset(startOffset, readLimit); @@ -1740,7 +1741,7 @@ public void testLatestOffset_sequentialBatchAdvancement( // dsv2 Configuration hadoopConf = new Configuration(); PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(testTablePath, hadoopConf); + new PathBasedSnapshotManager(testTablePath, createKernelContext(hadoopConf)); DeltaV2MicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); List dsv2Offsets = @@ -1866,7 +1867,7 @@ public void testLatestOffset_noNewDataAtLatestVersion( // dsv2 Configuration hadoopConf = new Configuration(); PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(testTablePath, hadoopConf); + new PathBasedSnapshotManager(testTablePath, createKernelContext(hadoopConf)); DeltaV2MicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); Offset dsv2Offset = stream.latestOffset(startOffset, readLimit); @@ -1957,7 +1958,7 @@ public void testAvailableNow_SequentialBatchAdvancement( // dsv2 source Configuration hadoopConf = new Configuration(); PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(testTablePath, hadoopConf); + new PathBasedSnapshotManager(testTablePath, createKernelContext(hadoopConf)); DeltaV2MicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); // Enable availableNow @@ -2100,7 +2101,7 @@ public void testPlanInputPartitions_dataParity( // DSv2: planInputPartitions + createReaderFactory PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(testTablePath, spark.sessionState().newHadoopConf()); + new PathBasedSnapshotManager(testTablePath, createKernelContext()); org.apache.spark.sql.delta.Snapshot deltaSnapshot = deltaLog.unsafeVolatileSnapshot(); StructType dataSchema = deltaSnapshot.metadata().schema(); @@ -2279,7 +2280,7 @@ public void testPlanInputPartitions_excludeRegex( // DSv2: planInputPartitions + createReaderFactory PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(testTablePath, spark.sessionState().newHadoopConf()); + new PathBasedSnapshotManager(testTablePath, createKernelContext()); org.apache.spark.sql.delta.Snapshot deltaSnapshot = deltaLog.unsafeVolatileSnapshot(); StructType fullSchema = deltaSnapshot.metadata().schema(); StructType partitionSchema = deltaSnapshot.metadata().partitionSchema(); @@ -2567,7 +2568,7 @@ public void testGetStartingVersion_noOptions(@TempDir File tempDir) throws Excep // dsv2 PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(testTablePath, new Configuration()); + new PathBasedSnapshotManager(testTablePath, createKernelContext(new Configuration())); DeltaV2MicroBatchStream dsv2Stream = createTestStreamWithDefaults(snapshotManager, new Configuration(), emptyDeltaOptions()); Optional dsv2Result = dsv2Stream.getStartingVersion(); @@ -2697,7 +2698,7 @@ public void testGetStartingVersion_protocolValidationNonFeatureExceptionFallback // dsv2 PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(testTablePath, new Configuration()); + new PathBasedSnapshotManager(testTablePath, createKernelContext(new Configuration())); DeltaV2MicroBatchStream dsv2Stream = createTestStreamWithDefaults( snapshotManager, @@ -2788,7 +2789,7 @@ class TimestampTestCase { // dsv2 PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(testTablePath, new Configuration()); + new PathBasedSnapshotManager(testTablePath, createKernelContext(new Configuration())); DeltaV2MicroBatchStream dsv2Stream = createTestStreamWithDefaults( snapshotManager, @@ -2812,7 +2813,7 @@ class TimestampTestCase { // dsv2 PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(testTablePath, new Configuration()); + new PathBasedSnapshotManager(testTablePath, createKernelContext(new Configuration())); DeltaV2MicroBatchStream dsv2Stream = createTestStreamWithDefaults( snapshotManager, @@ -2878,7 +2879,7 @@ public void testSchemaEvolution_onForwardAdditiveChanges_throwsError( // Create Delta V2 micro-batch stream Configuration hadoopConf = new Configuration(); PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(testTablePath, hadoopConf); + new PathBasedSnapshotManager(testTablePath, createKernelContext(hadoopConf)); DeltaV2MicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); @@ -2999,7 +3000,7 @@ public void testSchemaEvolution_onBackfillAdditiveChanges( // Test Delta V2 micro-batch stream Configuration hadoopConf = new Configuration(); PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(testTablePath, hadoopConf); + new PathBasedSnapshotManager(testTablePath, createKernelContext(hadoopConf)); DeltaV2MicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); try (CloseableIterator kernelChanges = @@ -3055,7 +3056,7 @@ public void testSchemaEvolution_onForwardNonAdditiveChanges_throwsError( // Create Delta V2 micro-batch stream Configuration hadoopConf = new Configuration(); PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(testTablePath, hadoopConf); + new PathBasedSnapshotManager(testTablePath, createKernelContext(hadoopConf)); DeltaV2MicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); @@ -3164,7 +3165,7 @@ public void testSchemaEvolution_onBackfillNonAdditiveChanges_throwsError( // Create Delta V2 micro-batch stream Configuration hadoopConf = new Configuration(); PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(testTablePath, hadoopConf); + new PathBasedSnapshotManager(testTablePath, createKernelContext(hadoopConf)); DeltaV2MicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); @@ -3296,7 +3297,7 @@ public void testSchemaEvolution_onStreamStartOnce(@TempDir File tempDir) throws // Test Delta V2 micro-batch stream Configuration hadoopConf = new Configuration(); PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(testTablePath, hadoopConf); + new PathBasedSnapshotManager(testTablePath, createKernelContext(hadoopConf)); DeltaV2MicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); DeltaUnsupportedOperationException dsv2Exception = @@ -3361,7 +3362,7 @@ public void testSchemaTrackingLifecycle_forward( Configuration hadoopConf = new Configuration(); PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(testTablePath, hadoopConf); + new PathBasedSnapshotManager(testTablePath, createKernelContext(hadoopConf)); String schemaTrackingLocation = new File(tempDir, "schema_tracking").getAbsolutePath(); String checkpointLocation = new File(tempDir, "checkpoint").getAbsolutePath(); java.util.Map optionMap = @@ -3498,7 +3499,7 @@ public void testSchemaTrackingLifecycle_backfill( Configuration hadoopConf = new Configuration(); PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(testTablePath, hadoopConf); + new PathBasedSnapshotManager(testTablePath, createKernelContext(hadoopConf)); String schemaTrackingLocation = new File(tempDir, "schema_tracking").getAbsolutePath(); String checkpointLocation = new File(tempDir, "checkpoint").getAbsolutePath(); java.util.Map optionMap = @@ -3619,7 +3620,7 @@ public void testSchemaTrackingRejectsFailOnDataLossFalse(@TempDir File tempDir) Configuration hadoopConf = new Configuration(); PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(testTablePath, hadoopConf); + new PathBasedSnapshotManager(testTablePath, createKernelContext(hadoopConf)); String schemaTrackingLocation = new File(tempDir, "schema_tracking").getAbsolutePath(); String checkpointLocation = new File(tempDir, "checkpoint").getAbsolutePath(); StructType schema = @@ -3989,7 +3990,7 @@ public void testFailOnDataLoss_false_getFileChangesParity(@TempDir File tempDir) // DSv2 Configuration hadoopConf = new Configuration(); PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(testTablePath, hadoopConf); + new PathBasedSnapshotManager(testTablePath, createKernelContext(hadoopConf)); DeltaV2MicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, options); try (CloseableIterator dsv2Changes = @@ -4052,7 +4053,7 @@ public void testFailOnDataLoss_true_getFileChangesThrowsParity(@TempDir File tem // DSv2: failOnDataLoss=true throws StartVersionNotFoundException with structured fields Configuration hadoopConf = new Configuration(); PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(testTablePath, hadoopConf); + new PathBasedSnapshotManager(testTablePath, createKernelContext(hadoopConf)); DeltaV2MicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, defaultOptions); io.delta.kernel.exceptions.StartVersionNotFoundException dsv2Exception = @@ -4098,7 +4099,7 @@ public void testFailOnDataLoss_false_midLogGapStillThrows(@TempDir File tempDir) Configuration hadoopConf = spark.sessionState().newHadoopConf(); PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(testTablePath, hadoopConf); + new PathBasedSnapshotManager(testTablePath, createKernelContext(hadoopConf)); DeltaOptions options = createDeltaOptions("failOnDataLoss", "false"); DeltaV2MicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, options); @@ -4563,7 +4564,7 @@ private void testAndCompareStartingVersion( // DSv2: Create DeltaV2MicroBatchStream and get starting version PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(testTablePath, new Configuration()); + new PathBasedSnapshotManager(testTablePath, createKernelContext(new Configuration())); DeltaV2MicroBatchStream dsv2Stream = createTestStreamWithDefaults( snapshotManager, @@ -4919,7 +4920,7 @@ public void testDeltaSourceSnapshot_emptyTable(@TempDir File tempDir) throws Exc Configuration hadoopConf = spark.sessionState().newHadoopConf(); PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(testTablePath, hadoopConf); + new PathBasedSnapshotManager(testTablePath, createKernelContext(hadoopConf)); DeltaV2MicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); try { diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/DeltaV2ScanBuilderTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/DeltaV2ScanBuilderTest.java index 5573afd1af7..74dc4cf216f 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/DeltaV2ScanBuilderTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/DeltaV2ScanBuilderTest.java @@ -539,7 +539,7 @@ private DeltaV2ScanBuilder newScanBuilder( StructType partitionSchema, StructType tableSchema) { PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(path, spark.sessionState().newHadoopConf()); + new PathBasedSnapshotManager(path, createKernelContext()); Snapshot snapshot = snapshotManager.loadLatestSnapshot(); Engine engine = DefaultEngine.create(spark.sessionState().newHadoopConf()); return new DeltaV2ScanBuilder( diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/MetadataEvolutionHandlerTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/MetadataEvolutionHandlerTest.java index 29a77348264..dde8afd1341 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/MetadataEvolutionHandlerTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/MetadataEvolutionHandlerTest.java @@ -197,7 +197,7 @@ private MetadataEvolutionHandler buildLightweightHandler( private HandlerWithLog buildHandlerWithRealTable( String tablePath, long initVersion, boolean seedLogWithInitEntry) { PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(tablePath, spark.sessionState().newHadoopConf()); + new PathBasedSnapshotManager(tablePath, createKernelContext()); SnapshotImpl snapshot = DeltaV2Snapshot$.MODULE$.getKernelSnapshot(snapshotManager.loadSnapshotAt(initVersion)); Metadata tableMetadata = snapshot.getMetadata(); @@ -997,7 +997,7 @@ private Option invokeGetTrackingLog( String tableName = "t_" + UUID.randomUUID().toString().replace('-', '_'); createEmptyTestTable(tablePath, tableName); PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(tablePath, spark.sessionState().newHadoopConf()); + new PathBasedSnapshotManager(tablePath, createKernelContext()); Snapshot snapshot = snapshotManager.loadLatestSnapshot(); return MetadataEvolutionHandler.getMetadataTrackingLogForMicroBatchStream( spark, @@ -1137,7 +1137,7 @@ private Optional invokeGetPersistedMetadata( String tableName = "t_" + UUID.randomUUID().toString().replace('-', '_'); createEmptyTestTable(tablePath, tableName); PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(tablePath, spark.sessionState().newHadoopConf()); + new PathBasedSnapshotManager(tablePath, createKernelContext()); Snapshot snapshot = snapshotManager.loadLatestSnapshot(); return MetadataEvolutionHandler.getPersistedMetadataForMicroBatchStream( spark, snapshot, options, snapshotManager, defaultEngine); @@ -1159,7 +1159,7 @@ public void testGetPersistedMetadata_returnsSeededEntry(@TempDir File tempDir) t String tableName = "t_" + UUID.randomUUID().toString().replace('-', '_'); createEmptyTestTable(tablePath, tableName); PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(tablePath, spark.sessionState().newHadoopConf()); + new PathBasedSnapshotManager(tablePath, createKernelContext()); Snapshot dsv1Snapshot = snapshotManager.loadLatestSnapshot(); SnapshotImpl snapshot = DeltaV2Snapshot$.MODULE$.getKernelSnapshot(dsv1Snapshot); @@ -1214,7 +1214,7 @@ private SnapshotImpl loadSourceSnapshot(File tempDir) { String tableName = "t_" + UUID.randomUUID().toString().replace('-', '_'); createEmptyTestTable(tablePath, tableName); PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(tablePath, spark.sessionState().newHadoopConf()); + new PathBasedSnapshotManager(tablePath, createKernelContext()); return DeltaV2Snapshot$.MODULE$.getKernelSnapshot(snapshotManager.loadLatestSnapshot()); } @@ -1444,7 +1444,7 @@ public void testGetMergedConsecutive( } PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(tablePath, spark.sessionState().newHadoopConf()); + new PathBasedSnapshotManager(tablePath, createKernelContext()); PersistedMetadata current = buildCurrentMetadataAtV0(tablePath, snapshotManager); Option result = @@ -1500,7 +1500,7 @@ public void testGetMergedConsecutive_stopsAtCdfUpdateCommit(@TempDir File tempDi spark.sql(String.format("UPDATE %s SET c3 = 10 WHERE id = 1", tableName)); // v4 PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(tablePath, spark.sessionState().newHadoopConf()); + new PathBasedSnapshotManager(tablePath, createKernelContext()); // current = v2 (the ALTER ADD COLUMN). Merger walks forward from here. SnapshotImpl v2Snapshot = diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/ScanFileRDDTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/ScanFileRDDTest.java index cc759a0a423..4c8fadf1bdf 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/ScanFileRDDTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/ScanFileRDDTest.java @@ -42,7 +42,8 @@ public void testScanFileRDDProducesAddFileRows() { spark.range(100).repartition(5).write().format("delta").save(tablePath); Configuration hadoopConf = spark.sessionState().newHadoopConf(); - PathBasedSnapshotManager mgr = new PathBasedSnapshotManager(tablePath, hadoopConf); + PathBasedSnapshotManager mgr = + new PathBasedSnapshotManager(tablePath, createKernelContext(hadoopConf)); SnapshotImpl snapshot = DeltaV2Snapshot$.MODULE$.getKernelSnapshot(mgr.loadLatestSnapshot()); SerializableReadOnlySnapshot serializable = @@ -67,7 +68,8 @@ public void testScanFileRDDSortable() { spark.range(50).repartition(3).write().format("delta").save(tablePath); Configuration hadoopConf = spark.sessionState().newHadoopConf(); - PathBasedSnapshotManager mgr = new PathBasedSnapshotManager(tablePath, hadoopConf); + PathBasedSnapshotManager mgr = + new PathBasedSnapshotManager(tablePath, createKernelContext(hadoopConf)); SnapshotImpl snapshot = DeltaV2Snapshot$.MODULE$.getKernelSnapshot(mgr.loadLatestSnapshot()); SerializableReadOnlySnapshot serializable = diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/snapshot/PathBasedSnapshotManagerTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/snapshot/PathBasedSnapshotManagerTest.java index 5c7ab67f4f9..9e1b4bc60d2 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/snapshot/PathBasedSnapshotManagerTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/snapshot/PathBasedSnapshotManagerTest.java @@ -46,8 +46,7 @@ public void testUnsafeVolatileSnapshot(@TempDir File tempDir) { String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_volatile_snapshot"; createEmptyTestTable(testTablePath, testTableName); - snapshotManager = - new PathBasedSnapshotManager(testTablePath, spark.sessionState().newHadoopConf()); + snapshotManager = new PathBasedSnapshotManager(testTablePath, createKernelContext()); DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); org.apache.spark.sql.delta.Snapshot deltaSnapshot = deltaLog.unsafeVolatileSnapshot(); Snapshot kernelSnapshot = snapshotManager.loadLatestSnapshot(); @@ -63,8 +62,7 @@ public void testLoadLatestSnapshot(@TempDir File tempDir) { String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_update"; createEmptyTestTable(testTablePath, testTableName); - snapshotManager = - new PathBasedSnapshotManager(testTablePath, spark.sessionState().newHadoopConf()); + snapshotManager = new PathBasedSnapshotManager(testTablePath, createKernelContext()); DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); Snapshot initialSnapshot = snapshotManager.loadLatestSnapshot(); @@ -90,8 +88,7 @@ public void testMultipleLoadLatestSnapshot(@TempDir File tempDir) { String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_multiple_updates"; createEmptyTestTable(testTablePath, testTableName); - snapshotManager = - new PathBasedSnapshotManager(testTablePath, spark.sessionState().newHadoopConf()); + snapshotManager = new PathBasedSnapshotManager(testTablePath, createKernelContext()); DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); @@ -116,8 +113,7 @@ public void testLoadSnapshotAt(@TempDir File tempDir) { String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_load_at_version"; createEmptyTestTable(testTablePath, testTableName); - snapshotManager = - new PathBasedSnapshotManager(testTablePath, spark.sessionState().newHadoopConf()); + snapshotManager = new PathBasedSnapshotManager(testTablePath, createKernelContext()); // Create multiple versions for (int i = 0; i < 3; i++) { @@ -163,8 +159,7 @@ public void testGetActiveCommitAtTime_pastTimestamp(@TempDir File tempDir) throw String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_commit_past"; setupTableWithDeletedVersions(testTablePath, testTableName); - snapshotManager = - new PathBasedSnapshotManager(testTablePath, spark.sessionState().newHadoopConf()); + snapshotManager = new PathBasedSnapshotManager(testTablePath, createKernelContext()); Thread.sleep(100); Timestamp timestamp = new Timestamp(System.currentTimeMillis()); @@ -198,8 +193,7 @@ public void testGetActiveCommitAtTime_futureTimestamp_canReturnLast(@TempDir Fil String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_commit_future_last"; setupTableWithDeletedVersions(testTablePath, testTableName); - snapshotManager = - new PathBasedSnapshotManager(testTablePath, spark.sessionState().newHadoopConf()); + snapshotManager = new PathBasedSnapshotManager(testTablePath, createKernelContext()); Timestamp futureTimestamp = new Timestamp(System.currentTimeMillis() + 10000); @@ -231,8 +225,7 @@ public void testGetActiveCommitAtTime_futureTimestamp_notMustBeRecreatable(@Temp String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_commit_future_not_recreatable"; setupTableWithDeletedVersions(testTablePath, testTableName); - snapshotManager = - new PathBasedSnapshotManager(testTablePath, spark.sessionState().newHadoopConf()); + snapshotManager = new PathBasedSnapshotManager(testTablePath, createKernelContext()); Timestamp futureTimestamp = new Timestamp(System.currentTimeMillis() + 10000); @@ -264,8 +257,7 @@ public void testGetActiveCommitAtTime_earlyTimestamp_canReturnEarliest(@TempDir String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_commit_early"; setupTableWithDeletedVersions(testTablePath, testTableName); - snapshotManager = - new PathBasedSnapshotManager(testTablePath, spark.sessionState().newHadoopConf()); + snapshotManager = new PathBasedSnapshotManager(testTablePath, createKernelContext()); Timestamp earlyTimestamp = new Timestamp(0); @@ -297,8 +289,7 @@ public void testGetActiveCommitAtTime_earlyTimestamp_notMustBeRecreatable_canRet String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_commit_early_not_recreatable"; setupTableWithDeletedVersions(testTablePath, testTableName); - snapshotManager = - new PathBasedSnapshotManager(testTablePath, spark.sessionState().newHadoopConf()); + snapshotManager = new PathBasedSnapshotManager(testTablePath, createKernelContext()); Timestamp earlyTimestamp = new Timestamp(0); @@ -377,8 +368,7 @@ public void testCheckVersionExists( String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_version_" + testName; setupTableWithDeletedVersions(testTablePath, testTableName); - snapshotManager = - new PathBasedSnapshotManager(testTablePath, spark.sessionState().newHadoopConf()); + snapshotManager = new PathBasedSnapshotManager(testTablePath, createKernelContext()); DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); if (shouldThrow) { diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/SerializableReadOnlySnapshotTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/SerializableReadOnlySnapshotTest.java index aff51b51dc2..f4693234c2d 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/SerializableReadOnlySnapshotTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/SerializableReadOnlySnapshotTest.java @@ -48,7 +48,8 @@ public void testSerializationRoundTrip() throws Exception { spark.range(10).write().format("delta").save(tablePath); Configuration hadoopConf = spark.sessionState().newHadoopConf(); - PathBasedSnapshotManager mgr = new PathBasedSnapshotManager(tablePath, hadoopConf); + PathBasedSnapshotManager mgr = + new PathBasedSnapshotManager(tablePath, createKernelContext(hadoopConf)); SnapshotImpl snapshot = DeltaV2Snapshot$.MODULE$.getKernelSnapshot(mgr.loadLatestSnapshot()); SerializableReadOnlySnapshot original = @@ -88,7 +89,8 @@ public void testToScanReturnsReadOnlyScan() { spark.range(5).write().format("delta").save(tablePath); Configuration hadoopConf = spark.sessionState().newHadoopConf(); - PathBasedSnapshotManager mgr = new PathBasedSnapshotManager(tablePath, hadoopConf); + PathBasedSnapshotManager mgr = + new PathBasedSnapshotManager(tablePath, createKernelContext(hadoopConf)); SnapshotImpl snapshot = DeltaV2Snapshot$.MODULE$.getKernelSnapshot(mgr.loadLatestSnapshot()); SerializableReadOnlySnapshot serializable = diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/StreamingHelperTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/StreamingHelperTest.java index edf0cf97381..72c185f1773 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/StreamingHelperTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/StreamingHelperTest.java @@ -54,8 +54,7 @@ public void testUnsafeVolatileSnapshot(@TempDir File tempDir) { String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_volatile_snapshot"; createEmptyTestTable(testTablePath, testTableName); - snapshotManager = - new PathBasedSnapshotManager(testTablePath, spark.sessionState().newHadoopConf()); + snapshotManager = new PathBasedSnapshotManager(testTablePath, createKernelContext()); DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); org.apache.spark.sql.delta.Snapshot deltaSnapshot = deltaLog.unsafeVolatileSnapshot(); Snapshot kernelSnapshot = snapshotManager.loadLatestSnapshot(); @@ -71,8 +70,7 @@ public void testLoadLatestSnapshot(@TempDir File tempDir) { String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_update"; createEmptyTestTable(testTablePath, testTableName); - snapshotManager = - new PathBasedSnapshotManager(testTablePath, spark.sessionState().newHadoopConf()); + snapshotManager = new PathBasedSnapshotManager(testTablePath, createKernelContext()); DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); Snapshot initialSnapshot = snapshotManager.loadLatestSnapshot(); @@ -97,8 +95,7 @@ public void testMultipleLoadLatestSnapshot(@TempDir File tempDir) { String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_multiple_updates"; createEmptyTestTable(testTablePath, testTableName); - snapshotManager = - new PathBasedSnapshotManager(testTablePath, spark.sessionState().newHadoopConf()); + snapshotManager = new PathBasedSnapshotManager(testTablePath, createKernelContext()); DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); @@ -140,8 +137,7 @@ public void testGetActiveCommitAtTime_pastTimestamp(@TempDir File tempDir) throw String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_commit_past"; setupTableWithDeletedVersions(testTablePath, testTableName); - snapshotManager = - new PathBasedSnapshotManager(testTablePath, spark.sessionState().newHadoopConf()); + snapshotManager = new PathBasedSnapshotManager(testTablePath, createKernelContext()); Thread.sleep(100); Timestamp timestamp = new Timestamp(System.currentTimeMillis()); @@ -175,8 +171,7 @@ public void testGetActiveCommitAtTime_futureTimestamp_canReturnLast(@TempDir Fil String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_commit_future_last"; setupTableWithDeletedVersions(testTablePath, testTableName); - snapshotManager = - new PathBasedSnapshotManager(testTablePath, spark.sessionState().newHadoopConf()); + snapshotManager = new PathBasedSnapshotManager(testTablePath, createKernelContext()); Timestamp futureTimestamp = new Timestamp(System.currentTimeMillis() + 10000); @@ -208,8 +203,7 @@ public void testGetActiveCommitAtTime_futureTimestamp_notMustBeRecreatable(@Temp String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_commit_future_not_recreatable"; setupTableWithDeletedVersions(testTablePath, testTableName); - snapshotManager = - new PathBasedSnapshotManager(testTablePath, spark.sessionState().newHadoopConf()); + snapshotManager = new PathBasedSnapshotManager(testTablePath, createKernelContext()); Timestamp futureTimestamp = new Timestamp(System.currentTimeMillis() + 10000); @@ -241,8 +235,7 @@ public void testGetActiveCommitAtTime_earlyTimestamp_canReturnEarliest(@TempDir String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_commit_early"; setupTableWithDeletedVersions(testTablePath, testTableName); - snapshotManager = - new PathBasedSnapshotManager(testTablePath, spark.sessionState().newHadoopConf()); + snapshotManager = new PathBasedSnapshotManager(testTablePath, createKernelContext()); Timestamp earlyTimestamp = new Timestamp(0); @@ -274,8 +267,7 @@ public void testGetActiveCommitAtTime_earlyTimestamp_notMustBeRecreatable_canRet String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_commit_early_not_recreatable"; setupTableWithDeletedVersions(testTablePath, testTableName); - snapshotManager = - new PathBasedSnapshotManager(testTablePath, spark.sessionState().newHadoopConf()); + snapshotManager = new PathBasedSnapshotManager(testTablePath, createKernelContext()); Timestamp earlyTimestamp = new Timestamp(0); @@ -354,8 +346,7 @@ public void testCheckVersionExists( String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_version_" + testName; setupTableWithDeletedVersions(testTablePath, testTableName); - snapshotManager = - new PathBasedSnapshotManager(testTablePath, spark.sessionState().newHadoopConf()); + snapshotManager = new PathBasedSnapshotManager(testTablePath, createKernelContext()); DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); if (shouldThrow) { @@ -416,8 +407,7 @@ public void testCollectMetadataActionsFromRangeUnsafe( String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_collect_metadata_" + scenario; setupTableWithMetadataChangeAtV2(testTablePath, testTableName); - snapshotManager = - new PathBasedSnapshotManager(testTablePath, spark.sessionState().newHadoopConf()); + snapshotManager = new PathBasedSnapshotManager(testTablePath, createKernelContext()); Map result = StreamingHelper.collectMetadataActionsFromRangeUnsafe( @@ -433,8 +423,7 @@ public void testCollectMetadataActionsFromRangeUnsafe_returnsCorrectMetadataPerV String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_collect_metadata_content"; setupTableWithMetadataChangeAtV2(testTablePath, testTableName); - snapshotManager = - new PathBasedSnapshotManager(testTablePath, spark.sessionState().newHadoopConf()); + snapshotManager = new PathBasedSnapshotManager(testTablePath, createKernelContext()); Map result = StreamingHelper.collectMetadataActionsFromRangeUnsafe( @@ -493,8 +482,7 @@ public void testCollectProtocolActionsFromRangeUnsafe( String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_collect_protocol_" + scenario; setupTableWithProtocolUpgradeAtV2(testTablePath, testTableName); - snapshotManager = - new PathBasedSnapshotManager(testTablePath, spark.sessionState().newHadoopConf()); + snapshotManager = new PathBasedSnapshotManager(testTablePath, createKernelContext()); Map result = StreamingHelper.collectProtocolActionsFromRangeUnsafe( @@ -510,8 +498,7 @@ public void testCollectProtocolActionsFromRangeUnsafe_returnsCorrectProtocolPerV String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_collect_protocol_content"; setupTableWithProtocolUpgradeAtV2(testTablePath, testTableName); - snapshotManager = - new PathBasedSnapshotManager(testTablePath, spark.sessionState().newHadoopConf()); + snapshotManager = new PathBasedSnapshotManager(testTablePath, createKernelContext()); Map result = StreamingHelper.collectProtocolActionsFromRangeUnsafe( diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/write/DeltaRowLevelOperationBuilderTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/write/DeltaRowLevelOperationBuilderTest.java index fe37f229b25..5a7b2103be4 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/write/DeltaRowLevelOperationBuilderTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/write/DeltaRowLevelOperationBuilderTest.java @@ -17,10 +17,10 @@ import static org.junit.jupiter.api.Assertions.*; -import io.delta.kernel.defaults.engine.DefaultEngine; import io.delta.kernel.engine.Engine; import io.delta.spark.internal.v2.DeltaV2TestBase; import io.delta.spark.internal.v2.catalog.DeltaV2Table; +import io.delta.spark.internal.v2.kernel.KernelContext; import io.delta.spark.internal.v2.snapshot.PathBasedSnapshotManager; import java.io.File; import java.util.Collections; @@ -105,9 +105,10 @@ public void constructorRejectsNull(@TempDir File tempDir) { DeltaV2Table table = createRowLevelTable(tempDir); RowLevelOperationInfo info = testInfo(RowLevelOperation.Command.DELETE); Configuration hadoopConf = spark.sessionState().newHadoopConf(); - Engine engine = DefaultEngine.create(hadoopConf); + KernelContext kernelContext = createKernelContext(hadoopConf); + Engine engine = kernelContext.getDefaultEngine(); Snapshot snapshot = - new PathBasedSnapshotManager(tempDir.getAbsolutePath(), engine).loadLatestSnapshot(); + new PathBasedSnapshotManager(tempDir.getAbsolutePath(), kernelContext).loadLatestSnapshot(); assertThrows( NullPointerException.class, @@ -137,9 +138,11 @@ private DeltaRowLevelOperationBuilder createBuilder( File tableDir, RowLevelOperation.Command command) { DeltaV2Table table = createRowLevelTable(new File(tableDir, "table_" + System.nanoTime())); Configuration hadoopConf = spark.sessionState().newHadoopConf(); - Engine engine = DefaultEngine.create(hadoopConf); + KernelContext kernelContext = createKernelContext(hadoopConf); + Engine engine = kernelContext.getDefaultEngine(); Snapshot snapshot = - new PathBasedSnapshotManager(table.getTablePath().toString(), engine).loadLatestSnapshot(); + new PathBasedSnapshotManager(table.getTablePath().toString(), kernelContext) + .loadLatestSnapshot(); return new DeltaRowLevelOperationBuilder( table, engine, hadoopConf, snapshot, testInfo(command)); } diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/write/DeltaV2BatchWriteTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/write/DeltaV2BatchWriteTest.java index f9b19801f21..2bf0bf7ce3a 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/write/DeltaV2BatchWriteTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/write/DeltaV2BatchWriteTest.java @@ -469,8 +469,7 @@ public void testCommit_partitioned_multipleColumns(@TempDir File tempDir) throws DataTypes.createStructField("p1", DataTypes.StringType, true), DataTypes.createStructField("p2", DataTypes.IntegerType, true) }); - PathBasedSnapshotManager mgr = - new PathBasedSnapshotManager(path, spark.sessionState().newHadoopConf()); + PathBasedSnapshotManager mgr = new PathBasedSnapshotManager(path, createKernelContext()); DeltaV2BatchWrite write = (DeltaV2BatchWrite) new DeltaV2Write( @@ -565,8 +564,7 @@ private DeltaV2BatchWrite newMultiColWrite(String path) { DataTypes.createStructField("p1", DataTypes.StringType, true), DataTypes.createStructField("p2", DataTypes.IntegerType, true) }); - PathBasedSnapshotManager mgr = - new PathBasedSnapshotManager(path, spark.sessionState().newHadoopConf()); + PathBasedSnapshotManager mgr = new PathBasedSnapshotManager(path, createKernelContext()); return (DeltaV2BatchWrite) new DeltaV2Write( defaultEngine, @@ -609,8 +607,7 @@ private DeltaV2BatchWrite newTimestampPartWrite(String path) { StructType part = DataTypes.createStructType( new StructField[] {DataTypes.createStructField("part", DataTypes.TimestampType, true)}); - PathBasedSnapshotManager mgr = - new PathBasedSnapshotManager(path, spark.sessionState().newHadoopConf()); + PathBasedSnapshotManager mgr = new PathBasedSnapshotManager(path, createKernelContext()); return (DeltaV2BatchWrite) new DeltaV2Write( defaultEngine, @@ -638,7 +635,7 @@ private String createPartitionedTable(File tempDir, String tableName) { private DeltaV2BatchWrite newPartitionedWrite(String path) { PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(path, spark.sessionState().newHadoopConf()); + new PathBasedSnapshotManager(path, createKernelContext()); Snapshot snapshot = snapshotManager.loadLatestSnapshot(); LogicalWriteInfo info = WriteTestUtils.logicalWriteInfo(PARTITIONED_FULL_SCHEMA, CaseInsensitiveStringMap.empty()); @@ -742,7 +739,7 @@ private DeltaV2BatchWrite newWrite(String path, String tableName) throws Excepti CatalogTable catalogTable = spark.sessionState().catalog().getTableMetadata(new TableIdentifier(tableName)); PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(path, spark.sessionState().newHadoopConf()); + new PathBasedSnapshotManager(path, createKernelContext()); Snapshot snapshot = snapshotManager.loadLatestSnapshot(); LogicalWriteInfo info = WriteTestUtils.logicalWriteInfo(TABLE_SCHEMA, CaseInsensitiveStringMap.empty()); diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/write/DeltaV2StreamingWriteTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/write/DeltaV2StreamingWriteTest.java index f1e2d01e276..dcb391567de 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/write/DeltaV2StreamingWriteTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/write/DeltaV2StreamingWriteTest.java @@ -594,7 +594,7 @@ private DeltaV2StreamingWrite newWrite(String path) { private DeltaV2StreamingWrite newWrite(String path, boolean variantShreddingEnabled) { PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(path, spark.sessionState().newHadoopConf()); + new PathBasedSnapshotManager(path, createKernelContext()); Snapshot snapshot = snapshotManager.loadLatestSnapshot(); LogicalWriteInfo info = WriteTestUtils.logicalWriteInfo(TABLE_SCHEMA, CaseInsensitiveStringMap.empty()); @@ -659,7 +659,7 @@ private String createVariantTable(File tempDir, String tableName, boolean shredd private DeltaV2StreamingWrite newVariantWrite(String path, boolean variantShreddingEnabled) { PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(path, spark.sessionState().newHadoopConf()); + new PathBasedSnapshotManager(path, createKernelContext()); Snapshot snapshot = snapshotManager.loadLatestSnapshot(); LogicalWriteInfo info = WriteTestUtils.logicalWriteInfo(VARIANT_TABLE_SCHEMA, CaseInsensitiveStringMap.empty()); @@ -702,7 +702,7 @@ private String createPartitionedTable(File tempDir, String tableName) { private DeltaV2StreamingWrite newPartitionedWrite(String path) { PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(path, spark.sessionState().newHadoopConf()); + new PathBasedSnapshotManager(path, createKernelContext()); Snapshot snapshot = snapshotManager.loadLatestSnapshot(); LogicalWriteInfo info = WriteTestUtils.logicalWriteInfo(PARTITIONED_FULL_SCHEMA, CaseInsensitiveStringMap.empty()); diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/write/DeltaV2WriteContextTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/write/DeltaV2WriteContextTest.java index 940f7d63d28..abf7d71a2cc 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/write/DeltaV2WriteContextTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/write/DeltaV2WriteContextTest.java @@ -26,10 +26,10 @@ import io.delta.kernel.Operation; import io.delta.kernel.TableManager; -import io.delta.kernel.defaults.engine.DefaultEngine; import io.delta.kernel.engine.Engine; import io.delta.kernel.utils.CloseableIterable; import io.delta.spark.internal.v2.DeltaV2TestBase; +import io.delta.spark.internal.v2.kernel.KernelContext; import io.delta.spark.internal.v2.snapshot.PathBasedSnapshotManager; import io.delta.spark.internal.v2.utils.ScalaUtils; import io.delta.spark.internal.v2.utils.SchemaUtils; @@ -62,9 +62,10 @@ public void createInitializesOperationIndependentState(@TempDir File tempDir) th String path = tempDir.getAbsolutePath(); StructType tableSchema = tableSchema(); Configuration hadoopConf = spark.sessionState().newHadoopConf(); - Engine engine = DefaultEngine.create(hadoopConf); + KernelContext kernelContext = createKernelContext(hadoopConf); + Engine engine = kernelContext.getDefaultEngine(); createKernelTable(path, tableSchema, engine); - Snapshot snapshot = new PathBasedSnapshotManager(path, engine).loadLatestSnapshot(); + Snapshot snapshot = new PathBasedSnapshotManager(path, kernelContext).loadLatestSnapshot(); DeltaV2WriteContext context = DeltaV2WriteContext.create( @@ -97,9 +98,10 @@ public void buildDataWriterFactoryProducesExecutorState(@TempDir File tempDir) t String path = tempDir.getAbsolutePath(); StructType tableSchema = tableSchema(); Configuration hadoopConf = spark.sessionState().newHadoopConf(); - Engine engine = DefaultEngine.create(hadoopConf); + KernelContext kernelContext = createKernelContext(hadoopConf); + Engine engine = kernelContext.getDefaultEngine(); createKernelTable(path, tableSchema, engine); - Snapshot snapshot = new PathBasedSnapshotManager(path, engine).loadLatestSnapshot(); + Snapshot snapshot = new PathBasedSnapshotManager(path, kernelContext).loadLatestSnapshot(); DeltaV2WriteContext context = DeltaV2WriteContext.create( @@ -132,8 +134,9 @@ public void wiredPartitionSchemaIsPreserved(@TempDir File tempDir) throws Except StructType partitionSchema = new StructType().add("part", DataTypes.StringType); Configuration hadoopConf = spark.sessionState().newHadoopConf(); - Engine engine = DefaultEngine.create(hadoopConf); - Snapshot snapshot = new PathBasedSnapshotManager(path, engine).loadLatestSnapshot(); + KernelContext kernelContext = createKernelContext(hadoopConf); + Engine engine = kernelContext.getDefaultEngine(); + Snapshot snapshot = new PathBasedSnapshotManager(path, kernelContext).loadLatestSnapshot(); DeltaV2WriteContext context = DeltaV2WriteContext.create( @@ -168,8 +171,9 @@ public void buildDataWriterFactoryResolvesColumnsCaseInsensitively(@TempDir File StructType partitionSchema = new StructType().add("part", DataTypes.StringType); Configuration hadoopConf = spark.sessionState().newHadoopConf(); - Engine engine = DefaultEngine.create(hadoopConf); - Snapshot snapshot = new PathBasedSnapshotManager(path, engine).loadLatestSnapshot(); + KernelContext kernelContext = createKernelContext(hadoopConf); + Engine engine = kernelContext.getDefaultEngine(); + Snapshot snapshot = new PathBasedSnapshotManager(path, kernelContext).loadLatestSnapshot(); DeltaV2WriteContext context = DeltaV2WriteContext.create( @@ -275,9 +279,10 @@ public void variantLayoutFollowsPropertyMatchesShreddingEligibility(@TempDir Fil private DeltaV2WriteContext buildContext(File dir, StructType schema) throws Exception { String path = dir.getAbsolutePath(); Configuration hadoopConf = spark.sessionState().newHadoopConf(); - Engine engine = DefaultEngine.create(hadoopConf); + KernelContext kernelContext = createKernelContext(hadoopConf); + Engine engine = kernelContext.getDefaultEngine(); createKernelTable(path, schema, engine); - Snapshot snapshot = new PathBasedSnapshotManager(path, engine).loadLatestSnapshot(); + Snapshot snapshot = new PathBasedSnapshotManager(path, kernelContext).loadLatestSnapshot(); return DeltaV2WriteContext.create( engine, hadoopConf, diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/write/DeltaV2WriteTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/write/DeltaV2WriteTest.java index 09a674ec167..46624cfcb21 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/write/DeltaV2WriteTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/write/DeltaV2WriteTest.java @@ -107,8 +107,7 @@ public void testDistributionAndOrdering_partitionedSortsWithoutDistribution( StructType partitionSchema = DataTypes.createStructType( new StructField[] {DataTypes.createStructField("name", DataTypes.StringType, true)}); - PathBasedSnapshotManager mgr = - new PathBasedSnapshotManager(path, spark.sessionState().newHadoopConf()); + PathBasedSnapshotManager mgr = new PathBasedSnapshotManager(path, createKernelContext()); DeltaV2Write write = new DeltaV2Write( defaultEngine, @@ -155,8 +154,7 @@ public void testDistributionAndOrdering_multipleColumnsSortInPartitionOrder( DataTypes.createStructField("p1", DataTypes.StringType, true), DataTypes.createStructField("p2", DataTypes.IntegerType, true) }); - PathBasedSnapshotManager mgr = - new PathBasedSnapshotManager(path, spark.sessionState().newHadoopConf()); + PathBasedSnapshotManager mgr = new PathBasedSnapshotManager(path, createKernelContext()); DeltaV2Write write = new DeltaV2Write( defaultEngine, @@ -189,7 +187,7 @@ private String createTable(File tempDir, String tableName) { private DeltaV2Write newWrite(String path, CaseInsensitiveStringMap options) { PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(path, spark.sessionState().newHadoopConf()); + new PathBasedSnapshotManager(path, createKernelContext()); Snapshot snapshot = snapshotManager.loadLatestSnapshot(); LogicalWriteInfo info = WriteTestUtils.logicalWriteInfo(TABLE_SCHEMA, options); return new DeltaV2Write( diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/write/DeltaV2WriterCommitMessageTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/write/DeltaV2WriterCommitMessageTest.java index 477bbafdb3d..f1c868f1dec 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/write/DeltaV2WriterCommitMessageTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/write/DeltaV2WriterCommitMessageTest.java @@ -122,7 +122,7 @@ private WriterCommitMessage writeFile( private DeltaV2DataWriterFactory dataWriterFactory(String path) { PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(path, spark.sessionState().newHadoopConf()); + new PathBasedSnapshotManager(path, createKernelContext()); Snapshot snapshot = snapshotManager.loadLatestSnapshot(); LogicalWriteInfo info = WriteTestUtils.logicalWriteInfo(TABLE_SCHEMA, CaseInsensitiveStringMap.empty()); diff --git a/spark/v2/src/test/scala/io/delta/spark/internal/v2/snapshot/unitycatalog/UCManagedTableSnapshotManagerSuite.scala b/spark/v2/src/test/scala/io/delta/spark/internal/v2/snapshot/unitycatalog/UCManagedTableSnapshotManagerSuite.scala index a1d5cf037a5..210721235cf 100644 --- a/spark/v2/src/test/scala/io/delta/spark/internal/v2/snapshot/unitycatalog/UCManagedTableSnapshotManagerSuite.scala +++ b/spark/v2/src/test/scala/io/delta/spark/internal/v2/snapshot/unitycatalog/UCManagedTableSnapshotManagerSuite.scala @@ -22,8 +22,10 @@ import scala.jdk.CollectionConverters._ import io.delta.kernel.exceptions.{KernelException, VersionToLoadAfterLatestCommitException} import io.delta.kernel.unitycatalog.{InMemoryUCClient, UCCatalogManagedClient, UCCatalogManagedTestUtils, UCTableIdentifier} import io.delta.spark.internal.v2.exception.VersionNotFoundException +import io.delta.spark.internal.v2.kernel.KernelContext import io.delta.storage.commit.uccommitcoordinator.InvalidTargetTableException +import org.apache.spark.sql.delta.storage.LogStore import org.scalatest.Outcome import org.scalatest.funsuite.AnyFunSuite @@ -41,13 +43,15 @@ class UCManagedTableSnapshotManagerSuite private val testUcToken = "test-token" private val testUcAuthConfig = Map("token" -> testUcToken).asJava + private def kernelContext = new KernelContext(Map.empty, LogStore(spark)) + private def createManager( ucClient: InMemoryUCClient, tablePath: String) = { val client = new UCCatalogManagedClient(ucClient) val tableInfo = new UCTableInfo(testUcTableId, tablePath, testTableIdentifier, testUcUri, testUcAuthConfig) - new UCManagedTableSnapshotManager(client, tableInfo, defaultEngine) + new UCManagedTableSnapshotManager(client, tableInfo, kernelContext) } // ==================== Constructor ==================== @@ -64,19 +68,19 @@ class UCManagedTableSnapshotManagerSuite testUcAuthConfig) val ex1 = intercept[NullPointerException] { - new UCManagedTableSnapshotManager(null, tableInfo, defaultEngine) + new UCManagedTableSnapshotManager(null, tableInfo, kernelContext) } assert(ex1.getMessage == "ucCatalogManagedClient is null") val ex2 = intercept[NullPointerException] { - new UCManagedTableSnapshotManager(client, null, defaultEngine) + new UCManagedTableSnapshotManager(client, null, kernelContext) } assert(ex2.getMessage == "tableInfo is null") val ex3 = intercept[NullPointerException] { new UCManagedTableSnapshotManager(client, tableInfo, null) } - assert(ex3.getMessage == "engine is null") + assert(ex3.getMessage == "kernelContext is null") } // ==================== loadLatestSnapshot ==================== @@ -102,7 +106,7 @@ class UCManagedTableSnapshotManagerSuite testUcUri, testUcAuthConfig) val client = new UCCatalogManagedClient(ucClient) - val manager = new UCManagedTableSnapshotManager(client, tableInfo, defaultEngine) + val manager = new UCManagedTableSnapshotManager(client, tableInfo, kernelContext) val ex = intercept[RuntimeException] { manager.loadLatestSnapshot() @@ -335,7 +339,7 @@ class UCManagedTableSnapshotManagerSuite testUcUri, testUcAuthConfig) val client = new UCCatalogManagedClient(ucClient) - val manager = new UCManagedTableSnapshotManager(client, tableInfo, defaultEngine) + val manager = new UCManagedTableSnapshotManager(client, tableInfo, kernelContext) val ex1 = intercept[RuntimeException] { manager.loadLatestSnapshot() } assert(ex1.getCause.isInstanceOf[InvalidTargetTableException]) diff --git a/spark/v2/src/test/scala/io/delta/spark/internal/v2/tablemanager/CachedSnapshotManagerSuite.scala b/spark/v2/src/test/scala/io/delta/spark/internal/v2/tablemanager/CachedSnapshotManagerSuite.scala index b04a7b36eda..aed740ab984 100644 --- a/spark/v2/src/test/scala/io/delta/spark/internal/v2/tablemanager/CachedSnapshotManagerSuite.scala +++ b/spark/v2/src/test/scala/io/delta/spark/internal/v2/tablemanager/CachedSnapshotManagerSuite.scala @@ -386,16 +386,14 @@ class CachedSnapshotManagerSuite CachedSnapshotManagerRecordingFileSystem.clear() assert(manager.loadLatestSnapshot().version == 0L) assertRecordingFileSystemObserved("first-operation") - val retainedEngine = kernelContext.getDefaultEngine() appendToDeltaTable(dir) SparkSession.setActiveSession(secondOperationSession) setRecordingMarkers(secondOperationSession, "second-operation") CachedSnapshotManagerRecordingFileSystem.clear() assert(manager.loadSnapshotAt(1L).version == 1L) - assert(kernelContext.getDefaultEngine() eq retainedEngine) val expectedSessionMarker = - "first-operation" + "second-operation" assertRecordingFileSystemObserved(expectedSessionMarker) } finally { if (manager != null) manager.retire() diff --git a/spark/v2/src/test/scala/io/delta/spark/internal/v2/tablemanager/DeltaV2TableManagerImplSuite.scala b/spark/v2/src/test/scala/io/delta/spark/internal/v2/tablemanager/DeltaV2TableManagerImplSuite.scala index 81340aea928..5d57510e12c 100644 --- a/spark/v2/src/test/scala/io/delta/spark/internal/v2/tablemanager/DeltaV2TableManagerImplSuite.scala +++ b/spark/v2/src/test/scala/io/delta/spark/internal/v2/tablemanager/DeltaV2TableManagerImplSuite.scala @@ -76,14 +76,14 @@ class DeltaV2TableManagerImplSuite spark.range(2, 3, 1, 1).write.format("delta").mode("append").save(path) forPathAndCatalogManagers(path) { (manager, catalogTableOpt) => - val kernelEngine = manager.kernelContext.getDefaultEngine() + val kernelContext = manager.kernelContext val atVersionZeroManager = manager.snapshotManager(catalogTableOpt) val atVersionOneManager = manager.snapshotManager(catalogTableOpt) val latestManager = manager.snapshotManager(catalogTableOpt) assert(atVersionZeroManager eq atVersionOneManager) assert(atVersionOneManager eq latestManager) - assert(manager.kernelContext.getDefaultEngine() eq kernelEngine) + assert(manager.kernelContext eq kernelContext) val atVersionZero = atVersionZeroManager.loadSnapshotAt(0) assert(atVersionZero.version == 0) @@ -100,7 +100,7 @@ class DeltaV2TableManagerImplSuite } } - test("cached composite reuses the table manager and Kernel Engine") { + test("cached composite reuses the table manager and KernelContext") { withSQLConf(DeltaSQLConf.DELTA_LOG_CACHE_SIZE.key -> "1000") { withTempDir { dir => val first = DeltaV2TableManagerCache @@ -117,7 +117,7 @@ class DeltaV2TableManagerImplSuite .asInstanceOf[DeltaV2TableManagerImpl] assert(first eq second) - assert(first.kernelContext.getDefaultEngine() eq second.kernelContext.getDefaultEngine()) + assert(first.kernelContext eq second.kernelContext) } } } From 891d3855cfaf5993a2f78ed5886a8b56733dc660 Mon Sep 17 00:00:00 2001 From: Yumingxuan Guo Date: Mon, 12 Oct 2026 16:29:31 -0700 Subject: [PATCH 06/29] [Spark] Serialize AMT checkpoints as action arrays (#7824) #### Which Delta project/connector is this regarding? - [x] Spark ## Description AMT checkpoints currently serialize as an object with separate fields. Change the checkpoint payload to an array of single-action envelopes, as specified by the [adaptive metadata RFC](https://github.com/delta-io/delta/pull/6978#discussion_r3532500414). Introduce `AMTCheckpointAction` and `AMTCheckpointSingleAction`, carry the checkpoint version in `CheckpointMetadata`, and preserve the field accessors and field-based constructor. Validate that each required singleton action appears exactly once, allow unknown envelopes, and preserve action order. Align Jackson serialization and Spark decoding, including the sidecar `type` field. ## How was this patch tested? Adds and updates coverage in `AdaptiveMetadataActionsSerializerSuite`, `ActionSerializerSuite`, and `LastCheckpointInfoSuite` for action wrapping, JSON round trips, required-action validation, unknown envelopes, Spark schemas, and decoding with sidecars. Updates AMT checkpoint, snapshot, inheritance, writer, and field-spec tests for the new representation. Whitespace and patch-integrity checks pass. Scala tests have not been run locally for this revision; CI validation is pending. ## Does this PR introduce _any_ user-facing changes? The AMT checkpoint JSON payload changes from a field object to an array of action envelopes, matching the RFC. The checkpoint version is stored in its `checkpointMetadata` action. Signed-off-by: Yumingxuan Guo <171533467+yumingxuanguo-db@users.noreply.github.com> Co-authored-by: Yumingxuan Guo <171533467+yumingxuanguo-db@users.noreply.github.com> --- .../spark/sql/delta/actions/actions.scala | 239 +++++++++++--- .../sql/delta/ActionSerializerSuite.scala | 65 +++- .../sql/delta/LastCheckpointInfoSuite.scala | 26 +- .../sql/delta/amt/AMTCheckpointTestBase.scala | 14 +- .../delta/amt/AMTCheckpointWriteSuite.scala | 46 ++- .../sql/delta/amt/AMTFieldSpecSuite.scala | 13 +- .../delta/amt/AMTInheritanceReadSuite.scala | 7 +- .../sql/delta/amt/AMTSnapshotSuite.scala | 6 +- .../sql/delta/amt/AMTWriterManagerSuite.scala | 20 +- ...aptiveMetadataActionsSerializerSuite.scala | 304 ++++++++++++++---- 10 files changed, 606 insertions(+), 134 deletions(-) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/actions/actions.scala b/spark/src/main/scala/org/apache/spark/sql/delta/actions/actions.scala index 2bbe7cacec3..0d4a755e178 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/actions/actions.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/actions/actions.scala @@ -22,9 +22,11 @@ import java.sql.Timestamp import java.util.Locale import java.util.concurrent.TimeUnit +import scala.annotation.meta.getter import scala.annotation.tailrec import scala.collection.JavaConverters._ import scala.collection.mutable +import scala.reflect.ClassTag import scala.util.control.NonFatal import com.databricks.spark.util.TagDefinition @@ -218,6 +220,11 @@ sealed trait Action { def json: String = JsonUtils.toJson(wrap) } +/** An action that can be included in an AMT [[Checkpoint]]. */ +sealed trait AMTCheckpointAction { + def wrapAsAMTCheckpointSingleAction: AMTCheckpointSingleAction +} + /** * Used to block older clients from reading or writing the log when backwards incompatible changes * are made to the protocol. Readers and writers are responsible for checking that they meet the @@ -238,6 +245,7 @@ case class Protocol private ( @JsonInclude(Include.NON_ABSENT) writerFeatures: Option[Set[String]]) extends Action + with AMTCheckpointAction with SparkAbstractProtocol with StorageAbstractProtocol with TableFeatureSupport { @@ -258,6 +266,9 @@ case class Protocol private ( override def wrap: SingleAction = SingleAction(protocol = this) + override def wrapAsAMTCheckpointSingleAction: AMTCheckpointSingleAction = + AMTCheckpointSingleAction(protocol = this) + /** * Return a reader-friendly string representation of this Protocol. * @@ -677,8 +688,10 @@ case class SetTransaction( appId: String, version: Long, @JsonDeserialize(contentAs = classOf[java.lang.Long]) - lastUpdated: Option[Long]) extends Action { + lastUpdated: Option[Long]) extends Action with AMTCheckpointAction { override def wrap: SingleAction = SingleAction(txn = this) + override def wrapAsAMTCheckpointSingleAction: AMTCheckpointSingleAction = + AMTCheckpointSingleAction(txn = this) } /** @@ -694,8 +707,15 @@ case class SetTransaction( case class DomainMetadata( domain: String, configuration: String, - removed: Boolean) extends Action with StorageAbstractDomainMetadata { + removed: Boolean) + extends Action + with AMTCheckpointAction + with StorageAbstractDomainMetadata { + override def wrap: SingleAction = SingleAction(domainMetadata = this) + override def wrapAsAMTCheckpointSingleAction: AMTCheckpointSingleAction = + AMTCheckpointSingleAction(domainMetadata = this) + override def getDomain: String = domain override def getConfiguration: String = configuration override def isRemoved: Boolean = removed @@ -1342,7 +1362,10 @@ case class Metadata( configuration: Map[String, String] = Map.empty, @JsonDeserialize(contentAs = classOf[java.lang.Long]) createdTime: Option[Long] = None) - extends Action with SparkAbstractMetadata with StorageAbstractMetadata { + extends Action + with AMTCheckpointAction + with SparkAbstractMetadata + with StorageAbstractMetadata { // The `schema` and `partitionSchema` methods should be vals or lazy vals, NOT // defs, because parsing StructTypes from JSON is extremely expensive and has @@ -1434,6 +1457,9 @@ case class Metadata( override def wrap: SingleAction = SingleAction(metaData = this) + override def wrapAsAMTCheckpointSingleAction: AMTCheckpointSingleAction = + AMTCheckpointSingleAction(metaData = this) + override def getId: String = id override def getName: String = name @@ -1738,7 +1764,10 @@ case class ContentRoot( path: String, sizeInBytes: Long, version: Long, - tags: Map[String, String] = null) { + tags: Map[String, String] = null) extends AMTCheckpointAction { + + override def wrapAsAMTCheckpointSingleAction: AMTCheckpointSingleAction = + AMTCheckpointSingleAction(contentRoot = this) private def tag(key: ContentRoot.Tags.KeyType): Option[String] = Option(tags).flatMap(_.get(key.name)) @@ -1808,7 +1837,7 @@ object ContentRoot { } /** - * Closed set of values for [[SidecarFile.sidecarType]] under the `adaptiveMetadata-preview` + * Closed set of values for the `type` field on [[SidecarFile]] under `adaptiveMetadata-preview`. * feature. Tells a reader which slice of the checkpoint's non-content metadata a sidecar * carries. * @@ -1834,48 +1863,130 @@ object SidecarType { /** * Top-level Delta action emitted by a commit which also writes AMT. - * Embeds the full checkpoint state -- `contentRoot` plus the non-content metadata snapshot - * (protocol, metadata, domain metadata, txns, sidecars) -- that a reader needs to - * reconstruct the table at the recorded version without replaying earlier commits. Distinct - * from [[CheckpointMetadata]], which describes V2 checkpoint sidecars. + * Contains the actions needed to reconstruct the table at the version recorded by + * [[CheckpointMetadata]]: the content root, protocol, metadata, domain metadata, transaction + * identifiers, and sidecar references. Entries can appear in any order. + * + * Exactly one [[CheckpointMetadata]], [[ContentRoot]], [[Protocol]], and [[Metadata]] must be + * present. The checkpoint version may lag the enclosing commit, and the content root version + * must be less than or equal to the checkpoint version. * * For domain metadata and transaction identifiers, the data can be carried inline, in * sidecars, or split across both (e.g., small changes inline with the bulk in a sidecar). * A given entry must not be duplicated across inline and sidecar. * - * @param version version at which this checkpoint is valid. May belong to a previous - * commit (the checkpoint can lag the enclosing commit). - * @param contentRoot pointer to the Iceberg v4 root manifest. `contentRoot.version` is - * the table version the tree reflects. - * @param protocol protocol snapshot at `version`. Must be non null. - * @param metaData metadata snapshot at `version`. Must be non null. - * @param domainMetadata all [[DomainMetadata]] entries carried inline. An empty list means - * there are no domain metadata entries inline (they may still be - * carried via sidecars). - * @param txns transaction identifiers carried inline. An empty list means there are - * no transaction identifiers inline (they may still be carried via - * sidecars). - * @param sidecars sidecars that carry the long tail of non-content metadata (transaction - * ids, domain metadata, ...). Empty when all non-content metadata is - * inline. + * @param actions wrapped actions comprising this checkpoint. */ +@JsonDeserialize(using = classOf[Checkpoint.Deserializer]) case class Checkpoint( - version: Long, - contentRoot: ContentRoot, - protocol: Protocol, - metaData: Metadata, - domainMetadata: Seq[DomainMetadata], - txns: Seq[SetTransaction], - sidecars: Seq[SidecarFile]) extends Action { + // The AMT RFC requires a bare action array. JsonValue on the getter makes Jackson serialize + // this array directly, without an {"actions": ...} wrapper. + @(JsonValue @getter) + actions: Seq[AMTCheckpointSingleAction]) extends Action { + + require(actions != null, "Checkpoint actions must not be null.") + require(actions.forall(_ != null), "Checkpoint actions must not contain null entries.") + + @JsonIgnore + private val unwrappedActions: Seq[AMTCheckpointAction] = actions.map(_.unwrap) + + @JsonIgnore + val checkpointMetadata: CheckpointMetadata = getRequiredAction[CheckpointMetadata] + + /** + * Version at which this checkpoint is valid. May belong to a previous commit (the checkpoint + * can lag the enclosing commit). + */ + @JsonIgnore + def version: Long = checkpointMetadata.version + + /** + * Pointer to the Iceberg v4 root manifest. `contentRoot.version` is the table version the tree + * reflects. + */ + @JsonIgnore + val contentRoot: ContentRoot = getRequiredAction[ContentRoot] + + /** Protocol snapshot at `version`. Must be non null. */ + @JsonIgnore + val protocol: Protocol = getRequiredAction[Protocol] + + /** Metadata snapshot at `version`. Must be non null. */ + @JsonIgnore + val metaData: Metadata = getRequiredAction[Metadata] + + /** + * All [[DomainMetadata]] entries carried inline. An empty list means there are no domain + * metadata entries inline (they may still be carried via sidecars). + */ + @JsonIgnore + lazy val domainMetadata: Seq[DomainMetadata] = unwrappedActions.collect { + case action: DomainMetadata => action + } + + /** + * Transaction identifiers carried inline. An empty list means there are no transaction + * identifiers inline (they may still be carried via sidecars). + */ + @JsonIgnore + lazy val txns: Seq[SetTransaction] = unwrappedActions.collect { + case action: SetTransaction => action + } + + /** + * Sidecars that carry the long tail of non-content metadata (transaction ids, domain metadata, + * ...). Empty when all non-content metadata is inline. + */ + @JsonIgnore + lazy val sidecars: Seq[SidecarFile] = unwrappedActions.collect { + case action: SidecarFile => action + } // AMT checkpoint sidecars must always declare their type. - require(sidecars.forall(_.sidecarType.isDefined), - "All sidecars in a Checkpoint must have a sidecarType.") + require(sidecars.forall(_.`type`.isDefined), + "All sidecars in a Checkpoint must have a type.") require( contentRoot.version <= version, s"contentRoot.version (${contentRoot.version}) must be <= checkpoint version ($version).") - override def wrap: SingleAction = SingleAction(checkpoint = this) + override def wrap: SingleAction = SingleAction(checkpoint = actions) + + private def getRequiredAction[T <: AMTCheckpointAction : ClassTag]: T = { + // ClassTag makes this pattern check T's runtime class instead of its erased upper bound. + val matchedActions = unwrappedActions.collect { case action: T => action } + lazy val actionName = implicitly[ClassTag[T]].runtimeClass.getSimpleName + require(matchedActions.size == 1, + s"A Checkpoint must contain exactly one $actionName action, found ${matchedActions.size}.") + matchedActions.head + } +} + +object Checkpoint { + /** Builds a checkpoint from its individual fields. */ + def apply( + version: Long, + contentRoot: ContentRoot, + protocol: Protocol, + metaData: Metadata, + domainMetadata: Seq[DomainMetadata], + txns: Seq[SetTransaction], + sidecars: Seq[SidecarFile]): Checkpoint = { + val flattenedActions = Seq(CheckpointMetadata(version), contentRoot, protocol, metaData) ++ + domainMetadata ++ txns ++ sidecars + fromActions(flattenedActions) + } + + /** Wraps the actions comprising a checkpoint, preserving their order. */ + def fromActions(actions: Seq[AMTCheckpointAction]): Checkpoint = + Checkpoint(actions.map(_.wrapAsAMTCheckpointSingleAction)) + + /** Reads the bare action array used when a Checkpoint is serialized directly. */ + class Deserializer extends JsonDeserializer[Checkpoint] { + override def deserialize(p: JsonParser, ctxt: DeserializationContext): Checkpoint = { + val actions = ctxt.readValue(p, classOf[Array[AMTCheckpointSingleAction]]) + Checkpoint(actions.toSeq) + } + } } /** @@ -1905,17 +2016,19 @@ case class SidecarFile( modificationTime: Long, tags: Map[String, String] = null, // Applicable only for AMT checkpoint sidecars. - // Sidecars corresponding to V2Checkpoints do not have concept of sidecarType. - @JsonProperty("type") + // Sidecars corresponding to V2Checkpoints do not have concept of type. @JsonInclude(Include.NON_ABSENT) - sidecarType: Option[String] = None) - extends CheckpointOnlyAction { + `type`: Option[String] = None) + extends CheckpointOnlyAction with AMTCheckpointAction { - // Either no sidecarType is supplied, or it must be one of the known [[SidecarType]] values. - sidecarType.foreach(SidecarType.validate) + // Either no type is supplied, or it must be one of the known [[SidecarType]] values. + `type`.foreach(SidecarType.validate) override def wrap: SingleAction = SingleAction(sidecar = this) + override def wrapAsAMTCheckpointSingleAction: AMTCheckpointSingleAction = + AMTCheckpointSingleAction(sidecar = this) + def toFileStatus(logPath: Path): FileStatus = { val partFilePath = new Path(FileNames.sidecarDirPath(logPath), path) new FileStatus(sizeInBytes, false, 0, 0, modificationTime, partFilePath) @@ -1933,7 +2046,8 @@ object SidecarFile { } /** - * Holds information about the Delta Checkpoint. This action will only be part of checkpoints. + * Holds information about the Delta Checkpoint. This action will only be part of checkpoints, + * including an AMT [[Checkpoint]] * * @param version version of the checkpoint * @param tags attributes of the checkpoint, defaults to null (which is semantically same as an @@ -1943,10 +2057,13 @@ object SidecarFile { case class CheckpointMetadata( version: Long, tags: Map[String, String] = null) - extends CheckpointOnlyAction { + extends CheckpointOnlyAction with AMTCheckpointAction { override def wrap: SingleAction = SingleAction(checkpointMetadata = this) + override def wrapAsAMTCheckpointSingleAction: AMTCheckpointSingleAction = + AMTCheckpointSingleAction(checkpointMetadata = this) + import CheckpointMetadata.Tags /** Number of actions in the [[SidecarFile]]s */ @@ -2014,7 +2131,9 @@ case class SingleAction( sidecar: SidecarFile = null, domainMetadata: DomainMetadata = null, commitInfo: CommitInfo = null, - checkpoint: Checkpoint = null) { + // Use a Seq so Spark infers the action array required by the AMT RFC. Spark ignores JsonValue; + // using Checkpoint here would infer a struct with an extra "actions" field. + checkpoint: Seq[AMTCheckpointSingleAction] = null) { def unwrap: Action = { if (add != null) { @@ -2038,7 +2157,7 @@ case class SingleAction( } else if (commitInfo != null) { commitInfo } else if (checkpoint != null) { - checkpoint + Checkpoint(checkpoint) } else { null } @@ -2062,6 +2181,38 @@ object SingleAction extends Logging { Column(Literal(null, ScalaReflection.schemaFor[Metadata].dataType)) } +/** A serialization helper for a single action within an AMT [[Checkpoint]]. */ +case class AMTCheckpointSingleAction( + checkpointMetadata: CheckpointMetadata = null, + contentRoot: ContentRoot = null, + protocol: Protocol = null, + metaData: Metadata = null, + domainMetadata: DomainMetadata = null, + txn: SetTransaction = null, + sidecar: SidecarFile = null) { + + /** Returns the enclosed action, or null when no recognized action is present. */ + def unwrap: AMTCheckpointAction = { + if (checkpointMetadata != null) { + checkpointMetadata + } else if (contentRoot != null) { + contentRoot + } else if (protocol != null) { + protocol + } else if (metaData != null) { + metaData + } else if (domainMetadata != null) { + domainMetadata + } else if (txn != null) { + txn + } else if (sidecar != null) { + sidecar + } else { + null + } + } +} + /** Serializes Maps containing JSON strings without extra escaping. */ class JsonMapSerializer extends JsonSerializer[Map[String, String]] { def serialize( diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/ActionSerializerSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/ActionSerializerSuite.scala index 558a3a98e60..af3d4089ab5 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/ActionSerializerSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/ActionSerializerSuite.scala @@ -33,7 +33,7 @@ import org.apache.hadoop.fs.Path import org.apache.spark.sql.{QueryTest, SaveMode} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.test.SharedSparkSession -import org.apache.spark.sql.types.StructType +import org.apache.spark.sql.types.{ArrayType, StructType} import org.apache.spark.util.Utils // scalastyle:off: removeFile @@ -717,6 +717,69 @@ class ActionSerializerSuite extends QueryTest with SharedSparkSession with Delta } } + test("Checkpoint - Spark log schema stores an array of action structs") { + val elementSchema = Action.logSchema("checkpoint").dataType match { + case ArrayType(schema: StructType, _) => schema + case other => fail(s"Expected an array of checkpoint action structs, found $other") + } + // Compare Spark's inferred array element schema with AMTCheckpointSingleAction's fields. + // Keep the expected names explicit to catch accidental changes to the schema. + assert(elementSchema.fieldNames.toSeq === Seq( + "checkpointMetadata", + "contentRoot", + "protocol", + "metaData", + "domainMetadata", + "txn", + "sidecar")) + assert(elementSchema.fields.forall(_.dataType.isInstanceOf[StructType])) + } + + // Snapshot replay can decode trailing commit JSON with Spark and call `SingleAction.unwrap`. + // A deferred Checkpoint must pass validation before replay ignores its pointer. + // Spark ignores Jackson's annotations and custom deserializer, so Jackson round-trip tests + // do not cover this path or guarantee that required sidecar types survive decoding. + Seq(false, true).foreach { withSidecars => + test(s"Checkpoint - Spark decodes the action array with sidecars=$withSidecars") { + import testImplicits._ + + val sidecars = if (withSidecars) { + Seq( + SidecarFile( + path = "domain-metadata.parquet", + sizeInBytes = 100L, + modificationTime = 0L, + `type` = Some(SidecarType.Type.DomainMetadata)), + SidecarFile( + path = "transactions.parquet", + sizeInBytes = 2048L, + modificationTime = 0L, + `type` = Some(SidecarType.Type.Txn))) + } else { + Seq.empty + } + val checkpoint = Checkpoint( + version = 1L, + contentRoot = ContentRoot("metadata/root.parquet", sizeInBytes = 4096L, version = 1L), + protocol = Protocol(minReaderVersion = 3, minWriterVersion = 7), + metaData = Metadata(id = "metadata-id"), + domainMetadata = Seq( + DomainMetadata("domain-1", "{}", removed = false), + DomainMetadata("domain-2", "{}", removed = false)), + txns = Seq( + SetTransaction("app-1", 7L, Some(100L)), + SetTransaction("app-2", 8L, None)), + sidecars = sidecars) + val decoded = spark.read.schema(Action.logSchema) + .option("mode", "FAILFAST") + .json(Seq(checkpoint.json).toDS()) + .as[SingleAction](SingleAction.encoder) + .collect().toSeq + assert(decoded === Seq(checkpoint.wrap)) + assert(decoded.map(_.unwrap) === Seq(checkpoint)) + } + } + testActionSerDe( "AddCDCFile (without tags) - json serialization/deserialization", AddCDCFile("part=p1/f1", partitionValues = Map("x" -> "2"), size = 10), diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/LastCheckpointInfoSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/LastCheckpointInfoSuite.scala index 1550cd6f6fe..a6321efbb41 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/LastCheckpointInfoSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/LastCheckpointInfoSuite.scala @@ -17,7 +17,7 @@ package org.apache.spark.sql.delta // scalastyle:off import.ordering.noEmptyLine -import org.apache.spark.sql.delta.actions.{Checkpoint, ContentRoot, Metadata, Protocol} +import org.apache.spark.sql.delta.actions.{Checkpoint, ContentRoot, DomainMetadata, Metadata, Protocol, SetTransaction, SidecarFile, SidecarType} import org.apache.spark.sql.delta.amt.{AMTLeafComparisons, AMTSingleAction, AMTWriteResult, DataManifestEntry, ManifestInfo, SingleAMTWriteMetrics, Tracking} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest @@ -236,7 +236,7 @@ class LastCheckpointInfoSuite extends SharedSparkSession === LastCheckpointInfo.serializeToJson(ci2, addChecksum = true)) } - /** A [[Checkpoint]] action describing `version`, with a root manifest at `rootPath`. */ + /** A [[Checkpoint]] at `version`, with a root at `rootPath` and every supported action type. */ private def sampleCheckpointAction( version: Long = 1L, rootPath: String = "metadata/root-abc.parquet", @@ -245,9 +245,23 @@ class LastCheckpointInfoSuite extends SharedSparkSession contentRoot = ContentRoot(path = rootPath, sizeInBytes = rootSizeInBytes, version = version), protocol = Protocol(minReaderVersion = 3, minWriterVersion = 7), metaData = Metadata(id = "metadata-id", name = "t"), - domainMetadata = Nil, - txns = Nil, - sidecars = Nil) + domainMetadata = Seq( + DomainMetadata(domain = "user.tag", configuration = """{"k":"v"}""", removed = false), + DomainMetadata(domain = "user.other", configuration = "{}", removed = false)), + txns = Seq( + SetTransaction(appId = "app-1", version = 7L, lastUpdated = Some(100L)), + SetTransaction(appId = "app-2", version = 8L, lastUpdated = None)), + sidecars = Seq( + SidecarFile( + path = "domain-metadata.parquet", + sizeInBytes = 1024L, + modificationTime = 100L, + `type` = Some(SidecarType.Type.DomainMetadata)), + SidecarFile( + path = "transactions.parquet", + sizeInBytes = 2048L, + modificationTime = 200L, + `type` = Some(SidecarType.Type.Txn)))) private def sampleAMTCheckpoint( version: Long = 1L, @@ -431,6 +445,8 @@ class LastCheckpointInfoSuite extends SharedSparkSession checkpointType = Some(LastCheckpointInfo.CheckpointType.AMT), amtCheckpoint = Some(sampleAMTCheckpoint())) val json = LastCheckpointInfo.serializeToJson(ci, addChecksum = true) + val checkpointJson = JsonUtils.mapper.readTree(json).path("amtCheckpoint").path("checkpoint") + assert(checkpointJson.isArray) val deserialized = LastCheckpointInfo.deserializeFromJson(json, validate = true) val amt = deserialized.amtCheckpoint.getOrElse(fail("round-trip dropped the amtCheckpoint.")) val expectedAmt = ci.amtCheckpoint.get diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTCheckpointTestBase.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTCheckpointTestBase.scala index ce4bd1218cd..8cc7e13b0d6 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTCheckpointTestBase.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTCheckpointTestBase.scala @@ -23,7 +23,7 @@ import scala.collection.immutable.ListMap // scalastyle:off import.ordering.noEmptyLine import com.databricks.spark.util.{Log4jUsageLogger, MetricDefinitions} import org.apache.spark.sql.delta.{AdaptiveMetadataTableFeature, CommitStats, DeltaLog, DeltaOperations, Snapshot} -import org.apache.spark.sql.delta.actions.{Action, AddFile, Checkpoint, ContentRoot, Metadata, Protocol, RemoveFile} +import org.apache.spark.sql.delta.actions.{Action, AddFile, AMTCheckpointAction, Checkpoint, ContentRoot, Metadata, Protocol, RemoveFile} import org.apache.spark.sql.delta.actions.TableFeatureProtocolUtils._ import org.apache.spark.sql.delta.coordinatedcommits.CatalogOwnedTestBaseSuite import org.apache.spark.sql.delta.sources.DeltaSQLConf @@ -96,6 +96,18 @@ trait AMTCheckpointTestBase tableRoot = new Path("file:/fake-table-root")) } + /** Updates matching checkpoint actions, preserving entry order and unmatched envelopes. */ + protected def updateCheckpointActions( + checkpoint: Checkpoint)( + update: PartialFunction[AMTCheckpointAction, AMTCheckpointAction]): Checkpoint = { + val getUpdatedActionOpt: AMTCheckpointAction => Option[AMTCheckpointAction] = update.lift + checkpoint.copy(actions = checkpoint.actions.map { wrappedAMTCheckpointSingleAction => + getUpdatedActionOpt(wrappedAMTCheckpointSingleAction.unwrap) + .map(_.wrapAsAMTCheckpointSingleAction) + .getOrElse(wrappedAMTCheckpointSingleAction) + }) + } + /** * Runs `body` on the DATA (content_type=0) entry rows across `paths`, read straight off disk * (pass a provider's manifest paths -- its root and/or live leaves). The read runs with the diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTCheckpointWriteSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTCheckpointWriteSuite.scala index 44aa92d6019..c7613d33d87 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTCheckpointWriteSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTCheckpointWriteSuite.scala @@ -20,7 +20,7 @@ import java.io.File import com.databricks.spark.util.{Log4jUsageLogger, MetricDefinitions, UsageRecord} import org.apache.spark.sql.delta.{Checkpoints, CommitStats, DeltaOperations, LastCheckpointInfo, RowId} -import org.apache.spark.sql.delta.actions.{AddFile, Checkpoint, ContentRoot, DomainMetadata, SetTransaction} +import org.apache.spark.sql.delta.actions.{Action, AddFile, Checkpoint, ContentRoot, DomainMetadata, SetTransaction, SingleAction} import org.apache.spark.sql.delta.coordinatedcommits.TrackingInMemoryCommitCoordinatorBuilder import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util.{DeltaCommitFileProvider, FileNames, JsonUtils} @@ -96,6 +96,44 @@ class AMTCheckpointWriteSuite extends AMTCheckpointTestBase { } } + testAcrossAMTCheckpointScenarios( + "manifest commit checkpoint round-trips through Jackson and Spark", + "amt_manifest_commit_serde")( + setup = name => sql(s"INSERT INTO $name VALUES (1)"), + inlineCheckpointTriggerActionsOrSQL = Some { _ => + Left((Seq( + DomainMetadata("serialization-test", """{"value":1}""", removed = false), + SetTransaction("serialization-app", 7L, lastUpdated = None)), + DeltaOperations.ManualUpdate)) + }) { context => + val expectedCheckpoint = context.checkpoint + assert(expectedCheckpoint.domainMetadata.contains( + DomainMetadata("serialization-test", """{"value":1}""", removed = false))) + assert(expectedCheckpoint.txns == Seq( + SetTransaction("serialization-app", 7L, lastUpdated = None))) + + val snapshot = context.postCheckpointSnapshot + val deltaLog = snapshot.deltaLog + // A deferred checkpoint is published in the commit after the version it describes. + val commitPath = DeltaCommitFileProvider(snapshot).deltaFile(context.manifestCommitVersion) + val jacksonCheckpoints = deltaLog.store + .read(commitPath, deltaLog.newDeltaHadoopConf()) + .map(Action.fromJson) + .collect { case checkpoint: Checkpoint => checkpoint } + val sparkCheckpoints = allowReadWithinDeltaLog { + spark.read.schema(Action.logSchema) + .option("mode", "FAILFAST") + .json(commitPath.toString) + .as[SingleAction](SingleAction.encoder) + .collect().toSeq + .map(_.unwrap) + .collect { case checkpoint: Checkpoint => checkpoint } + } + + assert(jacksonCheckpoints == Seq(expectedCheckpoint)) + assert(sparkCheckpoints == Seq(expectedCheckpoint)) + } + testAcrossAMTCheckpointScenarios( "checkpoint backfills preceding commits while its manifest commit remains staged", "amt_checkpoint_backfill")( @@ -416,8 +454,10 @@ class AMTCheckpointWriteSuite extends AMTCheckpointTestBase { file_size_in_bytes = leafSize, manifest_info = emptyManifestInfo.copy(added_files_count = 1)) val (rootLoc, rootSize) = writeManifest("root with space.parquet", Seq(leafPointer.wrap)) - val checkpoint = base.copy( - contentRoot = ContentRoot(path = rootLoc, sizeInBytes = rootSize, version = base.version)) + val checkpoint = updateCheckpointActions(base) { + case _: ContentRoot => + ContentRoot(path = rootLoc, sizeInBytes = rootSize, version = base.version) + } // The synthesized pointers really do carry spaces and are not URL-encoded. assert(rootLoc.contains("root with space.parquet") && !rootLoc.contains("%20")) diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTFieldSpecSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTFieldSpecSuite.scala index f4ba3f5331e..18f4773d575 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTFieldSpecSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTFieldSpecSuite.scala @@ -22,7 +22,7 @@ import scala.collection.JavaConverters._ import org.apache.spark.sql.delta.DeltaColumnMapping import org.apache.spark.sql.delta.DeltaLog -import org.apache.spark.sql.delta.actions.Metadata +import org.apache.spark.sql.delta.actions.{ContentRoot, Metadata} import org.apache.spark.sql.delta.util.JsonUtils import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path @@ -89,7 +89,9 @@ class AMTFieldSpecSuite extends AMTCheckpointTestBase { provider: AMTCheckpointProvider, deltaLog: DeltaLog, metadata: Metadata): DataFrame = new AMTCheckpointProvider( manifestCommitVersion = provider.manifestCommitVersion, - checkpointAction = provider.checkpointAction.copy(metaData = metadata), + checkpointAction = updateCheckpointActions(provider.checkpointAction) { + case _: Metadata => metadata + }, leaves = provider.leaves, tableRoot = provider.tableRoot) .loadActionsForStateReconstruction(spark, deltaLog) @@ -502,10 +504,11 @@ class AMTFieldSpecSuite extends AMTCheckpointTestBase { // Reading the rewritten root through the production path recovers every leaf pointer // unchanged: `manifest_info` is resolved by its Iceberg field id, not its name. (If it // were not, the DATA_MANIFEST rows would unwrap with a missing `manifest_info`.) - val rewrittenCheckpoint = checkpoint.copy( - contentRoot = checkpoint.contentRoot.copy( + val rewrittenCheckpoint = updateCheckpointActions(checkpoint) { + case root: ContentRoot => root.copy( path = rewrittenFile.getAbsolutePath, - sizeInBytes = rewrittenFile.length())) + sizeInBytes = rewrittenFile.length()) + } val rewrittenProvider = AMTCheckpointProvider.fromCheckpoint( deltaLog, rewrittenCheckpoint, provider.manifestCommitVersion) assertLeavesEqual(rewrittenProvider.leaves, provider.leaves) diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTInheritanceReadSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTInheritanceReadSuite.scala index a5f9f5ce048..c33b269fa3a 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTInheritanceReadSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTInheritanceReadSuite.scala @@ -90,9 +90,10 @@ class AMTInheritanceReadSuite extends AMTCheckpointTestBase { val metadataDir = FileNames.amtMetadataDirPath(deltaLog.dataPath) val (location, sizeInBytes) = writeManifest(FileNames.newAMTRootManifestFile(metadataDir), rootEntries) - val checkpoint = - base.copy(contentRoot = - ContentRoot(path = location, sizeInBytes = sizeInBytes, version = base.version)) + val checkpoint = updateCheckpointActions(base) { + case _: ContentRoot => + ContentRoot(path = location, sizeInBytes = sizeInBytes, version = base.version) + } val provider = AMTCheckpointProvider.fromCheckpoint( deltaLog, checkpoint, manifestCommitVersion = checkpoint.version) provider.loadActionsForStateReconstruction(spark, deltaLog) diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTSnapshotSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTSnapshotSuite.scala index 29871bc409d..f07057b5125 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTSnapshotSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTSnapshotSuite.scala @@ -1391,7 +1391,9 @@ class AMTSnapshotSuite extends AMTCheckpointTestBase with DeletionVectorsTestUti outputSchema = Some(AMTSingleAction.persistedSchema(metadata, protocol)), format = new AMTParquetFileFormat) val size = rootFile.getFileSystem(hadoopConf).getFileStatus(rootFile).getLen - base.copy(contentRoot = ContentRoot( - path = rootFile.toString, sizeInBytes = size, version = base.version)) + updateCheckpointActions(base) { + case _: ContentRoot => + ContentRoot(path = rootFile.toString, sizeInBytes = size, version = base.version) + } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTWriterManagerSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTWriterManagerSuite.scala index 386e09f50a8..f5bcfe58380 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTWriterManagerSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTWriterManagerSuite.scala @@ -18,7 +18,7 @@ package org.apache.spark.sql.delta.amt import com.databricks.spark.util.Log4jUsageLogger import org.apache.spark.sql.delta.{ConcurrentAMTCheckpointLandedException, CurrentTransactionInfo, DeltaIllegalStateException, DeltaOperations, FullAMTWriteFailedWithConflict, LogSegment, Snapshot, SnapshotManagement, WinningCommitMetrics, WinningCommitSummary} -import org.apache.spark.sql.delta.actions.{Action, AddFile, BackReference, Checkpoint, ContentRoot, RemoveFile} +import org.apache.spark.sql.delta.actions.{Action, AddFile, BackReference, Checkpoint, CheckpointMetadata, ContentRoot, RemoveFile} import org.apache.spark.sql.delta.util.{FileNames, JsonUtils} import org.apache.hadoop.fs.{FileStatus, Path} @@ -202,7 +202,9 @@ class AMTWriterManagerSuite extends AMTCheckpointTestBase { // longer hard-fails on this -- it writes no tree; re-deriving the file actions' back // references against the winner tree happens in doCommit's rebaseBackReferences, exercised // end-to-end in AMTConflictResolutionSuite. - val winnerTree = baseTree.copy(version = baseTree.version + 1) + val winnerTree = updateCheckpointActions(baseTree) { + case metadata: CheckpointMetadata => metadata.copy(version = baseTree.version + 1) + } val retrySegment = advanceSegmentByOneCommit(snapshot.logSegment) val result = manager.writeAMT( nextAttemptVersion = snapshot.version + 2, @@ -238,7 +240,9 @@ class AMTWriterManagerSuite extends AMTCheckpointTestBase { // A concurrent winner installed its own full-rewrite tree at the target version. That tree // already provides an up-to-date AMT, so this losing full checkpoint is redundant and // skips. - val fullWinner = fullResult.checkpoint.copy(version = snapshot.version + 1) + val fullWinner = updateCheckpointActions(fullResult.checkpoint) { + case metadata: CheckpointMetadata => metadata.copy(version = snapshot.version + 1) + } val fullWinnerMetric = anyWinner.copy(checkpointAction = Some(fullWinner)) val retrySegment = advanceSegmentByOneCommit(snapshot.logSegment) val ex = intercept[ConcurrentAMTCheckpointLandedException] { @@ -283,7 +287,9 @@ class AMTWriterManagerSuite extends AMTCheckpointTestBase { // A concurrent winner installed a newer tree. Any winner tree supersedes a losing // incremental checkpoint, so it is redundant and skips. - val winnerTree = baseTree.copy(version = baseTree.version + 1) + val winnerTree = updateCheckpointActions(baseTree) { + case metadata: CheckpointMetadata => metadata.copy(version = baseTree.version + 1) + } val retrySegment = advanceSegmentByOneCommit(snapshot.logSegment) intercept[ConcurrentAMTCheckpointLandedException] { manager.writeAMT( @@ -324,8 +330,10 @@ class AMTWriterManagerSuite extends AMTCheckpointTestBase { isIncremental = true, lastManifestCommitWithFullRewrite = 0L, numLeaves = fullResult.checkpoint.contentRoot.numLeaves.getOrElse(0L)) - val incrementalWinner = - fullResult.checkpoint.copy(version = snapshot.version + 1, contentRoot = incrementalRoot) + val incrementalWinner = updateCheckpointActions(fullResult.checkpoint) { + case metadata: CheckpointMetadata => metadata.copy(version = snapshot.version + 1) + case _: ContentRoot => incrementalRoot + } val retrySegment = advanceSegmentByOneCommit(snapshot.logSegment) val ex = intercept[FullAMTWriteFailedWithConflict] { manager.writeAMT( diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AdaptiveMetadataActionsSerializerSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AdaptiveMetadataActionsSerializerSuite.scala index 06db55368d1..a1a9a4ccdc9 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AdaptiveMetadataActionsSerializerSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AdaptiveMetadataActionsSerializerSuite.scala @@ -24,7 +24,7 @@ import org.apache.spark.SparkFunSuite /** * JSON round-trip and invariant tests for the action-schema additions introduced for the * `adaptiveMetadata-preview` feature: the new top-level [[Checkpoint]] action, the helper - * case classes [[ContentRoot]] / [[SidecarType]], and the optional `SidecarFile.sidecarType` + * case classes [[ContentRoot]] / [[SidecarType]], and the optional `SidecarFile.type` * field. */ class AdaptiveMetadataActionsSerializerSuite extends SparkFunSuite { @@ -36,20 +36,44 @@ class AdaptiveMetadataActionsSerializerSuite extends SparkFunSuite { private val sampleProtocol = Protocol(minReaderVersion = 3, minWriterVersion = 7) private val sampleMetadata = Metadata(id = "metadata-id", name = "t") + private val sampleCheckpointMetadata = CheckpointMetadata( + version = 1L, + tags = Map("checkpoint-tag" -> "value")) + private val sampleDomainMetadatas = Seq( + DomainMetadata("user.tag", "{}", removed = false), + DomainMetadata("delta.rowTracking", "{}", removed = false)) + private val sampleTxns = Seq( + SetTransaction("app-1", 7L, Some(100L)), + SetTransaction("app-2", 8L, None)) + private val sampleSidecars = Seq( + dmSidecar(), + txnSidecar()) + + private def sampleRequiredActions: Seq[AMTCheckpointAction] = + Seq(sampleCheckpointMetadata, sampleRoot, sampleProtocol, sampleMetadata) + + private def sampleFullActions: Seq[AMTCheckpointAction] = + sampleRequiredActions ++ sampleDomainMetadatas ++ sampleTxns ++ sampleSidecars + + private def sampleRequiredCheckpoint: Checkpoint = + Checkpoint.fromActions(sampleRequiredActions) + + private def sampleFullCheckpoint: Checkpoint = + Checkpoint.fromActions(sampleFullActions) private def dmSidecar(path: String = "dm.parquet"): SidecarFile = SidecarFile( path = path, sizeInBytes = 1L, modificationTime = 0L, - sidecarType = Some(SidecarType.Type.DomainMetadata)) + `type` = Some(SidecarType.Type.DomainMetadata)) private def txnSidecar(path: String = "txn.parquet"): SidecarFile = SidecarFile( path = path, sizeInBytes = 1L, modificationTime = 0L, - sidecarType = Some(SidecarType.Type.Txn)) + `type` = Some(SidecarType.Type.Txn)) // ============================================================================================ // ContentRoot @@ -134,14 +158,14 @@ class AdaptiveMetadataActionsSerializerSuite extends SparkFunSuite { } } - test("SidecarFile.sidecarType: each SidecarType value round-trips through JSON") { + test("SidecarFile.type: each SidecarType value round-trips through JSON") { Seq(SidecarType.Type.DomainMetadata, SidecarType.Type.Txn) .foreach { value => val sf = SidecarFile( path = s"$value.parquet", sizeInBytes = 1L, modificationTime = 0L, - sidecarType = Some(value)) + `type` = Some(value)) assert(sf.json === s"""{"sidecar":{"path":"$value.parquet","sizeInBytes":1,""" + s""""modificationTime":0,"type":"$value"}}""") @@ -156,11 +180,11 @@ class AdaptiveMetadataActionsSerializerSuite extends SparkFunSuite { |}""".stripMargin) val parsed = Action.fromJson(sf.json).asInstanceOf[SidecarFile] assert(parsed === sf) - assert(parsed.sidecarType.contains(value)) + assert(parsed.`type`.contains(value)) } } - test("SidecarFile.sidecarType: absent from JSON when None") { + test("SidecarFile.type: absent from JSON when None") { val sf = SidecarFile(path = "s.parquet", sizeInBytes = 1L, modificationTime = 0L) assert(sf.json === """{"sidecar":{"path":"s.parquet","sizeInBytes":1,"modificationTime":0}}""") assert(JsonUtils.toPrettyJson(sf.wrap) === @@ -174,27 +198,52 @@ class AdaptiveMetadataActionsSerializerSuite extends SparkFunSuite { assert(Action.fromJson(sf.json) === sf) } - test("SidecarFile rejects unknown sidecarType values at construction") { + test("SidecarFile rejects unknown type values at construction") { Seq("systemDomainMetadata", "bogus", "").foreach { bad => val ex = intercept[IllegalArgumentException] { SidecarFile( path = "x.parquet", sizeInBytes = 1L, modificationTime = 0L, - sidecarType = Some(bad)) + `type` = Some(bad)) } assert(ex.getMessage.toLowerCase(java.util.Locale.ROOT).contains("unknown")) } } - test("SidecarFile accepts None sidecarType and known SidecarType values") { + test("SidecarFile accepts None type and known SidecarType values") { SidecarFile(path = "x.parquet", sizeInBytes = 1L, modificationTime = 0L) Seq(SidecarType.Type.DomainMetadata, SidecarType.Type.Txn).foreach { value => SidecarFile( path = "x.parquet", sizeInBytes = 1L, modificationTime = 0L, - sidecarType = Some(value)) + `type` = Some(value)) + } + } + + // ============================================================================================ + // AMTCheckpointSingleAction + // ============================================================================================ + + Seq[(String, AMTCheckpointAction)]( + "checkpointMetadata" -> sampleCheckpointMetadata, + "contentRoot" -> sampleRoot, + "protocol" -> sampleProtocol, + "metaData" -> sampleMetadata, + "domainMetadata" -> sampleDomainMetadatas.head, + "txn" -> sampleTxns.head, + "sidecar" -> sampleSidecars.head + ).foreach { case (fieldName, action) => + test(s"AMTCheckpointSingleAction: $fieldName wrap, unwrap, and JSON round-trip") { + val wrapped = action.wrapAsAMTCheckpointSingleAction + assert(wrapped.unwrap === action) + val json = JsonUtils.toJson(wrapped) + val node = JsonUtils.mapper.readTree(json) + assert(node.size() === 1) + assert(node.has(fieldName)) + assert(node.get(fieldName) === JsonUtils.mapper.readTree(JsonUtils.toJson(action))) + assert(JsonUtils.fromJson[AMTCheckpointSingleAction](json) === wrapped) } } @@ -213,19 +262,24 @@ class AdaptiveMetadataActionsSerializerSuite extends SparkFunSuite { sidecars = Seq.empty) assert(JsonUtils.toPrettyJson(cp.wrap) === """{ - | "checkpoint" : { - | "version" : 1, + | "checkpoint" : [ { + | "checkpointMetadata" : { + | "version" : 1 + | } + | }, { | "contentRoot" : { | "path" : "metadata/root-abc.parquet", | "sizeInBytes" : 4096, | "version" : 1 - | }, + | } + | }, { | "protocol" : { | "minReaderVersion" : 3, | "minWriterVersion" : 7, | "readerFeatures" : [ ], | "writerFeatures" : [ ] - | }, + | } + | }, { | "metaData" : { | "id" : "metadata-id", | "name" : "t", @@ -235,34 +289,99 @@ class AdaptiveMetadataActionsSerializerSuite extends SparkFunSuite { | }, | "partitionColumns" : [ ], | "configuration" : { } - | }, - | "domainMetadata" : [ ], - | "txns" : [ ], - | "sidecars" : [ ] - | } + | } + | } ] |}""".stripMargin) val parsed = Action.fromJson(cp.json).asInstanceOf[Checkpoint] assert(parsed === cp) } test("Checkpoint: routes through SingleAction.unwrap") { - val cp = Checkpoint( - version = 1L, - contentRoot = sampleRoot, - protocol = sampleProtocol, - metaData = sampleMetadata, - domainMetadata = Seq.empty, - txns = Seq.empty, - sidecars = Seq.empty) - val wrapped = cp.wrap - assert(wrapped.checkpoint === cp) - assert(wrapped.unwrap === cp) + val checkpoint = sampleRequiredCheckpoint + val wrapped = checkpoint.wrap + assert(wrapped.checkpoint === checkpoint.actions) + assert(wrapped.unwrap === checkpoint) + } + + test("Checkpoint: ser-de round-trips as a bare action array") { + val checkpoint = sampleFullCheckpoint + val json = JsonUtils.toJson(checkpoint) + val node = JsonUtils.mapper.readTree(json) + assert(node.isArray) + assert(node === JsonUtils.mapper.readTree(JsonUtils.toJson(checkpoint.actions))) + assert(node === JsonUtils.mapper.readTree(checkpoint.json).get("checkpoint")) + val roundTripped = JsonUtils.fromJson[Checkpoint](json) + assert(roundTripped === checkpoint) + assert(roundTripped.checkpointMetadata.tags === sampleCheckpointMetadata.tags) + } + + Seq( + "{}", + """{"unknownAction":{"value":1}}""" + ).foreach { unknownJson => + test(s"Checkpoint: ignores an unrecognized envelope $unknownJson") { + // Unrecognized JSON envelope unwraps to null. + val unknownAction = JsonUtils.fromJson[AMTCheckpointSingleAction](unknownJson) + assert(unknownAction.unwrap === null) + + // Checkpoint JSON with unrecognized envelope deserializes to a checkpoint with the + // unrecognized action, with other actions intact. + val checkpoint = sampleFullCheckpoint + val actionsJson = JsonUtils.toJson(checkpoint.actions) + val mixedJson = actionsJson.dropRight(1) + s",$unknownJson]" + val expectedMixedCheckpoint = checkpoint.copy( + actions = checkpoint.actions :+ AMTCheckpointSingleAction()) + val deserialized = JsonUtils.fromJson[Checkpoint](mixedJson) + assert(deserialized === expectedMixedCheckpoint) + assert(deserialized.checkpointMetadata === sampleCheckpointMetadata) + assert(deserialized.contentRoot === sampleRoot) + assert(deserialized.protocol === sampleProtocol) + assert(deserialized.metaData === sampleMetadata) + assert(deserialized.domainMetadata === sampleDomainMetadatas) + assert(deserialized.txns === sampleTxns) + assert(deserialized.sidecars === sampleSidecars) + assert(Action.fromJson(s"""{"checkpoint":$mixedJson}""") === expectedMixedCheckpoint) + + // The required actions invariant is still maintained during deserialization. + val e = intercept[IllegalArgumentException] { + JsonUtils.fromJson[Checkpoint](s"[$unknownJson]") + } + assert(e.getMessage.contains("exactly one CheckpointMetadata action, found 0")) + } } // ============================================================================================ // Checkpoint: round-trip across field shapes // ============================================================================================ + test("Checkpoint: extracts reordered singletons and interleaved repeated actions") { + val reorderedActions = Seq[AMTCheckpointAction]( + sampleTxns.head, + sampleMetadata, + sampleDomainMetadatas.last, + sampleSidecars.last, + sampleProtocol, + sampleCheckpointMetadata, + sampleTxns.last, + sampleRoot, + sampleSidecars.head, + sampleDomainMetadatas.head) + val reorderedCheckpoint = Checkpoint.fromActions(reorderedActions) + assert(reorderedCheckpoint.actions.map(_.unwrap) === reorderedActions) + + assert(reorderedCheckpoint.checkpointMetadata === sampleCheckpointMetadata) + assert(reorderedCheckpoint.version === sampleCheckpointMetadata.version) + assert(reorderedCheckpoint.contentRoot === sampleRoot) + assert(reorderedCheckpoint.protocol === sampleProtocol) + assert(reorderedCheckpoint.metaData === sampleMetadata) + + assert(reorderedCheckpoint.domainMetadata === sampleDomainMetadatas.reverse) + assert(reorderedCheckpoint.txns === sampleTxns) // order is preserved + assert(reorderedCheckpoint.sidecars === sampleSidecars.reverse) + + assert(Action.fromJson(reorderedCheckpoint.json) === reorderedCheckpoint) + } + test("Checkpoint: round-trips the full inline snapshot") { val protocol = Protocol(3, 7) .withReaderFeatures(Seq("deletionVectors", "v2Checkpoint")) @@ -270,32 +389,34 @@ class AdaptiveMetadataActionsSerializerSuite extends SparkFunSuite { val dm = Seq( DomainMetadata(domain = "delta.rowTracking", configuration = "{}", removed = false), DomainMetadata(domain = "user.tag", configuration = "{\"k\":\"v\"}", removed = false)) - val txns = Seq( - SetTransaction(appId = "app-1", version = 7L, lastUpdated = Some(100L)), - SetTransaction(appId = "app-2", version = 8L, lastUpdated = None)) val cp = Checkpoint( version = 41L, contentRoot = sampleRoot.copy(version = 41L), protocol = protocol, metaData = sampleMetadata, domainMetadata = dm, - txns = txns, + txns = sampleTxns, sidecars = Seq.empty) assert(JsonUtils.toPrettyJson(cp.wrap) === """{ - | "checkpoint" : { - | "version" : 41, + | "checkpoint" : [ { + | "checkpointMetadata" : { + | "version" : 41 + | } + | }, { | "contentRoot" : { | "path" : "metadata/root-abc.parquet", | "sizeInBytes" : 4096, | "version" : 41 - | }, + | } + | }, { | "protocol" : { | "minReaderVersion" : 3, | "minWriterVersion" : 7, | "readerFeatures" : [ "deletionVectors", "v2Checkpoint" ], | "writerFeatures" : [ "deletionVectors", "v2Checkpoint", "rowTracking" ] - | }, + | } + | }, { | "metaData" : { | "id" : "metadata-id", | "name" : "t", @@ -305,26 +426,31 @@ class AdaptiveMetadataActionsSerializerSuite extends SparkFunSuite { | }, | "partitionColumns" : [ ], | "configuration" : { } - | }, - | "domainMetadata" : [ { + | } + | }, { + | "domainMetadata" : { | "domain" : "delta.rowTracking", | "configuration" : "{}", | "removed" : false - | }, { + | } + | }, { + | "domainMetadata" : { | "domain" : "user.tag", | "configuration" : "{\"k\":\"v\"}", | "removed" : false - | } ], - | "txns" : [ { + | } + | }, { + | "txn" : { | "appId" : "app-1", | "version" : 7, | "lastUpdated" : 100 - | }, { + | } + | }, { + | "txn" : { | "appId" : "app-2", | "version" : 8 - | } ], - | "sidecars" : [ ] - | } + | } + | } ] |}""".stripMargin) val parsed = Action.fromJson(cp.json).asInstanceOf[Checkpoint] assert(parsed === cp) @@ -338,22 +464,27 @@ class AdaptiveMetadataActionsSerializerSuite extends SparkFunSuite { metaData = sampleMetadata, domainMetadata = Seq.empty, txns = Seq.empty, - sidecars = Seq(dmSidecar(), txnSidecar())) + sidecars = sampleSidecars) assert(JsonUtils.toPrettyJson(cp.wrap) === """{ - | "checkpoint" : { - | "version" : 1, + | "checkpoint" : [ { + | "checkpointMetadata" : { + | "version" : 1 + | } + | }, { | "contentRoot" : { | "path" : "metadata/root-abc.parquet", | "sizeInBytes" : 4096, | "version" : 1 - | }, + | } + | }, { | "protocol" : { | "minReaderVersion" : 3, | "minWriterVersion" : 7, | "readerFeatures" : [ ], | "writerFeatures" : [ ] - | }, + | } + | }, { | "metaData" : { | "id" : "metadata-id", | "name" : "t", @@ -363,21 +494,22 @@ class AdaptiveMetadataActionsSerializerSuite extends SparkFunSuite { | }, | "partitionColumns" : [ ], | "configuration" : { } - | }, - | "domainMetadata" : [ ], - | "txns" : [ ], - | "sidecars" : [ { + | } + | }, { + | "sidecar" : { | "path" : "dm.parquet", | "sizeInBytes" : 1, | "modificationTime" : 0, | "type" : "domainMetadata" - | }, { + | } + | }, { + | "sidecar" : { | "path" : "txn.parquet", | "sizeInBytes" : 1, | "modificationTime" : 0, | "type" : "txn" - | } ] - | } + | } + | } ] |}""".stripMargin) val parsed = Action.fromJson(cp.json).asInstanceOf[Checkpoint] assert(parsed === cp) @@ -393,7 +525,7 @@ class AdaptiveMetadataActionsSerializerSuite extends SparkFunSuite { metaData = sampleMetadata, domainMetadata = dm, txns = Seq.empty, - sidecars = Seq(dmSidecar(), txnSidecar())) + sidecars = sampleSidecars) val parsed = Action.fromJson(cp.json).asInstanceOf[Checkpoint] assert(parsed === cp) assert(parsed.domainMetadata === dm) @@ -401,7 +533,51 @@ class AdaptiveMetadataActionsSerializerSuite extends SparkFunSuite { assert(parsed.sidecars.length === 2) } - test("Checkpoint: rejects sidecar without sidecarType") { + // ============================================================================================ + // Checkpoint: validation + // ============================================================================================ + + for { + requiredAction <- sampleRequiredActions + count <- Seq(0, 2) + } { + val actionName = requiredAction.getClass.getSimpleName + test(s"Checkpoint: rejects $count $actionName actions at construction and JSON decoding") { + val actions = sampleFullActions.filterNot(_ == requiredAction) ++ + Seq.fill(count)(requiredAction) + val expectedMessage = s"exactly one $actionName action, found $count" + val ex = intercept[IllegalArgumentException] { + Checkpoint.fromActions(actions) + } + assert(ex.getMessage.contains(expectedMessage)) + val jsonEx = intercept[IllegalArgumentException] { + val wrapped = actions.map(_.wrapAsAMTCheckpointSingleAction) + JsonUtils.fromJson[Checkpoint](JsonUtils.toJson(wrapped)) + } + assert(jsonEx.getMessage.contains(expectedMessage)) + } + } + + test("Checkpoint: rejects a null action collection") { + val ex = intercept[IllegalArgumentException] { + Checkpoint(actions = null) + } + assert(ex.getMessage.contains("Checkpoint actions must not be null")) + } + + test("Checkpoint: rejects null entries at construction and JSON decoding") { + val actions = sampleFullCheckpoint.actions :+ null + val ex = intercept[IllegalArgumentException] { + Checkpoint(actions) + } + assert(ex.getMessage.contains("Checkpoint actions must not contain null entries")) + val jsonEx = intercept[IllegalArgumentException] { + JsonUtils.fromJson[Checkpoint](JsonUtils.toJson(actions)) + } + assert(jsonEx.getMessage.contains("Checkpoint actions must not contain null entries")) + } + + test("Checkpoint: rejects sidecar without type") { val untyped = SidecarFile(path = "x.parquet", sizeInBytes = 1L, modificationTime = 0L) val ex = intercept[IllegalArgumentException] { Checkpoint( @@ -413,7 +589,7 @@ class AdaptiveMetadataActionsSerializerSuite extends SparkFunSuite { txns = Seq.empty, sidecars = Seq(untyped)) } - assert(ex.getMessage.contains("sidecarType")) + assert(ex.getMessage.contains("must have a type")) } test("Checkpoint: round-trips multiple sidecars of the same type") { @@ -428,7 +604,7 @@ class AdaptiveMetadataActionsSerializerSuite extends SparkFunSuite { val parsed = Action.fromJson(cp.json).asInstanceOf[Checkpoint] assert(parsed === cp) assert(parsed.sidecars.length === 2) - assert(parsed.sidecars.forall(_.sidecarType.contains(SidecarType.Type.DomainMetadata))) + assert(parsed.sidecars.forall(_.`type`.contains(SidecarType.Type.DomainMetadata))) } test("Checkpoint: contentRoot.version may lag checkpoint.version") { From dc219cd793c9ec9b8a3f62ad1fd58fad6c8aa92f Mon Sep 17 00:00:00 2001 From: Jinhua Song <114370144+PhilPlato@users.noreply.github.com> Date: Mon, 12 Oct 2026 19:12:30 -0700 Subject: [PATCH 07/29] [Spark] Generate snapshot_id and modified_snapshot_id for AMT data entries (#7851) ## Description Generates `snapshot_id` and `modified_snapshot_id` for data entries written to adaptive metadata tree (AMT) checkpoints, so each data entry records the snapshot that added it and the snapshot that last modified it. This implements the file-modification tracking described in the protocol RFC #7820. Key changes: - Populate `snapshot_id` / `modified_snapshot_id` on AMT data entries in `AMTWriteHelper` and `IncrementalAMTWriter`, and carry them through the AMT action model. - Propagate the fields through the commit path (`OptimisticTransaction`, `Checkpoints`, `Snapshot`) and through commands that rewrite or re-add files (`CloneTableBase`, `RestoreTableCommand`, `TahoeRemoveFileIndex`). ## How was this patch tested? Added/updated coverage across the AMT suites (`AMTCheckpointWriteSuite`, `AMTIncrementalWriteInvariantSuite`, `AMTInheritanceSuite`, `AMTInheritanceReadSuite`, `AMTSnapshotSuite`, `AMTBackReferenceSuite`, `AMTPartitionValuesSuite`, `AMTSingleActionSerializerSuite`, `AMTUtilsSuite`) plus `ActionSerializerSuite`, `CheckpointsSuite`, and `LastCheckpointInfoSuite`. ## Does this PR introduce _any_ user-facing changes? No. Co-authored-by: Jinhua Song --- .../apache/spark/sql/delta/Checkpoints.scala | 16 ++- .../sql/delta/OptimisticTransaction.scala | 28 +++- .../org/apache/spark/sql/delta/Snapshot.scala | 4 +- .../spark/sql/delta/actions/actions.scala | 22 ++- .../spark/sql/delta/amt/AMTWriteHelper.scala | 2 +- .../sql/delta/amt/IncrementalAMTWriter.scala | 4 +- .../apache/spark/sql/delta/amt/actions.scala | 104 ++++++++++++-- .../sql/delta/commands/CloneTableBase.scala | 6 +- .../delta/commands/RestoreTableCommand.scala | 2 +- .../delta/files/TahoeRemoveFileIndex.scala | 4 +- .../sql/delta/ActionSerializerSuite.scala | 29 +++- .../spark/sql/delta/CheckpointsSuite.scala | 31 ++-- .../sql/delta/LastCheckpointInfoSuite.scala | 2 +- .../sql/delta/amt/AMTBackReferenceSuite.scala | 34 ++++- .../delta/amt/AMTCheckpointWriteSuite.scala | 2 +- .../AMTIncrementalWriteInvariantSuite.scala | 2 +- .../delta/amt/AMTInheritanceReadSuite.scala | 2 +- .../sql/delta/amt/AMTInheritanceSuite.scala | 6 +- .../delta/amt/AMTPartitionValuesSuite.scala | 2 +- .../amt/AMTSingleActionSerializerSuite.scala | 58 +++++++- .../sql/delta/amt/AMTSnapshotSuite.scala | 135 ++++++++++++++++-- .../spark/sql/delta/amt/AMTUtilsSuite.scala | 96 ++++++++++++- 22 files changed, 522 insertions(+), 69 deletions(-) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/Checkpoints.scala b/spark/src/main/scala/org/apache/spark/sql/delta/Checkpoints.scala index 1e21828fed1..52b9c4befb0 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/Checkpoints.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/Checkpoints.scala @@ -949,6 +949,10 @@ object Checkpoints /** The name of the last checkpoint file */ val LAST_CHECKPOINT_FILE_NAME = "_last_checkpoint" + /** RemoveFile fields intentionally omitted from classic and V2 checkpoints. */ + private[delta] val REMOVE_FILE_EXCLUDED_FROM_CHECKPOINT_FIELDS = + Seq("tags", "stats", "snapshotId", "modifiedSnapshotId") + /** * Determines the V2 checkpoint format to use for the given snapshot, if applicable. * @@ -1121,7 +1125,9 @@ object Checkpoints // separately after sidecar files are written. repartitioned .select("add", "remove") - .withColumn("remove", col("remove").dropFields("tags", "stats")) + .withColumn( + "remove", + col("remove").dropFields(REMOVE_FILE_EXCLUDED_FROM_CHECKPOINT_FIELDS: _*)) .where("add is not null or remove is not null") } else { // When V2 Checkpoint is disabled, the baseCheckpoint refers to the main classic checkpoint @@ -1129,7 +1135,9 @@ object Checkpoints // "checkpoint". repartitioned .drop("commitInfo", "cdc", "checkpointMetadata", "sidecar", "checkpoint") - .withColumn("remove", col("remove").dropFields("tags", "stats")) + .withColumn( + "remove", + col("remove").dropFields(REMOVE_FILE_EXCLUDED_FROM_CHECKPOINT_FIELDS: _*)) } } @@ -1651,8 +1659,8 @@ object Checkpoints additionalCols ++= partitionValues additionalCols ++= Checkpoints.extractStats(snapshot.statsSchema, "add.stats") } - // amtResidue and backReference are dropped on purpose here: V1/V2 checkpoints are incompatible - // with AMT, so these AMT-era extras do not belong in the checkpoint `add` struct. + // amtPassthrough, backReference, and snapshot IDs are dropped on purpose here: V1/V2 + // checkpoints are incompatible with AMT, so these extras do not belong in the `add` struct. val withAdd = state.withColumn("add", when(col("add").isNotNull, struct(Seq( col("add.path"), diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/OptimisticTransaction.scala b/spark/src/main/scala/org/apache/spark/sql/delta/OptimisticTransaction.scala index 85791970d3f..fbe1210005f 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/OptimisticTransaction.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/OptimisticTransaction.scala @@ -37,7 +37,7 @@ import org.apache.spark.sql.delta.DeltaGeoSpatial import org.apache.spark.sql.delta.DeltaOperations.{ChangeColumn, ChangeColumns, CreateTable, Operation, ReplaceColumns, ReplaceTable, UpdateSchema} import org.apache.spark.sql.delta.RowId.RowTrackingMetadataDomain import org.apache.spark.sql.delta.actions._ -import org.apache.spark.sql.delta.amt.{AMTCheckpointProvider, AMTCommitStats, AMTUtils, AMTWriteResult, AMTWriterManager} +import org.apache.spark.sql.delta.amt.{AMTCheckpointProvider, AMTCommitStats, AMTSnapshotId, AMTUtils, AMTWriteResult, AMTWriterManager} import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.commands.DeletionVectorUtils import org.apache.spark.sql.delta.commands.cdc.CDCReader @@ -440,6 +440,9 @@ trait OptimisticTransactionImpl extends TransactionHelper /** The transaction start time. */ private val txnStartNano = System.nanoTime() + /** The snapshot ID assigned to AMT changes made by this transaction. */ + private val amtSnapshotId: Long = AMTSnapshotId.generate() + override val snapshotToScan: Snapshot = snapshot /** @@ -2214,6 +2217,8 @@ trait OptimisticTransactionImpl extends TransactionHelper allActions = RowId.assignFreshRowIds(spark, protocol, snapshot, allActions, op) allActions = DefaultRowCommitVersion.assignIfMissing( spark, protocol, snapshot, allActions, getFirstAttemptVersion) + allActions = AMTSnapshotId.assignIfMissing( + metadata, protocol, allActions, amtSnapshotId) val (allActions3, icebergTrackedInfo) = generateIcebergMetadataForCommitLarge( allActions, catalogTable, attemptVersion, commitInfo) // Capture the committed DomainMetadata (e.g. clustering columns) for ALL tables so a @@ -3003,9 +3008,24 @@ trait OptimisticTransactionImpl extends TransactionHelper deltaLog, currentTransactionInfo.op, callerContext = "doCommit").foreach(_ => ()) + // DefaultRowCommitVersion is assigned while building CurrentTransactionInfo and updated by + // ConflictChecker when a winning commit advances the target version. + // AMT snapshot IDs are assigned here and preserved across retries. + val transactionInfoWithSnapshotIds: CurrentTransactionInfo = { + val actionsWithSnapshotIds = AMTSnapshotId.assignIfMissing( + currentTransactionInfo.metadata, + currentTransactionInfo.protocol, + currentTransactionInfo.actions.iterator, + amtSnapshotId).toSeq + if (actionsWithSnapshotIds == currentTransactionInfo.actions) { + currentTransactionInfo + } else { + currentTransactionInfo.copy(actions = actionsWithSnapshotIds) + } + } val amtWriteResultOpt = amtWriterManager.writeAMT( nextAttemptVersion = attemptVersion, - currentTransactionInfo = currentTransactionInfo, + currentTransactionInfo = transactionInfoWithSnapshotIds, preCommitLogSegment = preCommitLogSegment, winningCommitMetricsForConflictedRange = winningCommitMetrics) @@ -3021,7 +3041,7 @@ trait OptimisticTransactionImpl extends TransactionHelper spark, this, attemptVersion, - currentTransactionInfo + transactionInfoWithSnapshotIds , table ) if (isConversionPerformed) { @@ -3029,7 +3049,7 @@ trait OptimisticTransactionImpl extends TransactionHelper NANOSECONDS.toMillis(System.nanoTime() - startNanos)) } updatedInfo - }.getOrElse(currentTransactionInfo) + }.getOrElse(transactionInfoWithSnapshotIds) val baseActions = updatedCurrentTransactionInfo.finalActionsToCommit val actions = amtWriteResultOpt match { diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/Snapshot.scala b/spark/src/main/scala/org/apache/spark/sql/delta/Snapshot.scala index aeff9868129..4fc0936feab 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/Snapshot.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/Snapshot.scala @@ -719,7 +719,9 @@ class Snapshot( col("add.defaultRowCommitVersion"), col("add.clusteringProvider"), col("add.backReference"), - col("add.amtPassthrough") + col("add.amtPassthrough"), + col("add.snapshotId"), + col("add.modifiedSnapshotId") ))) .withColumn("remove", when( col("remove.path").isNotNull, diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/actions/actions.scala b/spark/src/main/scala/org/apache/spark/sql/delta/actions/actions.scala index 0d4a755e178..6f6fff9061d 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/actions/actions.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/actions/actions.scala @@ -972,7 +972,11 @@ case class AddFile( defaultRowCommitVersion: Option[Long] = None, clusteringProvider: Option[String] = None, backReference: Option[BackReference] = None, - amtPassthrough: Option[AMTPassthrough] = None + amtPassthrough: Option[AMTPassthrough] = None, + @JsonDeserialize(contentAs = classOf[java.lang.Long]) + snapshotId: Option[Long] = None, + @JsonDeserialize(contentAs = classOf[java.lang.Long]) + modifiedSnapshotId: Option[Long] = None ) extends FileAction with HasNumRecords { require(path.nonEmpty) @@ -999,7 +1003,10 @@ case class AddFile( baseRowId = baseRowId, defaultRowCommitVersion = defaultRowCommitVersion, stats = stats, - backReference = backReference + backReference = backReference, + // Set snapshotId to None so OptimisticTransaction stamps it during commit. + snapshotId = None, + modifiedSnapshotId = modifiedSnapshotId ) // scalastyle:on } @@ -1029,7 +1036,10 @@ case class AddFile( var addFileWithNewDv = this.copy( deletionVector = dvDescriptorWithoutMaxRowIndex, dataChange = dataChange, - backReference = None) + backReference = None, + // Reset modifiedSnapshotId to None, so that it is stamped back to the current + // txn's snapshot id by OptimisticTransaction. + modifiedSnapshotId = None) if (updateStats) { addFileWithNewDv = addFileWithNewDv.withoutTightBoundStats } @@ -1260,7 +1270,11 @@ case class RemoveFile( @JsonDeserialize(contentAs = classOf[java.lang.Long]) defaultRowCommitVersion: Option[Long] = None, override val stats: String = null, - backReference: Option[BackReference] = None + backReference: Option[BackReference] = None, + @JsonDeserialize(contentAs = classOf[java.lang.Long]) + snapshotId: Option[Long] = None, + @JsonDeserialize(contentAs = classOf[java.lang.Long]) + modifiedSnapshotId: Option[Long] = None ) extends FileAction with HasNumRecords { override def wrap: SingleAction = SingleAction(remove = this) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/amt/AMTWriteHelper.scala b/spark/src/main/scala/org/apache/spark/sql/delta/amt/AMTWriteHelper.scala index d050aa4423c..cc816d27116 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/amt/AMTWriteHelper.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/amt/AMTWriteHelper.scala @@ -347,7 +347,7 @@ object AMTWriteHelper extends DeltaLogging { private def initializeTracking(status: Int): Tracking = Tracking( status = status, snapshot_id = None, - dv_snapshot_id = None, + modified_snapshot_id = None, sequence_number = None, file_sequence_number = None, first_row_id = None, diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/amt/IncrementalAMTWriter.scala b/spark/src/main/scala/org/apache/spark/sql/delta/amt/IncrementalAMTWriter.scala index 6d599296af2..bf402fd151d 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/amt/IncrementalAMTWriter.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/amt/IncrementalAMTWriter.scala @@ -433,7 +433,9 @@ class IncrementalAMTWriter(spark: SparkSession, deltaLog: DeltaLog) { val tracking = if (commitAddedPaths.contains(r.path)) AMTWriteHelper.replacedTrackingForDataEntry() else AMTWriteHelper.removedTrackingForDataEntry() - DataEntry.fromAddFile(priorAddFile, tracking, tableRoot) + // scalastyle:off removeFile // This calls a DataEntry factory, not a RemoveFile constructor. + DataEntry.fromRemoveFile(priorAddFile, r, tracking, tableRoot) + // scalastyle:on removeFile } } diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/amt/actions.scala b/spark/src/main/scala/org/apache/spark/sql/delta/amt/actions.scala index ace09c00c8e..6c2d2bdf97b 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/amt/actions.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/amt/actions.scala @@ -16,8 +16,10 @@ package org.apache.spark.sql.delta.amt +import java.util.UUID + import org.apache.spark.sql.delta.DeltaColumnMapping -import org.apache.spark.sql.delta.actions.{Action, AddFile, DeletionVectorDescriptor, Metadata, Protocol} +import org.apache.spark.sql.delta.actions.{Action, AddFile, DeletionVectorDescriptor, Metadata, Protocol, RemoveFile} import org.apache.spark.sql.delta.storage.dv.DeletionVectorStore import com.fasterxml.jackson.annotation.JsonIgnore import com.fasterxml.jackson.core.JsonParser @@ -240,7 +242,7 @@ object AMTSingleAction { optional(1L, "snapshot_id"), optional(3L, "sequence_number"), optional(4L, "file_sequence_number"), - optional(5L, "dv_snapshot_id"), + optional(5L, "modified_snapshot_id"), optional(142L, "first_row_id"), optional(6L, "deleted_positions"), optional(7L, "replaced_positions")) @@ -499,6 +501,8 @@ case class DataEntry( deletionVector = dv, baseRowId = tracking.first_row_id, defaultRowCommitVersion = tracking.file_sequence_number, + snapshotId = tracking.snapshot_id, + modifiedSnapshotId = tracking.modified_snapshot_id, tags = tags.orNull, amtPassthrough = AMTPassthrough.fromDataEntry(this)) } @@ -508,6 +512,7 @@ object DataEntry { /** Creates [[DataEntry]] from AddFile. */ def fromAddFile(add: AddFile, tracking: Tracking, tableRoot: Path): DataEntry = { val passthrough = add.amtPassthrough + val deletionVector = Option(add.deletionVector).map(DeletionVector.fromDescriptor(_, tableRoot)) DataEntry( // RFC-2396 to Iceberg raw path conversion. location = AMTUtils.relativizeLocation( @@ -516,6 +521,8 @@ object DataEntry { // Round-trip the AddFile's row-tracking fields through the Iceberg tracking envelope so a // rowTracking-enabled table can reconstruct them on read. tracking = tracking.copy( + snapshot_id = add.snapshotId, + modified_snapshot_id = add.modifiedSnapshotId, first_row_id = add.baseRowId, sequence_number = add.defaultRowCommitVersion, file_sequence_number = add.effectiveFileSequenceNumber), @@ -526,8 +533,7 @@ object DataEntry { s"Cannot build AMT entry: AddFile has no record count (missing stats): ${add.path}.")), file_size_in_bytes = add.size, partition = Option(add.partitionValues).filter(_.nonEmpty), - deletion_vector = - Option(add.deletionVector).map(DeletionVector.fromDescriptor(_, tableRoot)), + deletion_vector = deletionVector, spec_id = passthrough.flatMap(_.spec_id), sort_order_id = passthrough.flatMap(_.sort_order_id), key_metadata = passthrough.flatMap(_.key_metadata), @@ -535,6 +541,26 @@ object DataEntry { tags = Option(add.tags).map(_.toMap).filter(_.nonEmpty), content_stats = AMTContentStats.fromStatsJson(add.stats)) } + + /** + * Creates a terminal [[DataEntry]] using the prior AddFile's complete content metadata and the + * RemoveFile's snapshot provenance. + */ + // scalastyle:off removeFile // This is a DataEntry factory, not a RemoveFile constructor. + def fromRemoveFile( + priorAdd: AddFile, + remove: RemoveFile, + tracking: Tracking, + tableRoot: Path): DataEntry = { + require( + tracking.status == Tracking.Status.Deleted || + tracking.status == Tracking.Status.Replaced, + s"Removed data entry must be DELETED or REPLACED, got ${tracking.status}.") + fromAddFile(priorAdd.copy( + snapshotId = remove.snapshotId, + modifiedSnapshotId = remove.modifiedSnapshotId), tracking, tableRoot) + } + // scalastyle:on removeFile } /** @@ -657,6 +683,68 @@ object AMTPassthrough { } } +/** + * Assigns AMT snapshot provenance at the commit-attempt boundary. + * + * Commands preserve an ID when the corresponding lifecycle is unchanged and clear it to request + * a new ID. + */ +object AMTSnapshotId { + + // scalastyle:off line.size.limit + /** + * Generates a non-negative random ID using the same algorithm as Apache Iceberg: + * https://github.com/apache/iceberg/blob/main/core/src/main/java/org/apache/iceberg/SnapshotIdGeneratorUtil.java + */ + // scalastyle:on line.size.limit + def generate(): Long = { + val uuid = UUID.randomUUID() + (uuid.getMostSignificantBits ^ uuid.getLeastSignificantBits) & Long.MaxValue + } + + private def hasModificationMetadata(add: AddFile): Boolean = { + val hasDeletionVector = add.deletionVector != null + hasDeletionVector + } + + private def hasModificationMetadata(remove: RemoveFile): Boolean = { + val hasDeletionVector = remove.deletionVector != null + hasDeletionVector + } + + def assignIfMissing( + metadata: Metadata, + protocol: Protocol, + actions: Iterator[Action], + snapshotId: Long): Iterator[Action] = { + require(snapshotId >= 0L, s"AMT snapshot ID must be non-negative: $snapshotId.") + if (!AMTUtils.amtEnabled(metadata, protocol)) { + return actions + } + + actions.map { + case add: AddFile => + // If an AddFile carries modification metadata (for example, a DV or column updates) and + // both snapshot IDs are missing, initialize both to the current snapshotId. This can happen + // in RESTORE / CLONE when a file is introduced with an existing DV or column file. + add.copy( + snapshotId = add.snapshotId.orElse(Some(snapshotId)), + modifiedSnapshotId = + add.modifiedSnapshotId.orElse( + if (hasModificationMetadata(add)) Some(snapshotId) else None)) + + case remove: RemoveFile => + remove.copy( + snapshotId = remove.snapshotId.orElse(Some(snapshotId)), + modifiedSnapshotId = + remove.modifiedSnapshotId.orElse( + if (hasModificationMetadata(remove)) Some(snapshotId) else None)) + + case action => action + } + } +} + /** * A pointer from the root to a leaf data manifest (`content_type = 3`). * @@ -758,17 +846,17 @@ case class DataManifestEntry( * * @param status Entry status (0=existing, 1=added, 2=deleted, 3=replaced, 4=modified). * @param snapshot_id Snapshot the file was added/deleted/replaced in; inherited from root. - * @param dv_snapshot_id Snapshot the DV was added in; null when the entry has no DV. + * @param modified_snapshot_id Snapshot where the file's modification metadata last changed. * @param sequence_number Data sequence number of the file. * @param file_sequence_number File sequence number (when the file was added). * @param first_row_id Id of the first row in the data file. - * @param deleted_positions Bitmap of positions deleted in this snapshot. - * @param replaced_positions Bitmap of positions replaced in this snapshot. + * @param deleted_positions Bitmap of positions deleted in the `modified_snapshot_id` snapshot. + * @param replaced_positions Bitmap of positions replaced in the `modified_snapshot_id` snapshot. */ case class Tracking( status: Int, // ID: 0, required. snapshot_id: Option[Long], // ID: 1, optional. - dv_snapshot_id: Option[Long], // ID: 5, optional. + modified_snapshot_id: Option[Long], // ID: 5, optional. sequence_number: Option[Long], // ID: 3, optional. file_sequence_number: Option[Long], // ID: 4, optional. first_row_id: Option[Long], // ID: 142, optional. diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/commands/CloneTableBase.scala b/spark/src/main/scala/org/apache/spark/sql/delta/commands/CloneTableBase.scala index 5f7752234bd..2aa5afde29b 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/commands/CloneTableBase.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/commands/CloneTableBase.scala @@ -228,11 +228,11 @@ abstract class CloneTableBase( var actions: Iterator[Action] = addFileIter.map { fileToCopy => val copiedFile = fileToCopy.copy(dataChange = dataChangeInFileAction) - // CLONE does not preserve Row IDs and Commit Versions, nor the source table's AMT - // back reference and passthrough. + // CLONE does not preserve Row IDs and Commit Versions, nor the source + // table's AMT back reference, snapshot IDs and passthrough. copiedFile.copy( baseRowId = None, defaultRowCommitVersion = None, backReference = None, - amtPassthrough = None) + amtPassthrough = None, snapshotId = None, modifiedSnapshotId = None) } sourceTable.snapshot.foreach { sourceSnapshot => // Handle DomainMetadata for cloning a table. diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/commands/RestoreTableCommand.scala b/spark/src/main/scala/org/apache/spark/sql/delta/commands/RestoreTableCommand.scala index f16d39e3019..93f2f3aa6c1 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/commands/RestoreTableCommand.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/commands/RestoreTableCommand.scala @@ -187,7 +187,7 @@ case class RestoreTableCommand(sourceTable: DeltaTableV2) val filesToAdd = normalizedSource .join(normalizedTarget, joinExprs, "left_anti") .select(column("srcAddFile").as[AddFile]) - .map(_.copy(dataChange = true)) + .map(_.copy(dataChange = true, snapshotId = None, modifiedSnapshotId = None)) val filesToRemove = normalizedTarget .join(normalizedSource, joinExprs, "left_anti") diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/files/TahoeRemoveFileIndex.scala b/spark/src/main/scala/org/apache/spark/sql/delta/files/TahoeRemoveFileIndex.scala index aa102c0c8b3..5b8e82967e0 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/files/TahoeRemoveFileIndex.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/files/TahoeRemoveFileIndex.scala @@ -81,7 +81,9 @@ class TahoeRemoveFileIndex( dataChange = r.dataChange, tags = r.tags, deletionVector = r.deletionVector, - baseRowId = r.baseRowId + baseRowId = r.baseRowId, + snapshotId = r.snapshotId, + modifiedSnapshotId = r.modifiedSnapshotId ) } } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/ActionSerializerSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/ActionSerializerSuite.scala index af3d4089ab5..637280c3637 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/ActionSerializerSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/ActionSerializerSuite.scala @@ -168,15 +168,30 @@ class ActionSerializerSuite extends QueryTest with SharedSparkSession with Delta assert(Action.fromJson(emptyPassthrough.json) === emptyPassthrough) } - test("AddFile amtPassthrough propagates to copy but not to removeWithTimestamp") { + test("AddFile snapshot IDs serialize and transition to RemoveFile") { val passthrough = Some(AMTPassthrough(spec_id = Some(7))) - val add = AddFile("a", Map.empty, 1, 2, dataChange = true, amtPassthrough = passthrough) + val add = AddFile( + "a", + Map.empty, + 1, + 2, + dataChange = true, + amtPassthrough = passthrough, + snapshotId = Some(11L), + modifiedSnapshotId = Some(12L)) + assert(add.json.contains("\"snapshotId\":11")) + assert(add.json.contains("\"modifiedSnapshotId\":12")) + assert(Action.fromJson(add.json) === add) + // copy (the mechanism the superseding AddFile of removeRows relies on) preserves it. - assert(add.copy(dataChange = false).amtPassthrough == passthrough) - // Unlike backReference, RemoveFile has no amtPassthrough field: the tombstone drops it. The - // surviving AddFile is what carries the AMT-native state forward. - assert(add.removeWithTimestamp().json.contains("\"remove\"")) - assert(!add.removeWithTimestamp().json.contains("amtPassthrough")) + val copied = add.copy(dataChange = false) + assert(copied.snapshotId.contains(11L)) + assert(copied.modifiedSnapshotId.contains(12L)) + val remove = add.removeWithTimestamp() + assert(!remove.json.contains("amtPassthrough")) + assert(remove.snapshotId.isEmpty) + assert(remove.modifiedSnapshotId.contains(12L)) + assert(Action.fromJson(remove.json) === remove) } // This is the same test as "removefile" in OSS, but due to a Jackson library upgrade the behavior diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/CheckpointsSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/CheckpointsSuite.scala index 9199bb1c672..37e938cd908 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/CheckpointsSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/CheckpointsSuite.scala @@ -49,7 +49,7 @@ import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.functions.{col, lit, when} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SharedSparkSession -import org.apache.spark.sql.types.StructType +import org.apache.spark.sql.types.{LongType, StructType} import org.apache.spark.types.variant.{Variant, VariantUtil} import org.apache.spark.unsafe.types.VariantVal @@ -724,7 +724,7 @@ class CheckpointsSuite } } - test("non-AMT tables do not persist backReference or amtPassthrough in checkpoint or commit") { + test("non-AMT tables do not persist AMT-only AddFile fields") { withClassicCheckpointPolicyForCatalogOwned { withTempDir { tempDir => val tablePath = tempDir.getAbsolutePath @@ -735,7 +735,7 @@ class CheckpointsSuite deltaLog.checkpoint() val version = deltaLog.snapshot.version - // 1. The classic checkpoint parquet must carry neither backReference nor amtPassthrough. + // 1. The classic checkpoint parquet must not carry AMT-only AddFile fields. val checkpointFile = FileNames.checkpointFileSingular(deltaLog.logPath, version).toString val checkpointSchema = spark.read.format("parquet").load(checkpointFile).schema @@ -749,8 +749,14 @@ class CheckpointsSuite s"checkpoint remove struct must not contain backReference, got: $removeFields") assert(!addFields.contains("amtPassthrough"), s"checkpoint add struct must not contain amtPassthrough, got: $addFields") + Seq("snapshotId", "modifiedSnapshotId").foreach { field => + assert(!addFields.contains(field), + s"checkpoint add struct must not contain $field, got: $addFields") + assert(!removeFields.contains(field), + s"checkpoint remove struct must not contain $field, got: $removeFields") + } - // 2. No commit JSON should mention backReference or amtPassthrough. + // 2. No commit JSON should mention AMT-only fields. val commitProvider = DeltaCommitFileProvider(deltaLog.unsafeVolatileSnapshot) val hadoopConf = deltaLog.newDeltaHadoopConf() (0L to version).foreach { v => @@ -761,6 +767,10 @@ class CheckpointsSuite assert(!content.exists(_.contains("amtPassthrough")), s"commit JSON for version $v must not contain amtPassthrough: " + s"${content.mkString("\n")}") + Seq("snapshotId", "modifiedSnapshotId").foreach { field => + assert(!content.exists(_.contains(field)), + s"commit JSON for version $v must not contain $field: ${content.mkString("\n")}") + } } } } @@ -1092,14 +1102,19 @@ class CheckpointsSuite val actionsToWrite = Checkpoints .buildCheckpoint(actionsDS, snapshot) - // buildCheckpoint drops backReference (add/remove) and amtPassthrough (add only) from the - // on-disk shape, so we re-materialize them as null columns before .as[SingleAction]. + // buildCheckpoint drops AMT-only fields from the on-disk shape, so re-materialize them as + // null columns before .as[SingleAction]. .withColumn("add", when(col("add.path").isNotNull, col("add") .withField("backReference", lit(null).cast(BackReference.STRUCT_TYPE)) - .withField("amtPassthrough", lit(null).cast(AMTPassthrough.STRUCT_TYPE)))) + .withField("amtPassthrough", lit(null).cast(AMTPassthrough.STRUCT_TYPE)) + .withField("snapshotId", lit(null).cast(LongType)) + .withField("modifiedSnapshotId", lit(null).cast(LongType)))) .withColumn("remove", when(col("remove.path").isNotNull, - col("remove").withField("backReference", lit(null).cast(BackReference.STRUCT_TYPE)))) + col("remove") + .withField("backReference", lit(null).cast(BackReference.STRUCT_TYPE)) + .withField("snapshotId", lit(null).cast(LongType)) + .withField("modifiedSnapshotId", lit(null).cast(LongType)))) .as[SingleAction] .collect() .toSeq diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/LastCheckpointInfoSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/LastCheckpointInfoSuite.scala index a6321efbb41..dd0b1975c7c 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/LastCheckpointInfoSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/LastCheckpointInfoSuite.scala @@ -287,7 +287,7 @@ class LastCheckpointInfoSuite extends SharedSparkSession tracking = Tracking( status = Tracking.Status.Added, snapshot_id = Some(11L), - dv_snapshot_id = Some(12L), + modified_snapshot_id = Some(12L), sequence_number = Some(13L), file_sequence_number = Some(14L), first_row_id = Some(15L), diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTBackReferenceSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTBackReferenceSuite.scala index 0e675f9115f..df5ef76b2f3 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTBackReferenceSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTBackReferenceSuite.scala @@ -807,20 +807,30 @@ class AMTBackReferenceSuite // table directory (see the naming note in `testAcrossAMTCheckpointScenarios`). val tgt = s"${src}_clone_tgt" val srcBackRefByPath = stampedBackRefs(context.postCheckpointSnapshot) + val srcSnapshotIds = liveAddFiles(context.postCheckpointSnapshot).flatMap(_.snapshotId).toSet withTable(tgt) { clone(src, tgt) val tgtLog = deltaLogForName(tgt) val latest = tgtLog.update().version - val clonedAdds = (0L to latest) + val addsBySnapshotId = (0L to latest) .flatMap(v => actionsAt(tgtLog, v)) - .collect { case a: AddFile => a } - assert(clonedAdds.size >= srcBackRefByPath.size, - s"the clone must add the ${srcBackRefByPath.size} source files to the target.") - clonedAdds.foreach { a => + .collect { case add: AddFile => add } + .groupBy(_.snapshotId) + val cloneGroups = addsBySnapshotId.collect { + case (Some(snapshotId), adds) if adds.size >= srcBackRefByPath.size => snapshotId -> adds + } + assert(cloneGroups.size == 1, + s"expected one CLONE snapshot with at least ${srcBackRefByPath.size} files, " + + s"got ${cloneGroups.map { case (id, adds) => id -> adds.size }}.") + val (cloneSnapshotId, clonedSourceAdds) = cloneGroups.head + clonedSourceAdds.foreach { a => assert(a.backReference.isEmpty, s"Cloned AddFile ${a.path} must not carry the source table's back reference.") + assert(!srcSnapshotIds.contains(cloneSnapshotId), + s"Cloned AddFile ${a.path} must start a new snapshot lifecycle.") + assert(a.modifiedSnapshotId.isEmpty) } } } @@ -844,6 +854,10 @@ class AMTBackReferenceSuite val name = context.tableName // The version RESTORE returns to, where every seeded file is stamped from a leaf. val restoredToByPath = stampedBackRefs(context.postCheckpointSnapshot) + val restoredToFiles = liveAddFiles(context.postCheckpointSnapshot) + assert(restoredToFiles.forall(_.snapshotId.isDefined), + "precondition: every restored-to file must have snapshot provenance.") + val restoredToSnapshotIds = restoredToFiles.flatMap(_.snapshotId).toSet val deltaLog = deltaLogForName(name) val restoreTarget = context.checkpoint.version @@ -884,6 +898,16 @@ class AMTBackReferenceSuite } assert(readded.size == restoredToByPath.size, s"RESTORE must re-add all ${restoredToByPath.size} restored-to files, saw ${readded.size}.") + val restoreSnapshotIds = readded.map(_.snapshotId).toSet + assert(restoreSnapshotIds.size == 1 && restoreSnapshotIds.head.isDefined, + s"RESTORE must assign one new random snapshot ID; got $restoreSnapshotIds.") + val restoreSnapshotId = restoreSnapshotIds.head.get + assert(!restoredToSnapshotIds.contains(restoreSnapshotId), + s"RESTORE snapshot ID $restoreSnapshotId must differ from restored file IDs " + + s"$restoredToSnapshotIds.") + readded.foreach { add => + assert(add.modifiedSnapshotId.isEmpty) + } /* // RESTORE's needs to recompute the back references for the re-added files, so we cannot simply // assert their emptiness directly. diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTCheckpointWriteSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTCheckpointWriteSuite.scala index c7613d33d87..76f4dae57da 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTCheckpointWriteSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTCheckpointWriteSuite.scala @@ -993,7 +993,7 @@ class AMTCheckpointWriteSuite extends AMTCheckpointTestBase { private def addedTracking: Tracking = Tracking( status = Tracking.Status.Added, snapshot_id = None, - dv_snapshot_id = None, + modified_snapshot_id = None, sequence_number = None, file_sequence_number = None, first_row_id = None, diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTIncrementalWriteInvariantSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTIncrementalWriteInvariantSuite.scala index aca46e373ba..31cb921cea4 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTIncrementalWriteInvariantSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTIncrementalWriteInvariantSuite.scala @@ -177,7 +177,7 @@ class AMTIncrementalWriteInvariantSuite extends AMTIncrementalWriteTestBase { file_format = AMTSingleAction.FileFormatParquet, tracking = Tracking( status = Tracking.Status.Existing, - snapshot_id = None, dv_snapshot_id = None, sequence_number = None, + snapshot_id = None, modified_snapshot_id = None, sequence_number = None, file_sequence_number = None, first_row_id = None, deleted_positions = None, replaced_positions = None), record_count = diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTInheritanceReadSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTInheritanceReadSuite.scala index c33b269fa3a..5554a51ee32 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTInheritanceReadSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTInheritanceReadSuite.scala @@ -131,7 +131,7 @@ class AMTInheritanceReadSuite extends AMTCheckpointTestBase { private def unassignedTracking(status: Int = Tracking.Status.Added): Tracking = Tracking( status = status, snapshot_id = None, - dv_snapshot_id = None, + modified_snapshot_id = None, sequence_number = None, file_sequence_number = None, first_row_id = None, diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTInheritanceSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTInheritanceSuite.scala index 579bb6e0262..05e85db1ac1 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTInheritanceSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTInheritanceSuite.scala @@ -35,7 +35,7 @@ class AMTInheritanceSuite extends SparkFunSuite { private def childTracking( status: Int = Tracking.Status.Added, snapshotId: Option[Long] = None, - dvSnapshotId: Option[Long] = None, + modifiedSnapshotId: Option[Long] = None, sequenceNumber: Option[Long] = None, fileSequenceNumber: Option[Long] = None, firstRowId: Option[Long] = None, @@ -43,7 +43,7 @@ class AMTInheritanceSuite extends SparkFunSuite { replacedPositions: Option[Array[Byte]] = None): Tracking = Tracking( status = status, snapshot_id = snapshotId, - dv_snapshot_id = dvSnapshotId, + modified_snapshot_id = modifiedSnapshotId, sequence_number = sequenceNumber, file_sequence_number = fileSequenceNumber, first_row_id = firstRowId, @@ -232,7 +232,7 @@ class AMTInheritanceSuite extends SparkFunSuite { val parent = parentPointer(Tracking( status = Tracking.Status.Existing, snapshot_id = Some(7L), - dv_snapshot_id = Some(8L), + modified_snapshot_id = Some(8L), sequence_number = Some(42L), file_sequence_number = Some(43L), first_row_id = Some(1000L), diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTPartitionValuesSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTPartitionValuesSuite.scala index d362ad374d1..e70d7db36e8 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTPartitionValuesSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTPartitionValuesSuite.scala @@ -39,7 +39,7 @@ class AMTPartitionValuesSuite extends AMTCheckpointTestBase { private def addedTracking: Tracking = Tracking( status = Tracking.Status.Added, snapshot_id = None, - dv_snapshot_id = None, + modified_snapshot_id = None, sequence_number = None, file_sequence_number = None, first_row_id = None, diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTSingleActionSerializerSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTSingleActionSerializerSuite.scala index c3b88a2378e..cb216ab6625 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTSingleActionSerializerSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTSingleActionSerializerSuite.scala @@ -46,7 +46,7 @@ class AMTSingleActionSerializerSuite extends QueryTest with SharedSparkSession { private def addedTracking: Tracking = Tracking( status = Tracking.Status.Added, snapshot_id = None, - dv_snapshot_id = None, + modified_snapshot_id = None, sequence_number = None, file_sequence_number = None, first_row_id = None, @@ -57,7 +57,7 @@ class AMTSingleActionSerializerSuite extends QueryTest with SharedSparkSession { private def rootTracking: Tracking = Tracking( status = Tracking.Status.Existing, snapshot_id = Some(7L), - dv_snapshot_id = None, + modified_snapshot_id = None, sequence_number = Some(3L), file_sequence_number = Some(3L), first_row_id = None, @@ -412,7 +412,7 @@ class AMTSingleActionSerializerSuite extends QueryTest with SharedSparkSession { private def entryWithPassthrough: DataEntry = DataEntry( location = "f.parquet", file_format = AMTSingleAction.FileFormatParquet, - tracking = addedTracking, + tracking = addedTracking.copy(snapshot_id = Some(11L)), record_count = 10L, file_size_in_bytes = 100L, spec_id = Some(7), @@ -425,16 +425,23 @@ class AMTSingleActionSerializerSuite extends QueryTest with SharedSparkSession { location = "f.parquet", file_format = AMTSingleAction.FileFormatParquet, tracking = addedTracking, record_count = 10L, file_size_in_bytes = 100L).toAddFile(tableRoot) assert(add.amtPassthrough.isEmpty, "a default entry must add no per-file passthrough") + assert(add.snapshotId.isEmpty) + assert(add.modifiedSnapshotId.isEmpty) } test("DataEntry round-trips toAddFile -> fromAddFile") { val add = entryWithPassthrough.toAddFile(tableRoot) assert(add.amtPassthrough.isDefined, "a genuine passthrough must be carried") + assert(add.snapshotId.contains(11L)) + assert(add.modifiedSnapshotId.isEmpty) val restored = DataEntry.fromAddFile(add, addedTracking, tableRoot) assert(restored.spec_id.contains(7)) assert(restored.sort_order_id.contains(3)) assert(restored.key_metadata.exists(_.sameElements(Array[Byte](4, 5, 6)))) assert(restored.split_offsets.contains(Seq(0L, 64L, 128L))) + assert(restored.tracking.snapshot_id.contains(11L)) + assert(restored.tracking.modified_snapshot_id.isEmpty, + "modified_snapshot_id must be null when the entry has no modification metadata") // file_format / format_version are not carried; they are reconstructed at the M1 defaults. assert(restored.file_format == AMTSingleAction.FileFormatParquet) assert(restored.format_version == AMTSingleAction.FormatVersionV4) @@ -447,6 +454,46 @@ class AMTSingleActionSerializerSuite extends QueryTest with SharedSparkSession { assert(restored.format_version == AMTSingleAction.FormatVersionV4) } + test("fromAddFile maps snapshot IDs only from top-level AddFile fields") { + val transitionTracking = + addedTracking.copy(snapshot_id = Some(99L), modified_snapshot_id = Some(100L)) + val withoutPassthrough = DataEntry.fromAddFile(sampleAddFile, transitionTracking, tableRoot) + assert(withoutPassthrough.tracking.snapshot_id.isEmpty) + assert(withoutPassthrough.tracking.modified_snapshot_id.isEmpty) + + val dv = DeletionVectorDescriptor.onDiskWithUuidRelativePath( + id = UUID.randomUUID(), sizeInBytes = 10, cardinality = 1L, offset = Some(1)) + val add = sampleAddFile.copy( + deletionVector = dv, + snapshotId = Some(11L), + modifiedSnapshotId = Some(12L)) + val withPassthrough = DataEntry.fromAddFile(add, transitionTracking, tableRoot) + assert(withPassthrough.tracking.snapshot_id.contains(11L)) + assert(withPassthrough.tracking.modified_snapshot_id.contains(12L)) + } + + test("fromRemoveFile combines prior content with terminal snapshot provenance") { + val dv = DeletionVectorDescriptor.onDiskWithUuidRelativePath( + id = UUID.randomUUID(), sizeInBytes = 10, cardinality = 1L, offset = Some(1)) + val priorAdd = sampleAddFile.copy( + deletionVector = dv, + amtPassthrough = Some(AMTPassthrough(spec_id = Some(7))), + snapshotId = Some(11L), + modifiedSnapshotId = Some(12L)) + val remove = priorAdd.remove.copy( + snapshotId = Some(21L), + modifiedSnapshotId = Some(12L)) + // scalastyle:off removeFile // This calls a DataEntry factory, not a RemoveFile constructor. + val entry = DataEntry.fromRemoveFile( + priorAdd, remove, addedTracking.copy(status = Tracking.Status.Deleted), tableRoot) + // scalastyle:on removeFile + assert(entry.location == priorAdd.path) + assert(entry.record_count == priorAdd.numPhysicalRecords.get) + assert(entry.spec_id.contains(7)) + assert(entry.tracking.snapshot_id.contains(21L)) + assert(entry.tracking.modified_snapshot_id.contains(12L)) + } + test("fromAddFile maps defaultRowCommitVersion to both sequence numbers") { val entry = DataEntry.fromAddFile( sampleAddFile.copy(defaultRowCommitVersion = Some(10L)), @@ -512,7 +559,7 @@ class AMTSingleActionSerializerSuite extends QueryTest with SharedSparkSession { } } - test("amtPassthrough round-trips through the commit JSON") { + test("AMT AddFile fields round-trip through the commit JSON") { val add = entryWithPassthrough.toAddFile(tableRoot) assert(add.amtPassthrough.isDefined) val json = add.json @@ -520,6 +567,7 @@ class AMTSingleActionSerializerSuite extends QueryTest with SharedSparkSession { Seq("spec_id", "sort_order_id", "key_metadata", "split_offsets").foreach { f => assert(json.contains(f), s"$f must be serialized; got $json") } + assert(json.contains("\"snapshotId\":11")) val roundTripped = Action.fromJson(json) match { case a: AddFile => a @@ -531,6 +579,8 @@ class AMTSingleActionSerializerSuite extends QueryTest with SharedSparkSession { assert(passthrough.sort_order_id.contains(3)) assert(passthrough.key_metadata.exists(_.sameElements(Array[Byte](4, 5, 6)))) assert(passthrough.split_offsets.contains(Seq(0L, 64L, 128L))) + assert(roundTripped.snapshotId.contains(11L)) + assert(roundTripped.modifiedSnapshotId.isEmpty) } test("an AddFile with no amtPassthrough keeps it out of the commit JSON") { diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTSnapshotSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTSnapshotSuite.scala index f07057b5125..5b67735e5a8 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTSnapshotSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTSnapshotSuite.scala @@ -18,7 +18,7 @@ package org.apache.spark.sql.delta.amt import org.apache.spark.sql.delta.{Checkpoints, DeletionVectorsTestUtils, DeltaLog, DeltaOperations, Snapshot} import org.apache.spark.sql.delta.DeltaTestUtils.createTestAddFile -import org.apache.spark.sql.delta.actions.{AddFile, Checkpoint, ContentRoot} +import org.apache.spark.sql.delta.actions.{AddFile, Checkpoint, ContentRoot, RemoveFile} import org.apache.spark.sql.delta.actions.DeletionVectorDescriptor import org.apache.spark.sql.delta.deletionvectors.ManifestBitmap import org.apache.spark.sql.delta.deletionvectors.RoaringBitmapArray @@ -56,18 +56,137 @@ class AMTSnapshotSuite extends AMTCheckpointTestBase with DeletionVectorsTestUti } testAcrossAMTCheckpointScenarios( - "plain AMT tables reconstruct AddFiles with no amtPassthrough", + "plain AMT tables reconstruct AddFiles with snapshot provenance", "amt_passthrough_roundtrip")( setup = name => sql(s"INSERT INTO $name VALUES (1)"), inlineCheckpointTriggerActionsOrSQL = Some(name => Right( s"INSERT INTO $name VALUES (2)"))) { context => - // These are plain parquet/v4 files with no Iceberg passthrough, so `toAddFile` carries - // nothing (avoids per-file overhead). + // Both INSERTs create plain data files. Reconstruction must retain each file's creation SID, + // while modifiedSnapshotId stays empty because neither file has modification metadata. val files = context.postCheckpointSnapshot.allFiles.collect() assert(files.nonEmpty) - assert(files.forall(_.amtPassthrough.isEmpty), - s"plain AMT files must carry no passthrough; got " + - s"${files.map(_.amtPassthrough).mkString(", ")}") + assert(files.forall(_.snapshotId.isDefined), + s"every AMT file must carry snapshot provenance; got " + + s"${files.map(file => (file.snapshotId, file.modifiedSnapshotId)).mkString(", ")}") + assert(files.forall(_.modifiedSnapshotId.isEmpty)) + } + + testAcrossAMTCheckpointScenarios( + "snapshot provenance tracks the first DV update", + "amt_snapshot_id_first_dv")( + setup = name => + Seq(1, 2).toDF("id").coalesce(1).write.mode("append").insertInto(name), + inlineCheckpointTriggerActionsOrSQL = Some(name => Right( + s"DELETE FROM $name WHERE id = 1"))) { context => + // The first DELETE terminates the no-DV tuple and adds a DV to the same data file. The old + // tuple's removal and the new DV are created in the same snapshot. + val actions = actionsAt( + context.postCheckpointSnapshot.deltaLog, + context.checkpoint.version) + val add = actions.collectFirst { + case action: AddFile if action.deletionVector != null => action + }.getOrElse(fail("expected a superseding AddFile with a deletion vector.")) + val remove = actions.collectFirst { + case action: RemoveFile if action.path == add.path => action + }.getOrElse(fail("expected a paired RemoveFile for the DV update.")) + + assert(add.snapshotId.isDefined) + assert(remove.snapshotId.isDefined) + assert(add.modifiedSnapshotId == remove.snapshotId) + assert(remove.modifiedSnapshotId.isEmpty) + + // Verify that the top-level IDs survive the AddFile -> DataEntry -> AddFile round-trip. + val reconstructed = context.postCheckpointSnapshot.allFiles.collect() + .find(_.path == add.path) + .getOrElse(fail("the DV-bearing AddFile must reconstruct from the AMT.")) + assert(reconstructed.snapshotId == add.snapshotId) + assert(reconstructed.modifiedSnapshotId == add.modifiedSnapshotId) + assert(reconstructed.amtPassthrough == add.amtPassthrough) + } + + testAcrossAMTCheckpointScenarios( + "snapshot provenance tracks a repeated DV update", + "amt_snapshot_id_repeated_dv")( + setup = name => + Seq(1, 2, 3).toDF("id").coalesce(1).write.mode("append").insertInto(name), + inlineCheckpointTriggerActionsOrSQL = Some(name => Right( + s"DELETE FROM $name WHERE id = 1"))) { context => + val deltaLog = context.postCheckpointSnapshot.deltaLog + val priorAdd = context.postCheckpointSnapshot.allFiles.collect() + .find(_.deletionVector != null) + .getOrElse(fail("expected the first DELETE to produce a deletion vector.")) + assert(priorAdd.snapshotId.isDefined) + assert(priorAdd.modifiedSnapshotId.isDefined) + + // A second DELETE preserves the data-file lifecycle, retires the old DV, and creates a new DV. + val versionBefore = deltaLog.update().version + sql(s"DELETE FROM ${context.tableName} WHERE id = 2") + val versionAfter = deltaLog.update().version + assert(versionAfter == versionBefore + 1) + val actions = actionsAt(deltaLog, versionAfter) + val add = actions.collectFirst { + case action: AddFile if action.path == priorAdd.path => action + }.getOrElse(fail("expected a superseding AddFile for the repeated DV update.")) + val remove = actions.collectFirst { + case action: RemoveFile if action.path == priorAdd.path => action + }.getOrElse(fail("expected a paired RemoveFile for the repeated DV update.")) + + // The AddFile keeps the data-file SID. The RemoveFile keeps the old modification SID, while + // its removal SID becomes the new AddFile's modification SID. + assert(add.snapshotId == priorAdd.snapshotId) + assert(remove.modifiedSnapshotId == priorAdd.modifiedSnapshotId) + assert(remove.snapshotId.isDefined) + assert(add.modifiedSnapshotId == remove.snapshotId) + } + + testAcrossAMTCheckpointScenarios( + "metadata-only re-add assigns missing snapshot IDs without emitting CDF", + "amt_snapshot_id_metadata_readd")( + setup = name => + Seq(1, 2).toDF("id").coalesce(1).write.mode("append").insertInto(name), + inlineCheckpointTriggerActionsOrSQL = Some(name => Right( + s"DELETE FROM $name WHERE id = 1"))) { context => + val deltaLog = context.postCheckpointSnapshot.deltaLog + val current = context.postCheckpointSnapshot.allFiles.collect() + .find(_.deletionVector != null) + .getOrElse(fail("expected the first DELETE to produce a deletion vector.")) + val versionBefore = deltaLog.update().version + + // Simulate a metadata-only re-add of an upgraded file whose provenance was not materialized. + // The file and DV tuple remain unchanged, so this should only fill missing provenance. + withSQLConf( + DeltaSQLConf.AMT_LARGE_COMMIT_ACTIONS_COUNT_THRESHOLD_FOR_INLINE_MANIFEST_COMMIT.key -> + "1") { + deltaLog.startTransaction().commit( + Seq(current.copy(dataChange = false, snapshotId = None, modifiedSnapshotId = None)), + DeltaOperations.ComputeStats(predicate = Nil)) + } + + val committed = actionsAt(deltaLog, versionBefore + 1).collectFirst { + case add: AddFile if add.path == current.path => add + }.getOrElse(fail("expected the metadata-only AddFile.")) + assert(committed.snapshotId.isDefined) + assert(committed.modifiedSnapshotId == committed.snapshotId) + + val snapshot = deltaLog.update() + val provider = amtProvider(snapshot).getOrElse(fail("expected an inline AMT checkpoint.")) + val root = provider.checkpointAction.contentRoot.getAbsolutePath(deltaLog.dataPath).toString + withManifestDataEntries(Seq(root)) { entries => + // Since the file and DV tuple did not change, the manifest entry remains EXISTING and does + // not carry deletion or replacement positions. + val tracking = entries.select( + col("tracking.status"), + col("tracking.snapshot_id"), + col("tracking.modified_snapshot_id"), + col("tracking.deleted_positions"), + col("tracking.replaced_positions")).collect() + assert(tracking.length == 1) + assert(tracking.head.getInt(0) == Tracking.Status.Existing) + assert(tracking.head.getLong(1) == committed.snapshotId.get) + assert(tracking.head.getLong(2) == committed.modifiedSnapshotId.get) + assert(tracking.head.isNullAt(3)) + assert(tracking.head.isNullAt(4)) + } } testAcrossAMTCheckpointScenarios( @@ -1359,7 +1478,7 @@ class AMTSnapshotSuite extends AMTCheckpointTestBase with DeletionVectorsTestUti private def addedTracking: Tracking = Tracking( status = Tracking.Status.Added, snapshot_id = None, - dv_snapshot_id = None, + modified_snapshot_id = None, sequence_number = None, file_sequence_number = None, first_row_id = None, diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTUtilsSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTUtilsSuite.scala index 7edd6c1fb4a..e697834af8d 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTUtilsSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTUtilsSuite.scala @@ -17,7 +17,13 @@ package org.apache.spark.sql.delta.amt import org.apache.spark.sql.delta.AdaptiveMetadataTableFeature -import org.apache.spark.sql.delta.actions.{Metadata, Protocol} +import org.apache.spark.sql.delta.actions.{ + AddFile, + DeletionVectorDescriptor, + Metadata, + Protocol, + RemoveFile +} import org.apache.hadoop.fs.Path import org.apache.spark.sql.QueryTest @@ -136,4 +142,92 @@ class AMTUtilsSuite extends QueryTest with SharedSparkSession { test("amtEnabled: false when the protocol lacks the AMT feature") { assert(!AMTUtils.amtEnabled(Metadata(), Protocol())) } + + test("AMTSnapshotId fills missing AddFile and RemoveFile lifecycle IDs") { + val protocol = Protocol.forTableFeature(AdaptiveMetadataTableFeature) + val dv = DeletionVectorDescriptor( + storageType = "u", + pathOrInlineDv = "test-dv", + offset = Some(1), + sizeInBytes = 10, + cardinality = 1) + val newFile = AddFile("new", Map.empty, 1L, 1L, dataChange = true) + val dvUpdate = AddFile( + "existing", Map.empty, 1L, 1L, dataChange = true, + deletionVector = dv, + snapshotId = Some(2L)) + val remove = AddFile( + "removed", Map.empty, 1L, 1L, dataChange = true, + deletionVector = dv, + snapshotId = Some(2L), + modifiedSnapshotId = Some(3L)).remove + val removeWithAssignedSnapshotId = remove.copy(snapshotId = Some(7L)) + val unstampedNoDataChangeRemove = + remove.copy(dataChange = false, snapshotId = None, modifiedSnapshotId = None) + val unstampedMetadataReAdd = + dvUpdate.copy(dataChange = false, snapshotId = None, modifiedSnapshotId = None) + + val assigned = AMTSnapshotId.assignIfMissing( + Metadata(), protocol, + Iterator( + newFile, + dvUpdate, + remove, + removeWithAssignedSnapshotId, + unstampedNoDataChangeRemove, + unstampedMetadataReAdd), + snapshotId = 5L).toSeq + val assignedNew = assigned(0).asInstanceOf[AddFile] + assert(assignedNew.snapshotId.contains(5L)) + assert(assignedNew.modifiedSnapshotId.isEmpty) + + val assignedDvUpdate = assigned(1).asInstanceOf[AddFile] + assert(assignedDvUpdate.snapshotId.contains(2L)) + assert(assignedDvUpdate.modifiedSnapshotId.contains(5L)) + + val assignedRemove = assigned(2).asInstanceOf[RemoveFile] + assert(assignedRemove.snapshotId.contains(5L)) + assert(assignedRemove.modifiedSnapshotId.contains(3L)) + + val preservedRemove = assigned(3).asInstanceOf[RemoveFile] + assert(preservedRemove.snapshotId.contains(7L)) + assert(preservedRemove.modifiedSnapshotId.contains(3L)) + + val assignedNoDataChangeRemove = assigned(4).asInstanceOf[RemoveFile] + assert(assignedNoDataChangeRemove.snapshotId.contains(5L)) + assert(assignedNoDataChangeRemove.modifiedSnapshotId.contains(5L)) + + val assignedMetadataReAdd = assigned(5).asInstanceOf[AddFile] + assert(assignedMetadataReAdd.snapshotId.contains(5L)) + assert(assignedMetadataReAdd.modifiedSnapshotId.contains(5L)) + } + + test("AMTSnapshotId preserves a materialized modification ID") { + val protocol = Protocol.forTableFeature(AdaptiveMetadataTableFeature) + val add = AddFile( + "materialized", + Map.empty, + 1L, + 1L, + dataChange = true, + modifiedSnapshotId = Some(3L)) + val remove = add.remove + + val assigned = AMTSnapshotId.assignIfMissing( + Metadata(), protocol, Iterator(add, remove), snapshotId = 5L).toSeq + assert(assigned(0).asInstanceOf[AddFile].modifiedSnapshotId.contains(3L)) + assert(assigned(1).asInstanceOf[RemoveFile].modifiedSnapshotId.contains(3L)) + } + + + test("AMTSnapshotId generates a non-negative random ID") { + assert(AMTSnapshotId.generate() >= 0L) + } + + test("AMTSnapshotId leaves actions unchanged when AMT is disabled") { + val add = AddFile("plain", Map.empty, 1L, 1L, dataChange = true) + val assigned = AMTSnapshotId.assignIfMissing( + Metadata(), Protocol(), Iterator(add), snapshotId = 5L).toSeq + assert(assigned == Seq(add)) + } } From 72b563515251e26a51571d80d62661a3af3b2835 Mon Sep 17 00:00:00 2001 From: Murali Ramanujam Date: Mon, 12 Oct 2026 19:48:21 -0700 Subject: [PATCH 08/29] [Spark] Avoid file planning for unfiltered Delta V2 size statistics (#7852) ## Description Spark's optimizer can request scan statistics (`estimateSizeInBytes` / `estimateStatistics`) before physical planning. Previously the Delta V2 scan always selected files to answer these requests, even for unfiltered scans where the result is already known. For an unfiltered scan (no data filters, no partition filters, no pushed limit, not a CDC read), the snapshot checksum already records the exact size of every file the scan would select. The scan now returns that size directly before planning, so an early optimizer statistics request no longer triggers file selection. Physical planning stays lazy and still selects files exactly once. Filtered, limit, and CDC scans continue to plan files so Spark receives their accurate post-pruning size. If the checksum does not record a table size, the scan falls back to planning rather than reporting an unknown size. ## How was this patch tested? Added tests to `DeltaV2ScanTest`: - Unfiltered scans report statistics without selecting files, and later physical planning selects files exactly once and reuses that selection. - Filtered scans and positive-limit scans plan files exactly once and reuse the selection for repeated statistics requests. - Tables written without checksums fall back to file planning for the size estimate. Existing statistics tests were updated to reflect that full statistics for unfiltered scans now come from snapshot metadata. ## Does this PR introduce _any_ user-facing changes? No. --- .../spark/internal/v2/read/DeltaV2Scan.java | 45 ++++-- .../internal/v2/read/DeltaV2ScanTest.java | 131 ++++++++++++++++-- 2 files changed, 154 insertions(+), 22 deletions(-) diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/DeltaV2Scan.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/DeltaV2Scan.java index bc5330eda98..fec73923ea4 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/DeltaV2Scan.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/DeltaV2Scan.java @@ -306,10 +306,7 @@ public String description() { } /** - * Returns the catalog size when available, or the compression-adjusted post-pruning file size, - * without constructing row or column statistics. - * - *

Intentionally no {@code @Override}: Delta also compiles this source against supported Spark + * Intentionally no {@code @Override}: Delta also compiles this source against supported Spark * versions whose {@link SupportsReportStatistics} interface predates this optional method. The * method still overrides the interface default when Delta is built against a Spark version that * provides it. @@ -318,7 +315,13 @@ public OptionalLong estimateSizeInBytes() { return estimateSizeInBytesInternal(); } - /** Package-private entry point for Scala wrappers compiled together with this Java source. */ + /** + * Package-private entry point for Scala wrappers compiled together with this Java source. + * + *

Returns the catalog size when available, otherwise a file-size estimate without constructing + * row or column statistics. Before planning the estimate uses snapshot metadata; afterwards it + * uses the selected files. + */ OptionalLong estimateSizeInBytesInternal() { if (isZeroLimit()) { return OptionalLong.of(0L); @@ -356,13 +359,22 @@ private static long selectedFileSizeInBytes(DeltaScan deltaScan) { return totalBytes; } - private OptionalLong estimateSelectedFileSizeInBytes() { - // Like PreparedDeltaFileIndex.sizeInBytes, this is the selected scan's size, not the - // full-snapshot checksum size. Reading it may trigger file selection, but not input-file - // construction. + private synchronized OptionalLong estimateSelectedFileSizeInBytes() { + // TODO: Revisit this shortcut after coordinated scan preparation lands. + // An early unfiltered optimizer request does not need file selection: the snapshot checksum is + // already the exact size of every selected file. Filtered, limit, and CDC scans still plan here + // so Spark receives their accurate post-pruning size. If checksum bytes are unavailable, fall + // through to planning rather than replacing an accurate statistic with an unknown value. + if (!planned && canUseSnapshotSizeForUnfilteredEstimate()) { + OptionalLong tableSizeInBytes = snapshotSizeInBytesIfKnown(); + if (tableSizeInBytes.isPresent()) { + return tableSizeInBytes; + } + } + final DeltaScan deltaScan = preparedScan(); final long totalBytes = - toOptionalLong(deltaScan.scanned().bytesCompressed()) + ScalaUtils.toJavaOptionalLong(deltaScan.scanned().bytesCompressed()) .orElseGet(() -> selectedFileSizeInBytes(deltaScan)); // Do not scale the selected-file bytes by readSchema. Delta returns false from // reflectsFullyPushedDownFilters(), so Spark re-adds fully pushed filters when adjusting @@ -377,10 +389,15 @@ private boolean isZeroLimit() { return pushedLimit.isPresent() && pushedLimit.getAsInt() == 0; } - private static OptionalLong toOptionalLong(scala.Option value) { - return value.isDefined() - ? OptionalLong.of(((Number) value.get()).longValue()) - : OptionalLong.empty(); + private boolean canUseSnapshotSizeForUnfilteredEstimate() { + return !isCDCRead + && dataFilters.length == 0 + && partitionFilters.length == 0 + && !pushedLimit.isPresent(); + } + + private OptionalLong snapshotSizeInBytesIfKnown() { + return ScalaUtils.toJavaOptionalLong(initialSnapshot.sizeInBytesIfKnown()); } private Statistics statistics( diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/DeltaV2ScanTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/DeltaV2ScanTest.java index de8b32619f2..6414934ea9a 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/DeltaV2ScanTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/DeltaV2ScanTest.java @@ -15,6 +15,7 @@ import java.util.Map; import java.util.OptionalInt; import java.util.OptionalLong; +import java.util.function.Supplier; import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; import org.apache.spark.sql.catalyst.TableIdentifier; @@ -33,6 +34,7 @@ import org.apache.spark.sql.connector.read.colstats.ColumnStatistics; import org.apache.spark.sql.delta.Snapshot; import org.apache.spark.sql.delta.actions.AddFile; +import org.apache.spark.sql.delta.sources.DeltaSQLConf; import org.apache.spark.sql.execution.datasources.FilePartition; import org.apache.spark.sql.execution.datasources.PartitionedFile; import org.apache.spark.sql.types.DataTypes; @@ -595,7 +597,11 @@ public void testPlannedSnapshotTriggersPlanningAndReturnsSameSnapshot() throws E } private static long getTotalBytes(DeltaV2Scan scan) throws Exception { - scan.estimateStatistics(); // May select files unless catalog statistics are available. + scan.preparedScan(); + return getTotalBytesWithoutPlanning(scan); + } + + private static long getTotalBytesWithoutPlanning(DeltaV2Scan scan) throws Exception { if (!getPlanned(scan)) { return 0L; } @@ -634,8 +640,7 @@ private static long getTotalRows(DeltaV2Scan scan) throws Exception { } private static boolean isRowCountKnown(DeltaV2Scan scan) throws Exception { - scan.estimateStatistics(); - return getPlanned(scan) && scan.preparedScan().scanned().rows().isDefined(); + return scan.preparedScan().scanned().rows().isDefined(); } // ================================================================================================ @@ -696,7 +701,7 @@ public void testNumRowsAfterRuntimeFiltering() throws Exception { DeltaV2ScanBuilder builder = (DeltaV2ScanBuilder) table.newScanBuilder(options); DeltaV2Scan scan = (DeltaV2Scan) builder.build(); - scan.estimateStatistics(); + scan.toBatch(); assertEquals(5L, getTotalRows(scan), "5 rows before filtering"); // Two rows in the table have city=hz (Alice and Bob, in different date partitions). @@ -1056,6 +1061,113 @@ public void testEstimatedSizeMatchesStatistics() throws Exception { assertEquals(estimatedSizeFromSizeOnlyApi, estimatedSizeFromStats); } + @Test + public void testUnfilteredOrdinaryStatisticsDoNotTriggerFilePlanning() throws Exception { + DeltaV2ScanBuilder builder = (DeltaV2ScanBuilder) table.newScanBuilder(options); + DeltaV2Scan scan = (DeltaV2Scan) builder.build(); + int[] supplierCalls = instrumentDeltaScanSupplier(scan); + + OptionalLong sizeOnly = scan.estimateSizeInBytes(); + Statistics fullStatistics = scan.estimateStatistics(); + + assertTrue(sizeOnly.isPresent(), "the snapshot checksum should contain the table size"); + assertEquals(sizeOnly, fullStatistics.sizeInBytes()); + assertEquals(0, supplierCalls[0], "unfiltered ordinary stats must not select files"); + assertFalse(getPlanned(scan), "unfiltered ordinary stats must leave planning lazy"); + + scan.toBatch(); + + assertEquals(1, supplierCalls[0], "physical planning should select files exactly once"); + assertTrue(getPlanned(scan), "physical planning should still plan the scan"); + assertEquals( + 5, getPartitionedFiles(scan).size(), "the unfiltered scan should select all files"); + assertEquals(sizeOnly, scan.estimateSizeInBytes()); + assertEquals(1, supplierCalls[0], "reading planned statistics must reuse the selection"); + } + + @Test + public void testFilteredOrdinaryStatisticsPlanFilesOnce() throws Exception { + DeltaV2ScanBuilder builder = (DeltaV2ScanBuilder) table.newScanBuilder(options); + pushFilters(builder, new org.apache.spark.sql.sources.EqualTo("city", "hz")); + DeltaV2Scan scan = (DeltaV2Scan) builder.build(); + int[] supplierCalls = instrumentDeltaScanSupplier(scan); + + OptionalLong sizeOnly = scan.estimateSizeInBytes(); + + assertTrue(sizeOnly.isPresent()); + assertTrue(getPlanned(scan), "filtered ordinary stats must plan files"); + assertEquals(1, supplierCalls[0], "filtered ordinary stats should plan exactly once"); + assertEquals(2, getPartitionedFiles(scan).size(), "the pushed filter should select two files"); + assertEquals(sizeOnly, scan.estimateStatistics().sizeInBytes()); + assertEquals(1, supplierCalls[0], "repeated filtered stats must reuse file selection"); + } + + @Test + public void testMissingChecksumFallsBackToFilePlanning(@TempDir File tempDir) throws Exception { + String path = tempDir.getAbsolutePath(); + String tblName = "stats_without_checksum"; + withTable( + new String[] {tblName}, + () -> { + withSQLConf( + DeltaSQLConf.DELTA_WRITE_CHECKSUM_ENABLED().key(), + "false", + () -> { + spark.sql( + String.format( + "CREATE TABLE %s (id INT) USING delta LOCATION '%s'", tblName, path)); + spark.sql(String.format("INSERT INTO %s VALUES (1)", tblName)); + }); + + Identifier id = Identifier.of(new String[] {"default"}, tblName); + DeltaV2Table tableWithoutChecksum = new DeltaV2Table(id, path); + DeltaV2ScanBuilder builder = + (DeltaV2ScanBuilder) tableWithoutChecksum.newScanBuilder(options); + DeltaV2Scan scan = (DeltaV2Scan) builder.build(); + int[] supplierCalls = instrumentDeltaScanSupplier(scan); + + OptionalLong sizeOnly = scan.estimateSizeInBytes(); + + assertTrue(sizeOnly.isPresent(), "file planning should provide the missing size"); + assertTrue(getPlanned(scan), "missing checksum bytes must fall back to file planning"); + assertEquals(1, supplierCalls[0], "missing checksum bytes should plan exactly once"); + assertEquals(sizeOnly, scan.estimateStatistics().sizeInBytes()); + assertEquals(1, supplierCalls[0], "all later statistics must reuse file selection"); + }); + } + + @Test + public void testPositiveLimitStatisticsPlanFilesOnce() throws Exception { + DeltaV2Scan scan = buildScanWithLimit(1); + int[] supplierCalls = instrumentDeltaScanSupplier(scan); + + OptionalLong sizeOnly = scan.estimateSizeInBytes(); + + assertTrue(sizeOnly.isPresent()); + assertTrue(getPlanned(scan), "positive-limit statistics must plan files"); + assertEquals(1, supplierCalls[0], "positive-limit statistics should plan exactly once"); + assertEquals(1, getPartitionedFiles(scan).size(), "LIMIT 1 should select one file"); + assertEquals(sizeOnly, scan.estimateStatistics().sizeInBytes()); + assertEquals(1, supplierCalls[0], "repeated limit statistics must reuse file selection"); + } + + @SuppressWarnings("unchecked") + private static int[] instrumentDeltaScanSupplier(DeltaV2Scan scan) throws Exception { + Field supplierField = DeltaV2Scan.class.getDeclaredField("deltaScanSupplier"); + supplierField.setAccessible(true); + Supplier originalSupplier = + (Supplier) supplierField.get(scan); + int[] supplierCalls = new int[] {0}; + supplierField.set( + scan, + (Supplier) + () -> { + supplierCalls[0]++; + return originalSupplier.get(); + }); + return supplierCalls; + } + @Test public void testEstimateSizeInBytesUsesCheapFileSizePath() throws Exception { withSQLConf( @@ -1071,6 +1183,7 @@ public void testEstimateSizeInBytesUsesCheapFileSizePath() throws Exception { DeltaV2Scan scan = (DeltaV2Scan) builder.build(); long sizeInBytes = scan.estimateSizeInBytes().getAsLong(); + assertFalse(getPlanned(scan), "size estimation must not plan Delta files"); assertEquals(getTotalBytes(scan), sizeInBytes); assertTrue(sizeInBytes > 0, "post-pruning size should be positive"); assertFalse( @@ -1516,8 +1629,8 @@ public void testSizeOnlyEstimateDoesNotUseCatalogColumnStats(@TempDir File tempD public void testEstimateStatisticsWithCatalogStats_noNumRows(@TempDir File tempDir) throws Exception { // Catalog stats with sizeInBytes but no numRows: the size-only API should use the catalog - // size without planning files, while full statistics should use the planned file size and - // leave row and column statistics empty. + // size without planning files, while full statistics should use the snapshot size and leave + // row and column statistics empty. String path = tempDir.getAbsolutePath(); String tblName = "stats_no_numrows"; spark.sql( @@ -1559,7 +1672,7 @@ public void testEstimateStatisticsWithCatalogStats_noNumRows(@TempDir File tempD assertNotEquals( 99999L, stats.sizeInBytes().getAsLong(), - "sizeInBytes should come from planned files, not catalog stats"); + "sizeInBytes should come from snapshot metadata, not catalog stats"); assertTrue( stats.columnStats().isEmpty(), "columnStats should be empty when catalog stats have no column stats"); @@ -1700,7 +1813,9 @@ public void testLimitPushdown_limit0ReportsZeroNumRowsNotStaleCatalog(@TempDir F stats.numRows().getAsLong(), "LIMIT 0 must report 0 rows, not the stale catalog count (999)"); assertEquals( - 0L, getTotalBytes(scan), "LIMIT 0 should read zero bytes even with catalog stats"); + 0L, + getTotalBytesWithoutPlanning(scan), + "LIMIT 0 should read zero bytes even with catalog stats"); assertFalse(getPlanned(scan), "LIMIT 0 statistics should remain on the fast path"); }); } From a8c02bcc2bbdec6c05038b75f8a0451a5df53631 Mon Sep 17 00:00:00 2001 From: Omar Elhadidy Date: Tue, 13 Oct 2026 16:19:09 +0200 Subject: [PATCH 09/29] [SPARK] Block non determinisitic predicate pushdown on cdf nodes (#7847) #### Which Delta project/connector is this regarding? - [x] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [ ] Other (fill in here) ## Description Stop pushing down non deterministic predicates to CDF nodes, these predicates will get reevaluated at query level thus potentially returning inconsistent results. ## How was this patch tested? New UTs ## Does this PR introduce _any_ user-facing changes? No --------- Co-authored-by: Omar Elhadidy --- .../delta/commands/cdc/CDCReaderBase.scala | 5 + .../sql/delta/sources/DeltaSQLConf.scala | 9 ++ .../spark/sql/delta/DeltaCDCSuite.scala | 105 +++++++++++++++++- .../spark/sql/delta/DeltaTestUtils.scala | 9 ++ 4 files changed, 125 insertions(+), 3 deletions(-) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/commands/cdc/CDCReaderBase.scala b/spark/src/main/scala/org/apache/spark/sql/delta/commands/cdc/CDCReaderBase.scala index a91d8a5074b..686424ce65d 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/commands/cdc/CDCReaderBase.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/commands/cdc/CDCReaderBase.scala @@ -117,8 +117,13 @@ abstract class DeltaCDFRelationBase( val outputMap = df.queryExecution.analyzed.output.map(a => a.name -> a).toMap val projections = requiredColumns.map(a => Column(outputMap(a.name))) + lazy val skipNondeterministicFilters = + sqlContext.sparkSession.sessionState.conf.getConf( + DeltaSQLConf.DELTA_CDF_SKIP_NONDETERMINISTIC_FILTER_PUSHDOWN) val filter = Column( filters + // Spark evaluates nondeterministic predicates in a filter above the CDF scan. + .filter(predicate => predicate.deterministic || !skipNondeterministicFilters) .map(_.transform { case a: Attribute => outputMap(a.name) }) .reduceOption(And) .getOrElse(Literal.TrueLiteral) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala b/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala index 2a4d9453584..b70c5942779 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala @@ -2703,6 +2703,15 @@ trait DeltaSQLConfBase extends DeltaSQLConfUtils { .createWithDefault(false) } + val DELTA_CDF_SKIP_NONDETERMINISTIC_FILTER_PUSHDOWN = + buildConf("changeDataFeed.skipNondeterministicFilterPushdown.enabled") + .internal() + .doc("When true, V1 CDF scans leave nondeterministic predicates to Spark's outer filter. " + + "When false, they also evaluate these predicates internally, restoring the previous " + + "behavior.") + .booleanConf + .createWithDefault(true) + val DELTA_STREAMING_UNSAFE_READ_ON_INCOMPATIBLE_COLUMN_MAPPING_SCHEMA_CHANGES = buildConf("streaming.unsafeReadOnIncompatibleColumnMappingSchemaChanges.enabled") .doc( diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaCDCSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaCDCSuite.scala index 299e7451777..99a13e4618e 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaCDCSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaCDCSuite.scala @@ -36,9 +36,9 @@ import org.apache.spark.sql.delta.util.{DeltaCommitFileProvider, FileNames} import org.apache.hadoop.fs.Path import org.apache.spark.SparkConf -import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest, Row} +import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest, Row, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier -import org.apache.spark.sql.functions.{col, current_timestamp, floor, lit, unix_timestamp} +import org.apache.spark.sql.functions.{col, current_timestamp, floor, lit, rand, udf, unix_timestamp} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.streaming.StreamingQueryException import org.apache.spark.sql.test.SharedSparkSession @@ -51,7 +51,8 @@ abstract class DeltaCDCSuiteBase with CheckCDCAnswer with DeltaSQLCommandTest with CatalogOwnedTestBaseSuite - with DeltaSQLTestUtils { + with DeltaSQLTestUtils + with CDFNondeterministicFilterTests { import testImplicits._ @@ -110,6 +111,15 @@ abstract class DeltaCDCSuiteBase schemaMode: Option[DeltaBatchCDFSchemaMode] = Some(BatchCDFSchemaLegacy), readerOptions: Map[String, String] = Map.empty): DataFrame + override def readCDF( + spark: SparkSession, + tableName: String, + startVersion: Long, + endVersion: Long): DataFrame = { + cdcRead(new TableName(tableName), + StartingVersion(startVersion.toString), EndingVersion(endVersion.toString)) + } + /** Create table utility method */ def ctas(srcTbl: String, dstTbl: String, disableCDC: Boolean = false): Unit = { val readDf = cdcRead(new TableName(srcTbl), StartingVersion("0"), EndingVersion("1")) @@ -920,6 +930,93 @@ abstract class DeltaCDCSuiteBase } } +trait CDFNondeterministicFilterTests extends QueryTest with DeltaSQLTestUtils with CDCTestMixin { + + private val cdfFilterTestNumRows: Int = 1000 + + private def withCDFForFilterTest(f: DataFrame => Unit): Unit = { + val tableName = "cdf_filter_test" + withTable(tableName) { + spark.range(cdfFilterTestNumRows / 2).coalesce(1) + .write.format("delta").saveAsTable(tableName) + spark.range(cdfFilterTestNumRows / 2, cdfFilterTestNumRows).coalesce(1) + .write.format("delta").mode("append").saveAsTable(tableName) + f(readCDF(spark, tableName, startVersion = 0L, endVersion = 1L)) + } + } + + for ((caseName, keepRow, expectedRowCount) <- Seq[(String, Long => Boolean, Int)]( + ("all rows", _ => true, cdfFilterTestNumRows), + ("no rows", _ => false, 0), + ("even ids", id => id % 2 == 0, cdfFilterTestNumRows / 2))) { + test(s"CDF evaluates a nondeterministic predicate once - $caseName") { + withCDFForFilterTest { cdf => + val calls = spark.sparkContext.longAccumulator("cdfPredicateCalls") + val predicate = + udf((id: Long) => { + calls.add(1L) + keepRow(id) + }).asNondeterministic() + + // checkAnswer executes additional actions, so collect once before checking the calls. + // Leave id out of the output to check it is retained for the residual predicate. + val rows = cdf.filter(predicate(col("id"))).select("_change_type").collect().toSeq + assert(rows === Seq.fill(expectedRowCount)(Row("insert"))) + assert(calls.value === cdfFilterTestNumRows.toLong) + } + } + } + + test("CDF preserves legacy predicate evaluation when the fix is disabled") { + withSQLConf( + DeltaSQLConf.DELTA_CDF_SKIP_NONDETERMINISTIC_FILTER_PUSHDOWN.key -> "false", + DeltaSQLConf.DELTA_CHANGELOG_V2_ENABLED.key -> "true") { + withCDFForFilterTest { cdf => + val calls = spark.sparkContext.longAccumulator("cdfPredicateCalls") + val predicate = + udf((_: Long) => { + calls.add(1L) + true + }).asNondeterministic() + val rows = cdf.filter(predicate(col("id"))).select("id").collect().toSeq + + assert(rows.sortBy(_.getLong(0)) === + (0L until cdfFilterTestNumRows.toLong).map(id => Row(id))) + assert(calls.value === cdfFilterTestNumRows.toLong * 2) + } + } + } + + test("CDF combines deterministic filters with a nondeterministic predicate") { + withCDFForFilterTest { cdf => + val calls = spark.sparkContext.longAccumulator("cdfPredicateCalls") + val predicate = + udf((_: Long) => { + calls.add(1L) + true + }).asNondeterministic() + val rows = cdf.filter( + col("id") < lit(250L) && col("_commit_version") === lit(0L) && predicate(col("id"))) + .select("id", "_change_type", "_commit_version").collect().toSeq + + assert(rows.sortBy(_.getLong(0)) === (0L until 250L).map(id => Row(id, "insert", 0L))) + assert(calls.value === 250L) + } + } + + for (useDisjunction <- Seq(false, true)) { + test(s"CDF applies a random filter once - disjunction=$useDisjunction") { + withCDFForFilterTest { cdf => + val sample = rand(42L) < lit(0.5) + val predicate = if (useDisjunction) col("id") < lit(0L) || sample else sample + val rows = cdf.filter(predicate).select("id").collect().toSeq + assert(rows.size > 400 && rows.size < 600, + s"Expected a 50% sample of $cdfFilterTestNumRows rows, got ${rows.size}") + } + } + } +} + class DeltaCDCScalaSuite extends DeltaCDCSuiteBase { /** Single method to do all kinds of CDC reads */ @@ -1201,6 +1298,7 @@ trait ChangelogV2CdcReadMixin extends DeltaCDCSuiteBase with ChangelogSyntaxSupp // Tests that do not apply to the V2 changelog read path: // - V1-specific error semantics (they intercept V1 error codes that the V2 path does not raise) + // - rollback flag evaluation counts, which apply only to V1 CDF scans // - mixed timestamp/version boundaries (the V2 CHANGES grammar only allows VERSION..VERSION or // TIMESTAMP..TIMESTAMP) // - timestamp -> version resolution, unsupported on the JVM kernel backend today: @@ -1214,6 +1312,7 @@ trait ChangelogV2CdcReadMixin extends DeltaCDCSuiteBase with ChangelogSyntaxSupp // rowTracking and domainMetadata table features the V1 test does not expect. protected def excludedV2Exact: Set[String] = Set( "An error should be thrown when CDC is not enabled", + "CDF preserves legacy predicate evaluation when the fix is disabled", "aggregating non-numeric cdc data columns", "changes - end timestamp exceeding latest commit timestamp", "changes - only start is a timestamp", diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaTestUtils.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaTestUtils.scala index adff3ead518..e1c0a5e65de 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaTestUtils.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaTestUtils.scala @@ -100,6 +100,15 @@ object DeltaTestUtilsBase { trait CDCTestMixin { // Setting the spark Conf is left to the test implementation. + /** Reads through the CDF scan so tests also exercise scan filter handling. */ + def readCDF( + spark: SparkSession, + tableName: String, + startVersion: Long, + endVersion: Long): DataFrame = { + spark.sql(s"SELECT * FROM table_changes('$tableName', $startVersion, $endVersion)") + } + def computeCDC( spark: SparkSession, deltaLog: DeltaLog, From c327335bc85adcbef80bc810b27e3bc06cec80cc Mon Sep 17 00:00:00 2001 From: Jiayuan Chen Date: Tue, 13 Oct 2026 10:44:04 -0700 Subject: [PATCH 10/29] [Spark] Support object identity in DV uniqueFileId and handle unencoded relative DVs in VACUUM (#7853) #### Which Delta project/connector is this regarding? - [x] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [ ] Other (fill in here) ## Description A few more places that identify deletion vectors (DVs) now support object identity, so that the same DV blob is recognized whether it is described by a table-relative (`r`) descriptor or a UUID-based (`u`) descriptor. 1. `DeletionVectorDescriptor.uniqueFileId` was a lazy val that always used the legacy `storageType + pathOrInlineDv` identity. It is replaced by `uniqueFileId(tableRoot, useObjectIdentity)`, which mirrors `uniqueId`: with object identity it returns the normalized table-relative identity of the DV file, without the offset. The legacy value is kept as the private `legacyUniqueFileId`, which `legacyUniqueId` is built on. 2. `getDeletionVectorRelativePathAndSize`, used by VACUUM, now treats unencoded relative (`r`) DVs like UUID-based (`u`) ones and returns their table-relative path. Before this change, `r` DVs fell through to the absolute-path branch, which asserts that the DV path is absolute. 3. Adds a `FileAction.useDeletionVectorObjectIdentity(snapshot, spark)` overload and uses it in the OPTIMIZE check that compares candidate file sets. ## How was this patch tested? Existing tests. `DeletionVectorDescriptorSuite` and `AMTCheckpointTestBase` are updated for the new `uniqueFileId` signature. ## Does this PR introduce _any_ user-facing changes? No. --- .../actions/DeletionVectorDescriptor.scala | 23 ++++++++++++++++--- .../spark/sql/delta/actions/actions.scala | 11 +++++++++ .../delta/commands/OptimizeTableCommand.scala | 4 +--- .../sql/delta/commands/VacuumCommand.scala | 2 +- .../DeletionVectorDescriptorSuite.scala | 8 +++---- .../sql/delta/amt/AMTCheckpointTestBase.scala | 2 +- 6 files changed, 38 insertions(+), 12 deletions(-) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/actions/DeletionVectorDescriptor.scala b/spark/src/main/scala/org/apache/spark/sql/delta/actions/DeletionVectorDescriptor.scala index d00eec4331a..02652f49eb8 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/actions/DeletionVectorDescriptor.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/actions/DeletionVectorDescriptor.scala @@ -96,14 +96,17 @@ case class DeletionVectorDescriptor( @transient lazy val legacyUniqueId: String = { offset match { - case Some(offset) => s"$uniqueFileId@$offset" - case None => uniqueFileId + case Some(offset) => s"$legacyUniqueFileId@$offset" + case None => legacyUniqueFileId } } + /** + * The legacy unique id of the file storing the DV. + */ @JsonIgnore @transient - lazy val uniqueFileId: String = s"$storageType$pathOrInlineDv" + private lazy val legacyUniqueFileId: String = s"$storageType$pathOrInlineDv" /** * Unique identifier for this DV. @@ -120,6 +123,20 @@ case class DeletionVectorDescriptor( if (useObjectIdentity) normalizedTableRelativeObjectId(tableRoot) else legacyUniqueId } + /** + * Unique identifier for the file storing this DV, i.e. [[uniqueId]] without the offset. Use this + * when grouping by DV file rather than by individual DV, since several DVs can be stored in one + * file at different offsets. See [[uniqueId]] for the `useObjectIdentity` semantics. + */ + def uniqueFileId(tableRoot: Path, useObjectIdentity: Boolean): String = { + if (useObjectIdentity) { + val (normalizedStorageType, normalizedPath) = normalizedTableRelativeObjectFile(tableRoot) + formatIdentity(normalizedStorageType, normalizedPath, offset = None) + } else { + legacyUniqueFileId + } + } + @JsonIgnore protected[delta] def isOnDisk: Boolean = !isInline diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/actions/actions.scala b/spark/src/main/scala/org/apache/spark/sql/delta/actions/actions.scala index 6f6fff9061d..1bef2086090 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/actions/actions.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/actions/actions.scala @@ -864,6 +864,17 @@ object FileAction { AMTUtils.amtEnabled(metadata, protocol) || spark.conf.get(DeltaSQLConf.DELETION_VECTORS_USE_OBJECT_IDENTITY_FOR_NON_AMT) + /** + * The same as the `(metadata, protocol, spark)` overload, for the metadata and protocol of + * `snapshot`. + * + * @param snapshot the table snapshot. + */ + def useDeletionVectorObjectIdentity( + snapshot: SnapshotDescriptor, + spark: SparkSession): Boolean = + useDeletionVectorObjectIdentity(snapshot.metadata, snapshot.protocol, spark) + /** * The unique tuple `(parquet file, deletion vector id)` of a file action, as a method on * [[AddFile]] / [[RemoveFile]]. See [[DeletionVectorDescriptor.uniqueId]] for the diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala b/spark/src/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala index 4f186379eab..4358b54ec51 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala @@ -475,9 +475,7 @@ class OptimizeExecutor( val newPartitionSchema = newTxn.metadata.partitionSchema // Note: When checking if the candidate set is the same, we need to consider (Path, DV) // as the key. - val useObjectIdentity = - FileAction.useDeletionVectorObjectIdentity( - snapshot.metadata, snapshot.protocol, sparkSession) + val useObjectIdentity = FileAction.useDeletionVectorObjectIdentity(snapshot, sparkSession) val candidateSetOld = filesToProcess .map(f => f.toUniqueFileActionTuple(snapshot.dataPath, useObjectIdentity)) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/commands/VacuumCommand.scala b/spark/src/main/scala/org/apache/spark/sql/delta/commands/VacuumCommand.scala index 9a7038cb0c9..2ea092ef663 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/commands/VacuumCommand.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/commands/VacuumCommand.scala @@ -744,7 +744,7 @@ trait VacuumCommandImpl extends DeltaCommand { dv match { case Some(dv) if dv.isOnDisk => - if (dv.isUuidRelative) { + if (dv.isUuidRelative || dv.isUnencodedRelative) { // We actually want a relative path here. Some((pathToUrlEncodedString(dv.absolutePath(new Path("."))), dv.sizeInBytes)) } else { diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/actions/DeletionVectorDescriptorSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/actions/DeletionVectorDescriptorSuite.scala index 41d8e674403..10f901d077c 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/actions/DeletionVectorDescriptorSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/actions/DeletionVectorDescriptorSuite.scala @@ -48,7 +48,7 @@ class DeletionVectorDescriptorSuite extends SparkFunSuite { // Unique id to identify the DV assert(dv.legacyUniqueId === s"i$encodedDVData") - assert(dv.uniqueFileId === s"i$encodedDVData") + assert(dv.uniqueFileId(testTablePath, useObjectIdentity = false) === s"i$encodedDVData") // There is no on-disk file name for an inline DV intercept[IllegalArgumentException] { dv.absolutePath(testTablePath) } @@ -77,7 +77,7 @@ class DeletionVectorDescriptorSuite extends SparkFunSuite { // Unique id to identify the DV val offsetSuffix = offset.map(o => s"@$o").getOrElse("") assert(dv.legacyUniqueId === s"p$testDVAbsPath$offsetSuffix") - assert(dv.uniqueFileId === s"p$testDVAbsPath") + assert(dv.uniqueFileId(testTablePath, useObjectIdentity = false) === s"p$testDVAbsPath") // Given the input already has an absolute path, it should return the path in DV assert(dv.absolutePath(testTablePath) === new Path(testDVAbsPath)) @@ -114,7 +114,7 @@ class DeletionVectorDescriptorSuite extends SparkFunSuite { val offsetSuffix = offset.map(o => s"@$o").getOrElse("") val encodedUUID = encodeUUID(uuid, "prefix") assert(dv.legacyUniqueId === s"u$encodedUUID$offsetSuffix") - assert(dv.uniqueFileId === s"u$encodedUUID") + assert(dv.uniqueFileId(testTablePath, useObjectIdentity = false) === s"u$encodedUUID") // Expect the DV final path to be under the given table path assert(dv.absolutePath(testTablePath) === @@ -158,7 +158,7 @@ class DeletionVectorDescriptorSuite extends SparkFunSuite { // Unique id to identify the DV val offsetSuffix = offset.map(o => s"@$o").getOrElse("") assert(dv.legacyUniqueId === s"r$testDVRelPath$offsetSuffix") - assert(dv.uniqueFileId === s"r$testDVRelPath") + assert(dv.uniqueFileId(testTablePath, useObjectIdentity = false) === s"r$testDVRelPath") // Expect the DV path to resolve under the given table path. assert(dv.absolutePath(testTablePath) === new Path(s"$testTablePath/$testDVRelPath")) diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTCheckpointTestBase.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTCheckpointTestBase.scala index 8cc7e13b0d6..22b3e19731e 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTCheckpointTestBase.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTCheckpointTestBase.scala @@ -632,7 +632,7 @@ trait AMTCheckpointTestBase Option(add.stats).map(JsonUtils.mapper.readTree), Option(add.deletionVector).map { dv => val physical = - if (dv.storageType == "i") dv.uniqueFileId + if (dv.storageType == "i") dv.uniqueFileId(tableLocation, useObjectIdentity = true) else dv.absolutePath(tableLocation).toString (physical, dv.offset, dv.sizeInBytes, dv.cardinality, dv.maxRowIndex) }) From 26ca1b9a6f741f402148a5754c2eaf82e51998b7 Mon Sep 17 00:00:00 2001 From: Anoop Johnson Date: Tue, 13 Oct 2026 11:15:18 -0700 Subject: [PATCH 11/29] [PROTOCOL] Use Mumbling bitmap encoding for manifest deletion vectors (#7850) #### Which Delta project/connector is this regarding? - [ ] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [x] V4 RFC ## Description Align the adaptive metadata tree RFC's manifest deletion vector encoding to use the Mumbling bitmap spec in place of Roaring. ## How was this patch tested? NA ## Does this PR introduce _any_ user-facing changes? No --- protocol_rfcs/iceberg-v4-metadata.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/protocol_rfcs/iceberg-v4-metadata.md b/protocol_rfcs/iceberg-v4-metadata.md index f7f6672a2cb..ae3776a1b68 100644 --- a/protocol_rfcs/iceberg-v4-metadata.md +++ b/protocol_rfcs/iceberg-v4-metadata.md @@ -424,7 +424,7 @@ Summary information for `content_type` = DATA_MANIFEST entries. Includes file/ro ### Manifest Deletion Vectors (MDVs) -MDVs track which entries in a leaf manifest have been deleted or replaced. The `dv` field in `manifest_info` contains a Roaring bitmap of all invalidated positions (cumulative, used for reading). The `deleted_positions` and `replaced_positions` fields in `tracking` represent what changed in the current commit only (used for CDF): +MDVs track which entries in a leaf manifest have been deleted or replaced. The `dv` field in `manifest_info` contains all invalidated positions (cumulative, used for reading). The `deleted_positions` and `replaced_positions` fields in `tracking` represent what changed in the current commit only (used for CDF). All three bitmaps are encoded using the [Mumbling bitmap](https://iceberg.apache.org/mumbling-spec/) spec: - **`manifest_info.dv`**: Cumulative set of all invalidated positions. Grows monotonically. Used by readers to skip entries. From 79af89ad12d09f0c1528ebac2edc30607d6b67a0 Mon Sep 17 00:00:00 2001 From: Prakhar Jain Date: Tue, 13 Oct 2026 11:58:34 -0700 Subject: [PATCH 12/29] [Spark] Back AMT manifest deletion vectors with the Mumbling bitmap format (#7855) The AMT manifest deletion vectors (the masking MDV in `manifest_info.dv` and the CDF `tracking.deleted_positions` / `replaced_positions`) were backed by a Roaring bitmap. This switches `ManifestBitmap` to an implementation backed by the immutable Mumbling bitmap format: `MumblingManifestBitmap` wraps a `MumblingBitmap`, and its mutable counterpart `MutableMumblingManifestBitmap` wraps a `MutableMumblingBitmap`, so `ManifestBitmap.fromPositions` / `fromSerializedByteArray` now produce and consume the Mumbling on-disk bytes. It also adds `MumblingRowIndexFilter` and `MumblingRowIndexFilterProvider`: a `RowIndexFilter` that drops a leaf row when its row index is contained in the MDV and keeps it otherwise, probing the bitmap per row (leaf row indices are dense and small, matching the point-membership shape of the Mumbling bitmap). The provider is self-contained and serializable so a scan task owns its own filter. The test-only position-dump helper (`toArrayForTesting`) is removed; the manifest-bitmap and AMT incremental-write tests now assert through `cardinality` / `contains` and exercise all three `RowIndexFilter` materialization entry points. --- .../deletionvectors/ManifestBitmap.scala | 94 ++++++------------- .../MumblingRowIndexFilter.scala | 89 ++++++++++++++++++ .../amt/AMTIncrementalWriteInlineSuite.scala | 24 ++--- .../amt/AMTIncrementalWriteTestBase.scala | 29 +++--- .../deletionvectors/ManifestBitmapSuite.scala | 93 ++++++++++++++++-- 5 files changed, 234 insertions(+), 95 deletions(-) create mode 100644 spark/src/main/scala/org/apache/spark/sql/delta/deletionvectors/MumblingRowIndexFilter.scala diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/deletionvectors/ManifestBitmap.scala b/spark/src/main/scala/org/apache/spark/sql/delta/deletionvectors/ManifestBitmap.scala index 0c0d1002d9d..7bfc1fa45d5 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/deletionvectors/ManifestBitmap.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/deletionvectors/ManifestBitmap.scala @@ -16,13 +16,14 @@ package org.apache.spark.sql.delta.deletionvectors +import java.nio.ByteBuffer + import scala.collection.Seq import org.apache.spark.sql.delta.RowIndexFilterProvider +import org.apache.spark.sql.delta.deletionvectors.mumbling.{MumblingBitmap, MumblingBitmapWriter} import org.apache.hadoop.fs.Path -import org.apache.spark.util.Utils - /** * An immutable bitmap of unsigned positions for the AMT manifest deletion vectors: the masking * MDV in `manifest_info.dv` and the CDF `tracking.deleted_positions` / @@ -44,14 +45,6 @@ trait ManifestBitmap { /** Serializes to the on-disk byte form carried in `manifest_info.dv` / the CDF bitmaps. */ def serializeAsByteArray(): Array[Byte] - /** The set positions in ascending order, for tests only. */ - final def toArrayForTesting: Array[Long] = { - assert(Utils.isTesting) - toArrayForTestingImpl - } - - protected def toArrayForTestingImpl: Array[Long] - /** Builds a [[RowIndexFilterProvider]] representing this bitmap. */ def toRowIndexFilterProvider(tableRoot: Path): RowIndexFilterProvider } @@ -68,80 +61,55 @@ trait MutableManifestBitmap { object ManifestBitmap { /** Returns a bitmap of the given positions. */ def fromPositions(values: Seq[Int]): ManifestBitmap = - RoaringManifestBitmap.fromPositions(values) + MumblingManifestBitmap.fromPositions(values) /** Deserializes a bitmap previously produced by [[ManifestBitmap.serializeAsByteArray]]. */ def fromSerializedByteArray(bytes: Array[Byte]): ManifestBitmap = - RoaringManifestBitmap.fromSerializedByteArray(bytes) + MumblingManifestBitmap.fromSerializedByteArray(bytes) } -/** - * A [[ManifestBitmap]] backed by a [[RoaringBitmapArray]] in the portable format. - */ -final class RoaringManifestBitmap private ( - private val serializedBytesOrPositions: Either[Array[Byte], Vector[Int]]) +/** A [[ManifestBitmap]] backed by an immutable [[MumblingBitmap]]. */ +final class MumblingManifestBitmap private[deletionvectors] ( + private val underlying: MumblingBitmap) extends ManifestBitmap { - // Materialized on first bit-level use: decoded from the on-disk bytes, or built from the - // positions. - private lazy val roaringBitmapArray: RoaringBitmapArray = - serializedBytesOrPositions match { - case Left(bytes) => RoaringBitmapArray.readFrom(bytes) - case Right(positions) => RoaringBitmapArray(positions.map(_.toLong): _*) - } + override def contains(value: Int): Boolean = value >= 0 && underlying.isSet(value) - override def contains(value: Int): Boolean = roaringBitmapArray.contains(value.toLong) + override def cardinality: Long = underlying.cardinality().toLong - override def cardinality: Long = roaringBitmapArray.cardinality - - override def isEmpty: Boolean = roaringBitmapArray.isEmpty + override def isEmpty: Boolean = underlying.cardinality() == 0 override def toMutable: MutableManifestBitmap = - MutableRoaringManifestBitmap.fromSerializedByteArray(serializeAsByteArray()) - - override protected def toArrayForTestingImpl: Array[Long] = roaringBitmapArray.toArray + new MutableMumblingManifestBitmap(underlying.toMutable()) - override def serializeAsByteArray(): Array[Byte] = - serializedBytesOrPositions match { - // Defensive copy: Don't expose internal buffer for a caller to mutate. - case Left(bytes) => bytes.clone() - case Right(_) => roaringBitmapArray.serializeAsByteArray(RoaringBitmapArrayFormat.Portable) - } - - override def toRowIndexFilterProvider(tableRoot: Path): RowIndexFilterProvider = { - // Capture serialized bytes so the scan task owns a self-contained provider. - val bytes = serializeAsByteArray() - RoaringManifestBitmap.Provider(bytes) + override def serializeAsByteArray(): Array[Byte] = { + val buffer = underlying.buffer() + val bytes = new Array[Byte](buffer.remaining()) + buffer.get(bytes) + bytes } + + override def toRowIndexFilterProvider(tableRoot: Path): RowIndexFilterProvider = + new MumblingRowIndexFilterProvider(serializeAsByteArray(), cardinality) } -object RoaringManifestBitmap { - private final case class Provider(serializedBytes: Array[Byte]) - extends RowIndexFilterProvider { - override def retrieve( - _hadoopConf: org.apache.hadoop.conf.Configuration): DropMarkedRowsFilter = - new DropMarkedRowsFilter(RoaringBitmapArray.readFrom(serializedBytes)) +object MumblingManifestBitmap { + def fromPositions(values: Seq[Int]): MumblingManifestBitmap = { + val bytes = MumblingBitmapWriter.serialize(values.distinct.sorted.toArray) + new MumblingManifestBitmap(new MumblingBitmap(ByteBuffer.wrap(bytes))) } - def fromPositions(values: Seq[Int]): RoaringManifestBitmap = - new RoaringManifestBitmap(Right(values.toVector)) - - def fromSerializedByteArray(serializedBytes: Array[Byte]): RoaringManifestBitmap = - new RoaringManifestBitmap(Left(serializedBytes.clone())) + def fromSerializedByteArray(bytes: Array[Byte]): MumblingManifestBitmap = + new MumblingManifestBitmap(new MumblingBitmap(ByteBuffer.wrap(bytes))) } -private final class MutableRoaringManifestBitmap private ( - private val roaringBitmapArray: RoaringBitmapArray) +/** A [[MutableManifestBitmap]] backed by a [[MutableMumblingBitmap]]. */ +private final class MutableMumblingManifestBitmap( + private val underlying: MutableMumblingBitmap) extends MutableManifestBitmap { - override def add(value: Int): Unit = roaringBitmapArray.add(value.toLong) + override def add(value: Int): Unit = underlying.add(value.toLong) override def toManifestBitmap: ManifestBitmap = - RoaringManifestBitmap.fromSerializedByteArray( - roaringBitmapArray.serializeAsByteArray(RoaringBitmapArrayFormat.Portable)) -} - -private object MutableRoaringManifestBitmap { - def fromSerializedByteArray(serializedBytes: Array[Byte]): MutableRoaringManifestBitmap = - new MutableRoaringManifestBitmap(RoaringBitmapArray.readFrom(serializedBytes)) + new MumblingManifestBitmap(underlying.toMumblingBitmap) } diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/deletionvectors/MumblingRowIndexFilter.scala b/spark/src/main/scala/org/apache/spark/sql/delta/deletionvectors/MumblingRowIndexFilter.scala new file mode 100644 index 00000000000..1c9e9d8608a --- /dev/null +++ b/spark/src/main/scala/org/apache/spark/sql/delta/deletionvectors/MumblingRowIndexFilter.scala @@ -0,0 +1,89 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.deletionvectors + +import java.nio.ByteBuffer + +import org.apache.spark.sql.delta.{RowIndexFilter, RowIndexFilterProvider} +import org.apache.spark.sql.delta.deletionvectors.mumbling.{MumblingBitmap, MumblingBitmapWriter} +import org.apache.hadoop.conf.Configuration + +import org.apache.spark.sql.execution.vectorized.WritableColumnVector +import org.apache.spark.sql.vectorized.ColumnVector + +/** + * A [[RowIndexFilter]] over an AMT manifest deletion vector (MDV): a leaf row is dropped when its + * row index is masked (contained) by the MDV, and kept otherwise. This is the AMT-local analog of + * [[org.apache.spark.sql.delta.util.BitmapFilter.IfContained]] and probes the bitmap per + * row (row indices within a leaf are dense and small), which matches the point-membership shape of + * the Mumbling bitmap. + */ +private[deletionvectors] final class MumblingRowIndexFilter(bitmap: MumblingBitmap) + extends RowIndexFilter { + + private def valueFor(rowIndex: Long): Byte = { + require( + rowIndex < MumblingBitmapWriter.MAX_POSITION_EXCLUSIVE, + s"Row index $rowIndex exceeds the Mumbling bitmap limit " + + s"${MumblingBitmapWriter.MAX_POSITION_EXCLUSIVE - 1}") + if (rowIndex >= 0 && bitmap.isSet(rowIndex.toInt)) { + RowIndexFilter.DROP_ROW_VALUE + } else { + RowIndexFilter.KEEP_ROW_VALUE + } + } + + override def materializeIntoVector( + start: Long, end: Long, batch: WritableColumnVector): Unit = { + val batchSize = (end - start).toInt + var rowId = 0 + while (rowId < batchSize) { + batch.putByte(rowId, valueFor(start + rowId.toLong)) + rowId += 1 + } + } + + override def materializeIntoVectorWithRowIndex( + batchSize: Int, rowIndexColumn: ColumnVector, batch: WritableColumnVector): Unit = { + var rowId = 0 + while (rowId < batchSize) { + batch.putByte(rowId, valueFor(rowIndexColumn.getLong(rowId))) + rowId += 1 + } + } + + override def materializeSingleRowWithRowIndex( + rowIndex: Long, batch: WritableColumnVector): Unit = + batch.putByte(0, valueFor(rowIndex)) +} + +/** + * A [[RowIndexFilterProvider]] for a manifest deletion vector carried inline in `manifest_info.dv`. + */ +private[deletionvectors] final class MumblingRowIndexFilterProvider( + inputSerializedBytes: Array[Byte], cardinality: Long) + extends RowIndexFilterProvider with Serializable { + + private val serializedBytes = inputSerializedBytes.clone() + + private def loadBitmap(): MumblingBitmap = + new MumblingBitmap(ByteBuffer.wrap(serializedBytes)) + + override def retrieve(hadoopConf: Configuration): RowIndexFilter = + new MumblingRowIndexFilter(loadBitmap()) + +} diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTIncrementalWriteInlineSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTIncrementalWriteInlineSuite.scala index 007397e6ae0..cac0124950a 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTIncrementalWriteInlineSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTIncrementalWriteInlineSuite.scala @@ -171,12 +171,13 @@ class AMTIncrementalWriteInlineSuite extends AMTIncrementalWriteTestBase with Gr numLeavesModifiedStatus = 1), inlineAMTCommitActions = Some(Seq(removeOf(amtDeltaLog, 1)))) val stamped = - leafPointers(amtDeltaLog.update()).values.filter(leafDeletedPositions(_).nonEmpty).toSeq + leafPointers(amtDeltaLog.update()).values + .filter(leafDeletedPositionCount(_) > 0).toSeq assert(stamped.size == 1, s"Exactly one leaf must carry this commit's deleted_positions; got ${stamped.size}.") - assert(leafDeletedPositions(stamped.head).size == 1, + assert(leafDeletedPositionCount(stamped.head) == 1, s"deleted_positions must hold this commit's single deletion; " + - s"got ${leafDeletedPositions(stamped.head)}.") + s"got ${leafDeletedPositionCount(stamped.head)}.") assert(mdvCardinality(stamped.head) == 1L, "The same leaf's cumulative MDV must also carry that one bit.") @@ -191,9 +192,9 @@ class AMTIncrementalWriteInlineSuite extends AMTIncrementalWriteTestBase with Gr inlineAMTCommitActions = Some(Seq(fakeAdd(31)))) val afterAppend = amtDeltaLog.update() leafPointers(afterAppend).foreach { case (location, leaf) => - assert(leafDeletedPositions(leaf).isEmpty, + assert(leafDeletedPositionCount(leaf) == 0, s"deleted_positions must reset on leaf $location when this commit deletes nothing " + - s"from it; got ${leafDeletedPositions(leaf)}.") + s"from it; got ${leafDeletedPositionCount(leaf)}.") } assert(leafPointers(afterAppend).values.map(mdvCardinality).sum == 1L, "The cumulative MDV bit from the earlier delete must survive the append.") @@ -225,11 +226,11 @@ class AMTIncrementalWriteInlineSuite extends AMTIncrementalWriteTestBase with Gr numLeavesModifiedStatus = 1), inlineAMTCommitActions = Some(removeAndReAddWithDV(amtDeltaLog, Left(1)))) val replaced = leafPointers(amtDeltaLog.update()).values - .filter(leafReplacedPositions(_).nonEmpty).toSeq + .filter(leafReplacedPositionCount(_) > 0).toSeq assert(replaced.size == 1, s"Exactly one leaf must carry this commit's replaced_positions; got ${replaced.size}.") - assert(leafReplacedPositions(replaced.head).size == 1 && - leafDeletedPositions(replaced.head).isEmpty, + assert(leafReplacedPositionCount(replaced.head) == 1 && + leafDeletedPositionCount(replaced.head) == 0, s"REPLACE sets replaced_positions, not deleted; got ${replaced.head.tracking}.") assert( rootDataEntryStatusToCount(amtDeltaLog).getOrElse(Tracking.Status.Modified, 0L) == 1L, @@ -449,10 +450,11 @@ class AMTIncrementalWriteInlineSuite extends AMTIncrementalWriteTestBase with Gr numLeavesExistingStatus = leavesBefore.size - 1), inlineAMTCommitActions = Some(fileToDelete.remove +: replaceActions)) val touched = leafPointers(amtDeltaLog.update()).values - .filter(l => leafDeletedPositions(l).nonEmpty || leafReplacedPositions(l).nonEmpty).toSeq + .filter(l => + leafDeletedPositionCount(l) > 0 || leafReplacedPositionCount(l) > 0).toSeq assert(touched.size == 1, s"exactly one leaf must be touched; got ${touched.size}.") - assert(leafDeletedPositions(touched.head).size == 1 && - leafReplacedPositions(touched.head).size == 1, + assert(leafDeletedPositionCount(touched.head) == 1 && + leafReplacedPositionCount(touched.head) == 1, s"leaf must carry a deleted AND replaced position; got ${touched.head.tracking}.") assertLiveAddFilesEquals(baselineDeltaLog, amtDeltaLog) } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTIncrementalWriteTestBase.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTIncrementalWriteTestBase.scala index 6c20acdedb9..0b4df38c75b 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTIncrementalWriteTestBase.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTIncrementalWriteTestBase.scala @@ -221,17 +221,17 @@ abstract class AMTIncrementalWriteTestBase extends AMTCheckpointTestBase { (liveAdds, tombstones) } - /** Decodes a manifest bitmap's set positions into a set (empty if the bytes are absent). */ - private def bitmapPositions(bytes: Array[Byte]): Set[Long] = - ManifestBitmap.fromSerializedByteArray(bytes).toArrayForTesting.toSet + /** Returns the number of set positions in a serialized manifest bitmap. */ + private def bitmapCardinality(bytes: Array[Byte]): Long = + ManifestBitmap.fromSerializedByteArray(bytes).cardinality - /** The per-commit CDF `tracking.deleted_positions` off a leaf pointer (empty if unset). */ - protected def leafDeletedPositions(leaf: DataManifestEntry): Set[Long] = - leaf.tracking.deleted_positions.map(bitmapPositions).getOrElse(Set.empty) + /** The per-commit CDF `tracking.deleted_positions` count off a leaf pointer. */ + protected def leafDeletedPositionCount(leaf: DataManifestEntry): Long = + leaf.tracking.deleted_positions.map(bitmapCardinality).getOrElse(0L) - /** The per-commit CDF `tracking.replaced_positions` off a leaf pointer (empty if unset). */ - protected def leafReplacedPositions(leaf: DataManifestEntry): Set[Long] = - leaf.tracking.replaced_positions.map(bitmapPositions).getOrElse(Set.empty) + /** The per-commit CDF `tracking.replaced_positions` count off a leaf pointer. */ + protected def leafReplacedPositionCount(leaf: DataManifestEntry): Long = + leaf.tracking.replaced_positions.map(bitmapCardinality).getOrElse(0L) /** The current snapshot's leaf pointers, keyed by relative location. */ protected def leafPointers(snapshot: Snapshot): Map[String, DataManifestEntry] = { @@ -378,8 +378,8 @@ abstract class AMTIncrementalWriteTestBase extends AMTCheckpointTestBase { .getOrElse(fail("AMT table must be checkpoint-provider-backed.")).leaves val statusToLeafCountMapping = newLeafPointers.groupBy(_.tracking.status).map { case (s, ps) => s -> ps.size } - val deleteCDFBitsAdded = newLeafPointers.map(p => leafDeletedPositions(p).size).sum - val replaceCDFBitsAdded = newLeafPointers.map(p => leafReplacedPositions(p).size).sum + val deleteCDFBitsAdded = newLeafPointers.map(leafDeletedPositionCount).sum + val replaceCDFBitsAdded = newLeafPointers.map(leafReplacedPositionCount).sum val derived = metrics.copy( numOldLeavesUpdated = updated, @@ -467,10 +467,9 @@ abstract class AMTIncrementalWriteTestBase extends AMTCheckpointTestBase { s"Leaf ${leaf.location}: MDV cardinality ($mdvDeclaredCardinality) exceeds physical " + s"entries " + s"($physicalEntries).") - decoded.toArrayForTesting.foreach { pos => - assert(pos >= 0 && pos < physicalEntries, - s"Leaf ${leaf.location}: MDV position $pos is out of range [0, $physicalEntries).") - } + val positionsInRange = (0L until physicalEntries).count(pos => decoded.contains(pos.toInt)) + assert(positionsInRange == decoded.cardinality, + s"Leaf ${leaf.location}: MDV contains a position outside [0, $physicalEntries).") livePhysicalEntries - mdvDeclaredCardinality }.sum diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/deletionvectors/ManifestBitmapSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/deletionvectors/ManifestBitmapSuite.scala index 09fa3aafbc3..00a7190f7d5 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/deletionvectors/ManifestBitmapSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/deletionvectors/ManifestBitmapSuite.scala @@ -16,7 +16,14 @@ package org.apache.spark.sql.delta.deletionvectors +import org.apache.spark.sql.delta.RowIndexFilter +import org.apache.spark.sql.delta.deletionvectors.mumbling.MumblingBitmapWriter +import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.fs.Path + import org.apache.spark.SparkFunSuite +import org.apache.spark.sql.execution.vectorized.OnHeapColumnVector +import org.apache.spark.sql.types.{ByteType, LongType} class ManifestBitmapSuite extends SparkFunSuite { @@ -26,8 +33,8 @@ class ManifestBitmapSuite extends SparkFunSuite { mutable.add(2) - assert(original.toArrayForTesting.sameElements(Array(1L, 3L))) - assert(mutable.toManifestBitmap.toArrayForTesting.sameElements(Array(1L, 2L, 3L))) + assertPositions(original, Seq(1, 3)) + assertPositions(mutable.toManifestBitmap, Seq(1, 2, 3)) } test("immutable snapshots do not change after further mutations") { @@ -38,8 +45,8 @@ class ManifestBitmapSuite extends SparkFunSuite { mutable.add(3) val secondSnapshot = mutable.toManifestBitmap - assert(firstSnapshot.toArrayForTesting.sameElements(Array(1L, 2L))) - assert(secondSnapshot.toArrayForTesting.sameElements(Array(1L, 2L, 3L))) + assertPositions(firstSnapshot, Seq(1, 2)) + assertPositions(secondSnapshot, Seq(1, 2, 3)) } test("deserialized bitmaps own their input bytes") { @@ -48,7 +55,7 @@ class ManifestBitmapSuite extends SparkFunSuite { java.util.Arrays.fill(bytes, 0.toByte) - assert(bitmap.toArrayForTesting.sameElements(Array(1L, 3L))) + assertPositions(bitmap, Seq(1, 3)) } test("bitmaps created from positions own their input sequence") { @@ -57,6 +64,80 @@ class ManifestBitmapSuite extends SparkFunSuite { positions += 2 - assert(bitmap.toArrayForTesting.sameElements(Array(1L, 3L))) + assertPositions(bitmap, Seq(1, 3)) + } + + test("ManifestBitmap uses the Mumbling format") { + val positions = Seq(1, 2, 300) + val bitmap = ManifestBitmap.fromPositions(positions) + val bytes = bitmap.serializeAsByteArray() + + assert(bitmap.isInstanceOf[MumblingManifestBitmap]) + assert(bytes(0) == 1) + assertPositions(ManifestBitmap.fromSerializedByteArray(bytes), positions) + } + + test("ManifestBitmap writes a mutated bitmap in the Mumbling format") { + val mutable = ManifestBitmap.fromPositions(Seq(1, 300)).toMutable + mutable.add(512) + val serialized = mutable.toManifestBitmap.serializeAsByteArray() + + assert(serialized(0) == 1) + assertPositions(ManifestBitmap.fromSerializedByteArray(serialized), Seq(1, 300, 512)) + } + + private def assertPositions(bitmap: ManifestBitmap, expected: Seq[Int]): Unit = { + assert(bitmap.cardinality == expected.distinct.size) + expected.foreach(position => assert(bitmap.contains(position))) + } + + + test("MumblingRowIndexFilter materializes every row-index filter entry point") { + val provider = ManifestBitmap.fromPositions(Seq(1, 3, 257)) + .toRowIndexFilterProvider(new Path("file:/tmp/table")) + val filter = provider.retrieve(new Configuration()) + + withByteVector(5) { batch => + filter.materializeIntoVector(0, 5, batch) + assert(batch.getBytes(0, 5).toSeq === Seq(0, 1, 0, 1, 0)) + } + + withByteVector(6) { batch => + val rowIndexes = new OnHeapColumnVector(6, LongType) + try { + Seq(-1L, 0L, 1L, 3L, 257L, 258L).zipWithIndex.foreach { + case (rowIndex, rowId) => rowIndexes.putLong(rowId, rowIndex) + } + filter.materializeIntoVectorWithRowIndex(6, rowIndexes, batch) + assert(batch.getBytes(0, 6).toSeq === Seq(0, 0, 1, 1, 1, 0)) + } finally { + rowIndexes.close() + } + } + + assert(singleRowValue(filter, rowIndex = 3L) == RowIndexFilter.DROP_ROW_VALUE) + assert(singleRowValue(filter, rowIndex = 4L) == RowIndexFilter.KEEP_ROW_VALUE) + intercept[IllegalArgumentException] { + singleRowValue(filter, rowIndex = MumblingBitmapWriter.MAX_POSITION_EXCLUSIVE.toLong) + } + } + + private def withByteVector(size: Int)(f: OnHeapColumnVector => Unit): Unit = { + val vector = new OnHeapColumnVector(size, ByteType) + try { + f(vector) + } finally { + vector.close() + } + } + + private def singleRowValue(filter: RowIndexFilter, rowIndex: Long): Byte = { + val vector = new OnHeapColumnVector(1, ByteType) + try { + filter.materializeSingleRowWithRowIndex(rowIndex, vector) + vector.getByte(0) + } finally { + vector.close() + } } } From ab0668b4d243b45456b42447a907f40c0fed16ef Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Tue, 13 Oct 2026 13:49:56 -0700 Subject: [PATCH 13/29] [Spark] Convert DeltaV2Table from Java to Scala (#7856) #### Which Delta project/connector is this regarding? - [x] Spark ## Description Convert `DeltaV2Table` from Java to Scala. Preserve its Java-facing collection and optional interfaces, synchronized schema initialization, and `KernelContext`-backed engine lookup. Constants and static helpers move to a Scala companion. Existing tests remain unchanged. Compatibility notes: - Scala `protected` methods are JVM-public. Java subclasses that override `protocol()`, `metadata()`, `kernelEngine()`, `initialSnapshot()`, or `catalogTable()` with protected visibility must widen their overrides to public when recompiling. - Helper visibility and reflection metadata differ from the Java implementation, including the public static forwarder for `getDecodedPath`. - Scala introduces some tuple and callback allocations; performance impact is unmeasured. ## How was this patch tested? Existing tests are unchanged. Local tests have not been run in this checkout; CI validation is pending. ## Does this PR introduce _any_ user-facing changes? No intended SQL behavior change. Java subclass and reflection compatibility differences are listed above. Signed-off-by: Hao Sun <86338940+HaoSun1993@users.noreply.github.com> --- .../internal/v2/catalog/DeltaV2Table.java | 741 ------------------ .../internal/v2/catalog/DeltaV2Table.scala | 684 ++++++++++++++++ 2 files changed, 684 insertions(+), 741 deletions(-) delete mode 100644 spark/v2/src/main/java/io/delta/spark/internal/v2/catalog/DeltaV2Table.java create mode 100644 spark/v2/src/main/scala/io/delta/spark/internal/v2/catalog/DeltaV2Table.scala diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/catalog/DeltaV2Table.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/catalog/DeltaV2Table.java deleted file mode 100644 index dc02ee4d6f0..00000000000 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/catalog/DeltaV2Table.java +++ /dev/null @@ -1,741 +0,0 @@ -/* - * Copyright (2025) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.delta.spark.internal.v2.catalog; - -import static io.delta.spark.internal.v2.utils.ScalaUtils.toJavaOptional; -import static io.delta.spark.internal.v2.utils.ScalaUtils.toScalaMap; -import static io.delta.spark.internal.v2.utils.ScalaUtils.toScalaOption; -import static io.delta.spark.internal.v2.utils.StatsUtils.toV2Statistics; -import static java.util.Objects.requireNonNull; - -import io.delta.kernel.engine.Engine; -import io.delta.kernel.internal.DeltaHistoryManager; -import io.delta.spark.internal.v2.exception.NoRecreatableHistoryException; -import io.delta.spark.internal.v2.exception.TableNotFoundException; -import io.delta.spark.internal.v2.exception.TimestampOutOfRangeException; -import io.delta.spark.internal.v2.kernel.KernelContext; -import io.delta.spark.internal.v2.read.DeltaV2ScanUtils; -import io.delta.spark.internal.v2.read.MetadataEvolutionHandler; -import io.delta.spark.internal.v2.read.cdc.CDCSchemaContext; -import io.delta.spark.internal.v2.shims.CatalogV2UtilShims; -import io.delta.spark.internal.v2.tablemanager.DeltaV2TableManager; -import io.delta.spark.internal.v2.tablemanager.DeltaV2TableManagerCache$; -import io.delta.spark.internal.v2.write.DeltaRowLevelOperationBuilder; -import io.delta.spark.internal.v2.write.DeltaV2WriteBuilder; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.EnumSet; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.OptionalLong; -import java.util.Set; -import java.util.function.Supplier; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.spark.sql.SparkSession; -import org.apache.spark.sql.catalyst.catalog.CatalogTable; -import org.apache.spark.sql.connector.catalog.Column; -import org.apache.spark.sql.connector.catalog.Identifier; -import org.apache.spark.sql.connector.catalog.MetadataColumn; -import org.apache.spark.sql.connector.catalog.SupportsMetadataColumns; -import org.apache.spark.sql.connector.catalog.SupportsRead; -import org.apache.spark.sql.connector.catalog.SupportsRowLevelOperations; -import org.apache.spark.sql.connector.catalog.SupportsWrite; -import org.apache.spark.sql.connector.catalog.Table; -import org.apache.spark.sql.connector.catalog.TableCapability; -import org.apache.spark.sql.connector.expressions.Expressions; -import org.apache.spark.sql.connector.expressions.Transform; -import org.apache.spark.sql.connector.read.ScanBuilder; -import org.apache.spark.sql.connector.read.Statistics; -import org.apache.spark.sql.connector.write.LogicalWriteInfo; -import org.apache.spark.sql.connector.write.RowLevelOperationBuilder; -import org.apache.spark.sql.connector.write.RowLevelOperationInfo; -import org.apache.spark.sql.connector.write.WriteBuilder; -import org.apache.spark.sql.delta.DeltaTableUtils; -import org.apache.spark.sql.delta.RowCommitVersion$; -import org.apache.spark.sql.delta.RowId$; -import org.apache.spark.sql.delta.RowTracking$; -import org.apache.spark.sql.delta.Snapshot; -import org.apache.spark.sql.delta.catalog.DeltaV2TableMarker; -import org.apache.spark.sql.delta.commands.cdc.CDCReader; -import org.apache.spark.sql.delta.sources.PersistedMetadata; -import org.apache.spark.sql.delta.util.DeltaFileSystemOptions; -import org.apache.spark.sql.delta.v2.interop.AbstractMetadata; -import org.apache.spark.sql.delta.v2.interop.AbstractProtocol; -import org.apache.spark.sql.delta.v2.interop.DeltaV2QueryContext; -import org.apache.spark.sql.delta.v2.interop.DeltaV2QueryContext$; -import org.apache.spark.sql.delta.v2.interop.DeltaV2SnapshotManager; -import org.apache.spark.sql.execution.datasources.FileFormat$; -import org.apache.spark.sql.types.DataType; -import org.apache.spark.sql.types.DataTypes; -import org.apache.spark.sql.types.StructField; -import org.apache.spark.sql.types.StructType; -import org.apache.spark.sql.util.CaseInsensitiveStringMap; -import scala.Option; -import scala.jdk.javaapi.CollectionConverters; - -/** DataSource V2 Table implementation for Delta Lake using the Delta Kernel API. */ -public class DeltaV2Table extends DeltaV2TableShimsWithLogging - implements Table, - SupportsRead, - SupportsWrite, - SupportsMetadataColumns, - SupportsRowLevelOperations, - DeltaV2TableMarker { - private static final String METADATA_COLUMN_NAME = FileFormat$.MODULE$.METADATA_NAME(); - private static final String ROW_ID_METADATA_FIELD_NAME = RowId$.MODULE$.ROW_ID(); - private static final String ROW_COMMIT_VERSION_METADATA_FIELD_NAME = - RowCommitVersion$.MODULE$.METADATA_STRUCT_FIELD_NAME(); - - private static final Set CAPABILITIES = buildCapabilities(); - private static final Set READ_ONLY_CAPABILITIES = buildReadOnlyCapabilities(); - - private static Set buildCapabilities() { - EnumSet caps = - EnumSet.of( - TableCapability.BATCH_READ, - TableCapability.MICRO_BATCH_READ, - TableCapability.BATCH_WRITE, - TableCapability.STREAMING_WRITE); - DeltaV2TableShims.schemaEvolutionCapability().ifPresent(caps::add); - return Collections.unmodifiableSet(caps); - } - - private static Set buildReadOnlyCapabilities() { - return Collections.unmodifiableSet( - EnumSet.of(TableCapability.BATCH_READ, TableCapability.MICRO_BATCH_READ)); - } - - private final Identifier identifier; - private final String tablePath; - private final Map options; - private final DeltaV2SnapshotManager snapshotManager; - private final DeltaV2QueryContext queryContext; - /** Snapshot created during connector setup */ - private final Snapshot initialSnapshot; - - private final Configuration hadoopConf; - private final KernelContext kernelContext; - - private final SchemaProvider schemaProvider; - private final Optional catalogTable; - private final boolean isCDCRead; - // True when this table is pinned to a time travel version. - private final boolean isTimeTravel; - - /** - * Creates a DeltaV2Table from a filesystem path without a catalog table. - * - * @param identifier logical table identifier used by Spark's catalog - * @param tablePath filesystem path to the Delta table root - * @throws NullPointerException if identifier or tablePath is null - */ - public DeltaV2Table(Identifier identifier, String tablePath) { - this(identifier, tablePath, Collections.emptyMap(), Optional.empty(), OptionalLong.empty()); - } - - /** - * Creates a DeltaV2Table from a filesystem path with options. - * - * @param identifier logical table identifier used by Spark's catalog - * @param tablePath filesystem path to the Delta table root - * @param options table options used to configure the Hadoop conf, table reads and writes - * @throws NullPointerException if identifier or tablePath is null - */ - public DeltaV2Table(Identifier identifier, String tablePath, Map options) { - this(identifier, tablePath, options, Optional.empty(), OptionalLong.empty()); - } - - /** - * Path-based constructor pinned to an explicit time travel version (if present). - * - * @param identifier logical table identifier used by Spark's catalog - * @param tablePath filesystem path to the Delta table root - * @param options table options used to configure the Hadoop conf, table reads and writes - * @param timeTravelVersion the table version to pin the initial snapshot to - */ - public DeltaV2Table( - Identifier identifier, - String tablePath, - Map options, - OptionalLong timeTravelVersion) { - this(identifier, tablePath, options, Optional.empty(), timeTravelVersion); - } - - /** - * Constructor that accepts a Spark CatalogTable and user-provided options. Extracts the table - * location and storage properties from the catalog table, then merges with user options. User - * options take precedence over catalog properties in case of conflicts. - * - * @param identifier logical table identifier used by Spark's catalog - * @param catalogTable the Spark CatalogTable containing table metadata including location - * @param options user-provided options to override catalog properties - */ - public DeltaV2Table( - Identifier identifier, CatalogTable catalogTable, Map options) { - this(identifier, catalogTable, options, OptionalLong.empty()); - } - - /** - * Catalog-table constructor pinned to an explicit time travel version (if present). - * - * @param identifier logical table identifier used by Spark's catalog - * @param catalogTable the Spark CatalogTable containing table metadata including location - * @param options user-provided options to override catalog properties - * @param timeTravelVersion the table version to pin the initial snapshot to - */ - public DeltaV2Table( - Identifier identifier, - CatalogTable catalogTable, - Map options, - OptionalLong timeTravelVersion) { - this( - identifier, - getDecodedPath(requireNonNull(catalogTable, "catalogTable is null").location()), - options, - Optional.of(catalogTable), - timeTravelVersion); - } - - /** - * Creates a DeltaV2Table backed by a Delta Kernel snapshot manager and initializes Spark-facing - * metadata (schemas, partitioning, capabilities). - * - *

Side effects: - Initializes a SnapshotManager for the given tablePath. - Loads the latest - * snapshot via the manager. - Builds Hadoop configuration from options for subsequent I/O. - - * Derives data schema, partition schema, and full table schema from the snapshot. - * - *

Notes: - Partition column order from the snapshot is preserved for partitioning and appended - * after data columns in the public Spark schema, per Spark conventions. - Read-time scan options - * are later merged with these options. - */ - private DeltaV2Table( - Identifier identifier, - String tablePath, - Map userOptions, - Optional catalogTable, - OptionalLong timeTravelVersion) { - this.identifier = requireNonNull(identifier, "identifier is null"); - this.tablePath = requireNonNull(tablePath, "tablePath is null"); - this.catalogTable = catalogTable; - this.queryContext = DeltaV2QueryContext$.MODULE$.apply(catalogTable); - Option catalogTableOpt = toScalaOption(catalogTable); - // Merge options: file system options from catalog + user options (user takes precedence) - // This follows the same pattern as DeltaTableV2 in delta-spark - Map merged = - new HashMap<>( - scala.collection.JavaConverters.mapAsJavaMap( - DeltaFileSystemOptions.extractCatalogTableFsOptions(catalogTableOpt))); - // User options override catalog properties - merged.putAll(userOptions); - this.options = Collections.unmodifiableMap(merged); - - SparkSession activeSession = SparkSession.active(); - this.hadoopConf = activeSession.sessionState().newHadoopConfWithOptions(toScalaMap(options)); - DeltaV2TableManager tableManager = - DeltaV2TableManagerCache$.MODULE$.forTable( - activeSession, tablePath, options, catalogTableOpt); - this.kernelContext = tableManager.kernelContext(); - this.snapshotManager = tableManager.snapshotManager(catalogTableOpt); - try { - if (timeTravelVersion.isPresent()) { - this.initialSnapshot = - loadSnapshotAtCheckedVersion(snapshotManager, timeTravelVersion.getAsLong()); - } else { - Supplier loadLatestSnapshot = - () -> snapshotManager.loadLatestSnapshot(queryContext); - this.initialSnapshot = recordFrameProfileValue("snapshot.loadLatest", loadLatestSnapshot); - } - } catch (io.delta.kernel.exceptions.TableNotFoundException e) { - // Rethrow as the Delta-module wrapper so catalog/interop layer never names a Kernel type. - throw new TableNotFoundException(tablePath); - } catch (io.delta.kernel.exceptions.KernelException e) { - // Missing earliest commit file surfaces as a base KernelException, so match on its message. - String reason = e.getMessage(); - if (reason != null && reason.contains("Missing delta file")) { - throw new NoRecreatableHistoryException(tablePath); - } - throw e; - } - - this.isCDCRead = CDCReader.isCDCRead(new CaseInsensitiveStringMap(this.options)); - this.isTimeTravel = timeTravelVersion.isPresent(); - - Optional persistedMetadata = - MetadataEvolutionHandler.getPersistedMetadataForMicroBatchStream( - SparkSession.active(), initialSnapshot, options, snapshotManager, kernelEngine()); - - StructType rawSchema; - List partitionColumnNames; - if (persistedMetadata.isPresent()) { - PersistedMetadata persisted = persistedMetadata.get(); - rawSchema = persisted.dataSchema(); - partitionColumnNames = Arrays.asList(persisted.partitionSchema().fieldNames()); - } else { - rawSchema = initialSnapshot.schema(); - partitionColumnNames = new ArrayList<>(initialSnapshot.metadata().getPartitionColumns()); - } - // Schema-related metadata is lazily computed on first access within SchemaProvider - this.schemaProvider = - new SchemaProvider(SparkSession.active(), rawSchema, partitionColumnNames); - } - - /** - * Helper method to decode URI path handling URL-encoded characters correctly. E.g., converts - * "spark%25dir%25prefix" to "spark%dir%prefix" - * - *

Uses Hadoop's Path class to properly handle all URI schemes (file, s3, abfss, gs, hdfs, - * etc.), not just file:// URIs. - */ - private static String getDecodedPath(java.net.URI location) { - Path hadoopPath = new Path(location); - // For local file system paths, return just the path component without the scheme - // to maintain consistency with path-based table construction where tablePath is a - // plain filesystem path string. - if (location.getScheme() == null || "file".equals(location.getScheme())) { - return hadoopPath.toUri().getPath(); - } - return hadoopPath.toString(); - } - - /** - * Returns the CatalogTable if this DeltaV2Table was created from a catalog table. - * - * @return Optional containing the CatalogTable, or empty if this table was created from a path - */ - public Optional getCatalogTable() { - return catalogTable; - } - - /** - * Returns the Path to the Delta table root. - * - * @return Path created from the table path - */ - public Path getTablePath() { - return new Path(tablePath); - } - - /** - * Returns the V2 identifier this table was constructed with. - * - * @return Identifier provided at construction - */ - public Identifier getIdentifier() { - return identifier; - } - - /** - * Returns the merged options (catalog storage + user options) this table was constructed with. - * - * @return Map of merged catalog storage and user options - */ - public Map getOptions() { - return options; - } - - /** - * Returns the snapshot manager backing this table. Catalog-driven features such as read-time CDF - * (TableCatalog.loadChangelog) use this to resolve versions, timestamps, and snapshots without - * having to build their own snapshot manager. - */ - public DeltaV2SnapshotManager getSnapshotManager() { - return snapshotManager; - } - - /** The table protocol from the initial snapshot. */ - protected AbstractProtocol protocol() { - return initialSnapshot.protocol(); - } - - /** The table metadata from the initial snapshot. */ - protected AbstractMetadata metadata() { - return initialSnapshot.metadata(); - } - - /** Inputs exposed to the Spark-version shim for metadata-only DELETE. */ - protected Engine kernelEngine() { - return kernelContext.getDefaultEngine(); - } - - protected Snapshot initialSnapshot() { - return initialSnapshot; - } - - protected Optional catalogTable() { - return catalogTable; - } - - /** Returns a copy of this table pinned to {@code version}. */ - public DeltaV2Table withVersion(long version) { - return catalogTable.isPresent() - ? new DeltaV2Table(identifier, catalogTable.get(), options, OptionalLong.of(version)) - : new DeltaV2Table(identifier, tablePath, options, OptionalLong.of(version)); - } - - /** Returns a copy of this table pinned to the snapshot active at {@code timestampMicros}. */ - public DeltaV2Table withTimestamp(long timestampMicros) { - return withVersion(resolveTimestampToVersion(snapshotManager, timestampMicros, queryContext)); - } - - /** - * Resolves a time travel timestamp to the active commit version using the Kernel snapshot - * manager. - * - *

This loads the latest snapshot more than once (here and inside the Kernel lookup), make it - * share a singular load once the snapshot manager exposes it TODO(#5999). - */ - private static long resolveTimestampToVersion( - DeltaV2SnapshotManager manager, long timestampMicros, DeltaV2QueryContext queryContext) { - long timeMillis = timestampMicros / 1000; - DeltaHistoryManager.Commit commit = - manager.getActiveCommitAtTime( - timeMillis, - /* canReturnLastCommit = */ true, - /* mustBeRecreatable = */ true, - /* canReturnEarliestCommit = */ true, - queryContext); - long latestVersion = manager.loadLatestSnapshot(queryContext).version(); - if (commit.getTimestamp() > timeMillis) { - // The earliest available commit is younger than the requested time. - throw new TimestampOutOfRangeException(timeMillis, commit.getTimestamp(), false); - } else if (commit.getVersion() == latestVersion && commit.getTimestamp() < timeMillis) { - // The requested time is after the latest commit. - throw new TimestampOutOfRangeException(timeMillis, commit.getTimestamp(), true); - } - return commit.getVersion(); - } - - /** - * Returns the table name in a format compatible with DeltaTableV2. - * - *

For catalog-based tables, returns the fully qualified table name (e.g., - * "spark_catalog.default.table_name"). For path-based tables, returns the path-based identifier - * (e.g., "delta.`/path/to/table`"). - * - * @return the table name string - */ - @Override - public String name() { - return catalogTable - .map(ct -> ct.identifier().unquotedString()) - .orElse("delta.`" + tablePath + "`"); - } - - @Override - public StructType schema() { - StructType base = schemaProvider.getPublicSchema(); - return isCDCRead ? CDCSchemaContext.appendCDCColumns(base) : base; - } - - @Override - public Column[] columns() { - return CatalogV2UtilShims.structTypeToV2Columns(schema()); - } - - @Override - public Transform[] partitioning() { - return schemaProvider.getPartitionTransforms(); - } - - @Override - public Map properties() { - Map props = new HashMap<>(initialSnapshot.metadata().getConfiguration()); - return Collections.unmodifiableMap(props); - } - - @Override - public Set capabilities() { - // A time travel pin cannot be written to. Dropping the write capabilities makes writes fall - // back to the v1 path. - return isTimeTravel ? READ_ONLY_CAPABILITIES : CAPABILITIES; - } - - /** - * Returns the version of the snapshot loaded when this table was constructed, as a String. - * - *

Add {@code @Override} annotation eventually: TODO(#7128) - */ - public String version() { - return Long.toString(initialSnapshot.version()); - } - - /** - * Exposes file-level and row-tracking metadata via a single DSv2 metadata struct column. - * - *

This always returns one metadata column named {@code _metadata}. The struct contains Spark - * file-source base metadata fields with the same names and types as {@link - * org.apache.spark.sql.execution.datasources.FileFormat#BASE_METADATA_FIELDS} ({@code file_path}, - * {@code file_name}, {@code file_size}, {@code file_block_start}, {@code file_block_length}, - * {@code file_modification_time}); when row tracking is enabled, {@code row_id} and {@code - * row_commit_version} are appended. Values follow {@link - * org.apache.spark.sql.execution.datasources.PartitionedFile} semantics (see {@code - * FileFormat.BASE_METADATA_EXTRACTORS}). - * - *

Field order mirrors Spark's canonical {@code BASE_METADATA_FIELDS} for parity with V1 Delta, - * but resolution is name-based - order is not load-bearing for correctness. - */ - @Override - public MetadataColumn[] metadataColumns() { - boolean rowTrackingEnabled = - RowTracking$.MODULE$.isEnabled(initialSnapshot.protocol(), initialSnapshot.metadata()); - - StructType metadataType = new StructType(); - for (StructField field : - CollectionConverters.asJava(FileFormat$.MODULE$.BASE_METADATA_FIELDS())) { - metadataType = metadataType.add(field); - } - if (rowTrackingEnabled) { - metadataType = - metadataType - .add(ROW_ID_METADATA_FIELD_NAME, DataTypes.LongType, false) - .add(ROW_COMMIT_VERSION_METADATA_FIELD_NAME, DataTypes.LongType, false); - } - final StructType finalMetadataType = metadataType; - - MetadataColumn[] columns = new MetadataColumn[1]; - columns[0] = - new MetadataColumn() { - @Override - public String name() { - return METADATA_COLUMN_NAME; - } - - @Override - public DataType dataType() { - return finalMetadataType; - } - - @Override - public boolean isNullable() { - return false; - } - }; - - return columns; - } - - @Override - public ScanBuilder newScanBuilder(CaseInsensitiveStringMap scanOptions) { - Map combined = new HashMap<>(this.options); - combined.putAll(scanOptions.asCaseSensitiveMap()); - CaseInsensitiveStringMap merged = new CaseInsensitiveStringMap(combined); - Optional catalogStats = - catalogTable - .flatMap(ct -> toJavaOptional(ct.stats())) - .map( - stats -> - toV2Statistics( - stats, - schemaProvider.getDataSchema(), - schemaProvider.getPartitionSchema())); - return DeltaV2ScanUtils.newScanBuilder( - name(), - initialSnapshot, - kernelEngine(), - catalogTable, - snapshotManager, - schemaProvider.getDataSchema(), - schemaProvider.getPartitionSchema(), - schemaProvider.getRawSchema(), - catalogStats, - merged); - } - - /** - * Validates that {@code version} exists in the Delta log, then loads the snapshot pinned to it. - */ - private Snapshot loadSnapshotAtCheckedVersion(DeltaV2SnapshotManager manager, long version) { - boolean mustBeRecreatable = true; - boolean allowOutOfRange = false; - manager.checkVersionExists(version, mustBeRecreatable, allowOutOfRange, queryContext); - final Supplier loadSnapshot = () -> manager.loadSnapshotAt(version, queryContext); - return recordFrameProfileValue("snapshot.loadAtVersion", loadSnapshot); - } - - @Override - public WriteBuilder newWriteBuilder(LogicalWriteInfo info) { - requireNonNull(info, "write info is null"); - return new DeltaV2WriteBuilder( - kernelEngine(), - tablePath, - hadoopConf, - initialSnapshot, - catalogTable, - snapshotManager, - schemaProvider.getDataSchema(), - schemaProvider.getPartitionSchema(), - info); - } - - /** - * Returns a builder for Delta row-level operations. The builder currently wires Spark planning to - * Delta's copy-on-write operation shell; the concrete ReplaceData write path is introduced in a - * follow-up PR. - */ - @Override - public RowLevelOperationBuilder newRowLevelOperationBuilder(RowLevelOperationInfo info) { - requireNonNull(info, "row-level operation info is null"); - return new DeltaRowLevelOperationBuilder( - this, kernelEngine(), hadoopConf, initialSnapshot, info); - } - - @Override - public String toString() { - return "DeltaV2Table{identifier=" + identifier + '}'; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - DeltaV2Table that = (DeltaV2Table) o; - return Objects.equals(identifier, that.identifier) - && Objects.equals(tablePath, that.tablePath) - && Objects.equals(options, that.options) - && Objects.equals(catalogTable, that.catalogTable) - && Objects.equals( - initialSnapshot.dataPath().toString(), that.initialSnapshot.dataPath().toString()) - && initialSnapshot.version() == that.initialSnapshot.version() - && Objects.equals( - initialSnapshot.metadata().getId(), that.initialSnapshot.metadata().getId()); - } - - @Override - public int hashCode() { - return Objects.hash( - identifier, - tablePath, - options, - catalogTable, - initialSnapshot.dataPath().toString(), - initialSnapshot.version(), - initialSnapshot.metadata().getId()); - } - - /** - * Private helper class that lazily computes and caches schema-related metadata. - * - *

This class encapsulates all schema computation logic including: - * - *

    - *
  • Raw schema conversion from Kernel to Spark - *
  • Public schema with internal metadata removed - *
  • Data and partition schema derivation - *
  • Column and partition transform creation - *
- * - *

All schema computations are deferred until first access. - */ - private static class SchemaProvider { - private final SparkSession sparkSession; - private final StructType rawSchema; - private final List partColNames; - - // Lazily computed fields - private boolean initialized = false; - private StructType publicSchema; - private StructType dataSchema; - private StructType partitionSchema; - private Transform[] partitionTransforms; - - SchemaProvider(SparkSession sparkSession, StructType rawSchema, List partColNames) { - this.sparkSession = sparkSession; - this.rawSchema = rawSchema; - this.partColNames = Collections.unmodifiableList(new ArrayList<>(partColNames)); - } - - private synchronized void ensureInitialized() { - if (initialized) { - return; - } - - // Create public schema by removing internal metadata (for schema() method) - this.publicSchema = - DeltaTableUtils.removeInternalDeltaMetadata( - sparkSession, DeltaTableUtils.removeInternalWriterMetadata(sparkSession, rawSchema)); - - final List dataFields = new ArrayList<>(); - final List partitionFields = new ArrayList<>(); - - // Build a map for O(1) field lookups to improve performance - // Use rawSchema (with metadata) for deriving data and partition schemas - Map fieldMap = new HashMap<>(); - for (StructField field : rawSchema.fields()) { - fieldMap.put(field.name(), field); - } - - // IMPORTANT: Add partition fields in the exact order specified by partColNames - // This is crucial because the order in partColNames may differ from the order - // in snapshotSchema, and we need to preserve the partColNames order for - // proper partitioning behavior - for (String partColName : partColNames) { - StructField field = fieldMap.get(partColName); - if (field != null) { - partitionFields.add(field); - } - } - - // Add remaining fields as data fields (non-partition columns) - // These are fields that exist in the schema but are not partition columns - for (StructField field : rawSchema.fields()) { - if (!partColNames.contains(field.name())) { - dataFields.add(field); - } - } - this.dataSchema = new StructType(dataFields.toArray(new StructField[0])); - this.partitionSchema = new StructType(partitionFields.toArray(new StructField[0])); - - this.partitionTransforms = - partColNames.stream().map(Expressions::identity).toArray(Transform[]::new); - - this.initialized = true; - } - - private T withInit(Supplier supplier) { - ensureInitialized(); - return supplier.get(); - } - - StructType getPublicSchema() { - return withInit(() -> publicSchema); - } - - StructType getDataSchema() { - return withInit(() -> dataSchema); - } - - StructType getPartitionSchema() { - return withInit(() -> partitionSchema); - } - - StructType getRawSchema() { - return withInit(() -> rawSchema); - } - - Transform[] getPartitionTransforms() { - return withInit(() -> partitionTransforms); - } - } -} diff --git a/spark/v2/src/main/scala/io/delta/spark/internal/v2/catalog/DeltaV2Table.scala b/spark/v2/src/main/scala/io/delta/spark/internal/v2/catalog/DeltaV2Table.scala new file mode 100644 index 00000000000..fa0ca6bfe68 --- /dev/null +++ b/spark/v2/src/main/scala/io/delta/spark/internal/v2/catalog/DeltaV2Table.scala @@ -0,0 +1,684 @@ +/* + * Copyright (2025) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.spark.internal.v2.catalog + +import java.lang.{Long => JLong} +import java.net.URI +import java.util.{ + ArrayList, + Arrays, + Collections, + EnumSet, + HashMap, + List => JList, + Map => JMap, + Objects, + Optional, + Set => JSet +} +import java.util.Objects.requireNonNull +import java.util.function.Supplier + +import scala.collection.JavaConverters._ + +import org.apache.spark.sql.delta.{ + DeltaTableUtils, + RowCommitVersion, + RowId, + RowTracking, + Snapshot +} +import org.apache.spark.sql.delta.catalog.DeltaV2TableMarker +import org.apache.spark.sql.delta.commands.cdc.CDCReader +import org.apache.spark.sql.delta.sources.PersistedMetadata +import org.apache.spark.sql.delta.util.DeltaFileSystemOptions +import io.delta.spark.internal.v2.exception.{ + NoRecreatableHistoryException, + TableNotFoundException, + TimestampOutOfRangeException +} +import org.apache.spark.sql.delta.v2.interop.{ + AbstractMetadata, + AbstractProtocol, + DeltaV2QueryContext, + DeltaV2SnapshotManager +} +import io.delta.spark.internal.v2.read.{DeltaV2ScanUtils, MetadataEvolutionHandler} +import io.delta.spark.internal.v2.read.cdc.CDCSchemaContext +import io.delta.spark.internal.v2.tablemanager.DeltaV2TableManagerCache +import io.delta.spark.internal.v2.utils.ScalaUtils.{ + toJavaOptional, + toScalaMap +} +import io.delta.spark.internal.v2.utils.StatsUtils.toV2Statistics +import io.delta.spark.internal.v2.write.{ + DeltaRowLevelOperationBuilder, + DeltaV2WriteBuilder +} +import io.delta.spark.internal.v2.shims.CatalogV2UtilShims +import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.fs.Path +import io.delta.kernel.engine.{Engine => KernelEngine} +import io.delta.kernel.exceptions.{ + KernelException, + TableNotFoundException => KernelTableNotFoundException +} +import io.delta.kernel.internal.{DeltaHistoryManager => KernelDeltaHistoryManager} + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.catalog.{CatalogStatistics, CatalogTable} +import org.apache.spark.sql.connector.catalog.{ + Column, + Identifier, + MetadataColumn, + SupportsMetadataColumns, + SupportsRead, + SupportsRowLevelOperations, + SupportsWrite, + Table, + TableCapability +} +import org.apache.spark.sql.connector.expressions.{Expressions, Transform} +import org.apache.spark.sql.connector.read.{ScanBuilder, Statistics} +import org.apache.spark.sql.connector.write.{ + LogicalWriteInfo, + RowLevelOperationBuilder, + RowLevelOperationInfo, + WriteBuilder +} +import org.apache.spark.sql.execution.datasources.FileFormat +import org.apache.spark.sql.types.{DataType, DataTypes, StructField, StructType} +import org.apache.spark.sql.util.CaseInsensitiveStringMap + +/** DataSource V2 Table implementation for Delta Lake using the Delta Kernel API. */ +/** + * Creates a DeltaV2Table backed by a Delta Kernel snapshot manager and initializes Spark-facing + * metadata (schemas, partitioning, capabilities). + * + *

Side effects: - Initializes a SnapshotManager for the given tablePath. - Loads the latest + * snapshot via the manager. - Builds Hadoop configuration from options for subsequent I/O. - + * Derives data schema, partition schema, and full table schema from the snapshot. + * + *

Notes: - Partition column order from the snapshot is preserved for partitioning and appended + * after data columns in the public Spark schema, per Spark conventions. - Read-time scan options + * are later merged with these options. + */ +class DeltaV2Table private ( + private val identifier: Identifier, + private val tablePath: String, + userOptions: JMap[String, String], + catalogTable: Optional[CatalogTable], + timeTravelVersion: Option[Long]) + extends DeltaV2TableShimsWithLogging + with Table + with SupportsRead + with SupportsWrite + with SupportsMetadataColumns + with SupportsRowLevelOperations + with DeltaV2TableMarker + with DeltaV2TableVersion { + + import DeltaV2Table._ + + requireNonNull(identifier, "identifier is null") + requireNonNull(tablePath, "tablePath is null") + private val catalogTableValue = catalogTable + private val queryContext = DeltaV2QueryContext(catalogTableValue) + // Merge options: file system options from catalog + user options (user takes precedence) + // This follows the same pattern as DeltaTableV2 in delta-spark + private val (options, hadoopConf, kernelContext, snapshotManager) = { + val catalogTableOpt = queryContext.catalogTableOpt + val merged = new HashMap[String, String]( + mapAsJavaMap( + DeltaFileSystemOptions.extractCatalogTableFsOptions(catalogTableOpt))) + // User options override catalog properties + merged.putAll(userOptions) + val options = Collections.unmodifiableMap[String, String](merged) + val activeSession = SparkSession.active + val hadoopConf = activeSession.sessionState.newHadoopConfWithOptions(toScalaMap(options)) + val tableManager = + DeltaV2TableManagerCache.forTable(activeSession, tablePath, options, catalogTableOpt) + ( + options, + hadoopConf, + tableManager.kernelContext, + tableManager.snapshotManager(catalogTableOpt)) + } + + /** Snapshot created during connector setup */ + private val initialSnapshotValue = loadInitialSnapshot(timeTravelVersion) + + private def loadInitialSnapshot(timeTravelVersion: Option[Long]): Snapshot = try { + if (timeTravelVersion.isDefined) { + loadSnapshotAtCheckedVersion(snapshotManager, timeTravelVersion.get) + } else { + val loadLatestSnapshot: Supplier[Snapshot] = + () => snapshotManager.loadLatestSnapshot(queryContext) + recordFrameProfileValue("snapshot.loadLatest", loadLatestSnapshot) + } + } catch { + case _: KernelTableNotFoundException => + // Rethrow as the Delta-module wrapper so catalog/interop layer never names a Kernel type. + throw new TableNotFoundException(tablePath) + case kernelError: KernelException => + // Missing earliest commit file surfaces as a base KernelException, so match on its message. + val reason = kernelError.getMessage + if (reason != null && reason.contains("Missing delta file")) { + throw new NoRecreatableHistoryException(tablePath) + } + throw kernelError + } + + private val isCDCRead = CDCReader.isCDCRead(new CaseInsensitiveStringMap(options)) + // True when this table is pinned to a time travel version. + private val isTimeTravel = timeTravelVersion.isDefined + + private val schemaProvider = { + val persistedMetadata: Optional[PersistedMetadata] = + MetadataEvolutionHandler.getPersistedMetadataForMicroBatchStream( + SparkSession.active, initialSnapshotValue, options, snapshotManager, kernelEngine()) + + val (rawSchema, partitionColumnNames) = if (persistedMetadata.isPresent) { + val persisted = persistedMetadata.get() + (persisted.dataSchema, Arrays.asList(persisted.partitionSchema.fieldNames: _*)) + } else { + val rawSchema = initialSnapshotValue.schema + val partitionColumnNames = + new ArrayList[String](initialSnapshotValue.metadata.getPartitionColumns) + (rawSchema, partitionColumnNames) + } + // Schema-related metadata is lazily computed on first access within SchemaProvider + new SchemaProvider(SparkSession.active, rawSchema, partitionColumnNames) + } + + /** + * Creates a DeltaV2Table from a filesystem path without a catalog table. + * + * @param identifier logical table identifier used by Spark's catalog + * @param tablePath filesystem path to the Delta table root + * @throws NullPointerException if identifier or tablePath is null + */ + def this(identifier: Identifier, tablePath: String) = + this(identifier, tablePath, Collections.emptyMap(), Optional.empty(), None) + + /** + * Creates a DeltaV2Table from a filesystem path with options. + * + * @param identifier logical table identifier used by Spark's catalog + * @param tablePath filesystem path to the Delta table root + * @param options table options used to configure the Hadoop conf, table reads and writes + * @throws NullPointerException if identifier or tablePath is null + */ + def this(identifier: Identifier, tablePath: String, options: JMap[String, String]) = + this(identifier, tablePath, options, Optional.empty(), None) + + /** + * Path-based constructor pinned to an explicit time travel version (if present). + * + * @param identifier logical table identifier used by Spark's catalog + * @param tablePath filesystem path to the Delta table root + * @param options table options used to configure the Hadoop conf, table reads and writes + * @param timeTravelVersion the table version to pin the initial snapshot to + */ + def this( + identifier: Identifier, + tablePath: String, + options: JMap[String, String], + timeTravelVersion: Option[Long]) = + this(identifier, tablePath, options, Optional.empty(), timeTravelVersion) + + /** + * Catalog-table constructor pinned to an explicit time travel version (if present). + * + * @param identifier logical table identifier used by Spark's catalog + * @param catalogTable the Spark CatalogTable containing table metadata including location + * @param options user-provided options to override catalog properties + * @param timeTravelVersion the table version to pin the initial snapshot to + */ + def this( + identifier: Identifier, + catalogTable: CatalogTable, + options: JMap[String, String], + timeTravelVersion: Option[Long]) = + this( + identifier, + DeltaV2Table.getDecodedPath(requireNonNull(catalogTable, "catalogTable is null").location), + options, + Optional.of(catalogTable), + timeTravelVersion) + + /** + * Constructor that accepts a Spark CatalogTable and user-provided options. Extracts the table + * location and storage properties from the catalog table, then merges with user options. User + * options take precedence over catalog properties in case of conflicts. + * + * @param identifier logical table identifier used by Spark's catalog + * @param catalogTable the Spark CatalogTable containing table metadata including location + * @param options user-provided options to override catalog properties + */ + def this( + identifier: Identifier, + catalogTable: CatalogTable, + options: JMap[String, String]) = + this(identifier, catalogTable, options, None) + + /** + * Returns the CatalogTable if this DeltaV2Table was created from a catalog table. + * + * @return Optional containing the CatalogTable, or empty if this table was created from a path + */ + override def getCatalogTable(): Optional[CatalogTable] = catalogTableValue + + /** + * Returns the Path to the Delta table root. + * + * @return Path created from the table path + */ + override def getTablePath(): Path = new Path(tablePath) + + /** + * Returns the V2 identifier this table was constructed with. + * + * @return Identifier provided at construction + */ + def getIdentifier(): Identifier = identifier + + /** + * Returns the merged options (catalog storage + user options) this table was constructed with. + * + * @return Map of merged catalog storage and user options + */ + def getOptions(): JMap[String, String] = options + + /** + * Returns the snapshot manager backing this table. Catalog-driven features such as read-time CDF + * (TableCatalog.loadChangelog) use this to resolve versions, timestamps, and snapshots without + * having to build their own snapshot manager. + */ + def getSnapshotManager(): DeltaV2SnapshotManager = snapshotManager + + /** The table protocol from the initial snapshot. */ + protected def protocol(): AbstractProtocol = initialSnapshotValue.protocol + + /** The table metadata from the initial snapshot. */ + protected def metadata(): AbstractMetadata = initialSnapshotValue.metadata + + /** Inputs exposed to the Spark-version shim for metadata-only DELETE. */ + protected def kernelEngine(): KernelEngine = kernelContext.getDefaultEngine + + protected def initialSnapshot(): Snapshot = initialSnapshotValue + + protected def catalogTable(): Optional[CatalogTable] = catalogTableValue + + /** Returns a copy of this table pinned to {@code version}. */ + def withVersion(version: Long): DeltaV2Table = { + if (catalogTableValue.isPresent) { + new DeltaV2Table(identifier, catalogTableValue.get(), options, Some(version)) + } else { + new DeltaV2Table(identifier, tablePath, options, Some(version)) + } + } + + /** Returns a copy of this table pinned to the snapshot active at {@code timestampMicros}. */ + def withTimestamp(timestampMicros: Long): DeltaV2Table = + withVersion(resolveTimestampToVersion(snapshotManager, timestampMicros, queryContext)) + + /** + * Returns the table name in a format compatible with DeltaTableV2. + * + *

For catalog-based tables, returns the fully qualified table name (e.g., + * "spark_catalog.default.table_name"). For path-based tables, returns the path-based identifier + * (e.g., "delta.`/path/to/table`"). + * + * @return the table name string + */ + override def name(): String = + catalogTableValue.map[String](ct => ct.identifier.unquotedString) + .orElse("delta.`" + tablePath + "`") + + override def schema(): StructType = { + val base = schemaProvider.getPublicSchema() + if (isCDCRead) CDCSchemaContext.appendCDCColumns(base) else base + } + + override def columns(): Array[Column] = { + CatalogV2UtilShims.structTypeToV2Columns(schema()) + } + + override def partitioning(): Array[Transform] = schemaProvider.getPartitionTransforms() + + override def properties(): JMap[String, String] = { + val props = new HashMap[String, String](initialSnapshotValue.metadata.getConfiguration) + Collections.unmodifiableMap[String, String](props) + } + + override def capabilities(): JSet[TableCapability] = { + // A time travel pin cannot be written to. Dropping the write capabilities makes writes fall + // back to the v1 path. + if (isTimeTravel) READ_ONLY_CAPABILITIES else CAPABILITIES + } + + /** + * Returns the version of the snapshot loaded when this table was constructed, as a String. + * + *

Add {@code @Override} annotation eventually: TODO(#7128) + */ + override def version(): String = JLong.toString(initialSnapshotValue.version) + + /** + * Exposes file-level and row-tracking metadata via a single DSv2 metadata struct column. + * + *

This always returns one metadata column named {@code _metadata}. The struct contains Spark + * file-source base metadata fields with the same names and types as {@link + * org.apache.spark.sql.execution.datasources.FileFormat#BASE_METADATA_FIELDS} ({@code file_path}, + * {@code file_name}, {@code file_size}, {@code file_block_start}, {@code file_block_length}, + * {@code file_modification_time}); when row tracking is enabled, {@code row_id} and {@code + * row_commit_version} are appended. Values follow {@link + * org.apache.spark.sql.execution.datasources.PartitionedFile} semantics (see {@code + * FileFormat.BASE_METADATA_EXTRACTORS}). + * + *

Field order mirrors Spark's canonical {@code BASE_METADATA_FIELDS} for parity with V1 Delta, + * but resolution is name-based - order is not load-bearing for correctness. + */ + override def metadataColumns(): Array[MetadataColumn] = { + val rowTrackingEnabled = + RowTracking.isEnabled(initialSnapshotValue.protocol, initialSnapshotValue.metadata) + + var metadataType = new StructType() + FileFormat.BASE_METADATA_FIELDS.foreach { field => + metadataType = metadataType.add(field) + } + if (rowTrackingEnabled) { + metadataType = metadataType + .add(ROW_ID_METADATA_FIELD_NAME, DataTypes.LongType, false) + .add(ROW_COMMIT_VERSION_METADATA_FIELD_NAME, DataTypes.LongType, false) + } + val finalMetadataType = metadataType + + val columns = new Array[MetadataColumn](1) + columns(0) = new MetadataColumn { + override def name(): String = METADATA_COLUMN_NAME + override def dataType(): DataType = finalMetadataType + override def isNullable(): Boolean = false + } + columns + } + + override def newScanBuilder(scanOptions: CaseInsensitiveStringMap): ScanBuilder = { + val combined = new HashMap[String, String](options) + combined.putAll(scanOptions.asCaseSensitiveMap()) + val merged = new CaseInsensitiveStringMap(combined) + val catalogStats: Optional[Statistics] = + catalogTableValue + .flatMap[CatalogStatistics](ct => toJavaOptional(ct.stats)) + .map[Statistics](stats => + toV2Statistics( + stats, schemaProvider.getDataSchema(), schemaProvider.getPartitionSchema())) + DeltaV2ScanUtils.newScanBuilder( + name(), + initialSnapshotValue, + kernelEngine(), + catalogTableValue, + snapshotManager, + schemaProvider.getDataSchema(), + schemaProvider.getPartitionSchema(), + schemaProvider.getRawSchema(), + catalogStats, + merged) + } + + /** + * Validates that {@code version} exists in the Delta log, then loads the snapshot pinned to it. + */ + private def loadSnapshotAtCheckedVersion( + manager: DeltaV2SnapshotManager, + version: Long): Snapshot = { + val mustBeRecreatable = true + val allowOutOfRange = false + manager.checkVersionExists(version, mustBeRecreatable, allowOutOfRange, queryContext) + val loadSnapshot: Supplier[Snapshot] = () => manager.loadSnapshotAt(version, queryContext) + recordFrameProfileValue("snapshot.loadAtVersion", loadSnapshot) + } + + + override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = { + requireNonNull(info, "write info is null") + new DeltaV2WriteBuilder( + kernelEngine(), + tablePath, + hadoopConf, + initialSnapshotValue, + catalogTableValue, + snapshotManager, + schemaProvider.getDataSchema(), + schemaProvider.getPartitionSchema(), + info) + } + + /** + * Returns a builder for Delta row-level operations. The builder currently wires Spark planning to + * Delta's copy-on-write operation shell; the concrete ReplaceData write path is introduced in a + * follow-up PR. + */ + override def newRowLevelOperationBuilder( + info: RowLevelOperationInfo): RowLevelOperationBuilder = { + requireNonNull(info, "row-level operation info is null") + new DeltaRowLevelOperationBuilder( + this, kernelEngine(), hadoopConf, initialSnapshotValue, info) + } + + override def toString(): String = "DeltaV2Table{identifier=" + identifier + '}' + + override def equals(o: Any): Boolean = { + if (!o.isInstanceOf[DeltaV2Table]) { + return false + } + val that = o.asInstanceOf[DeltaV2Table] + if (this eq that) { + return true + } + if (getClass != that.getClass) { + return false + } + Objects.equals(identifier, that.identifier) && + Objects.equals(tablePath, that.tablePath) && + Objects.equals(options, that.options) && + Objects.equals(catalogTableValue, that.catalogTableValue) && + Objects.equals( + initialSnapshotValue.dataPath.toString, that.initialSnapshotValue.dataPath.toString) && + initialSnapshotValue.version == that.initialSnapshotValue.version && + Objects.equals( + initialSnapshotValue.metadata.getId, that.initialSnapshotValue.metadata.getId) + } + + override def hashCode(): Int = + Objects.hash( + identifier, + tablePath, + options, + catalogTableValue, + initialSnapshotValue.dataPath.toString, + Long.box(initialSnapshotValue.version), + initialSnapshotValue.metadata.getId) +} + +private[catalog] object DeltaV2Table { + private val METADATA_COLUMN_NAME = FileFormat.METADATA_NAME + private val ROW_ID_METADATA_FIELD_NAME = RowId.ROW_ID + private val ROW_COMMIT_VERSION_METADATA_FIELD_NAME = RowCommitVersion.METADATA_STRUCT_FIELD_NAME + + private val CAPABILITIES = buildCapabilities() + private val READ_ONLY_CAPABILITIES = buildReadOnlyCapabilities() + + private def buildCapabilities(): JSet[TableCapability] = { + val caps = EnumSet.of( + TableCapability.BATCH_READ, + TableCapability.MICRO_BATCH_READ, + TableCapability.BATCH_WRITE, + TableCapability.STREAMING_WRITE) + DeltaV2TableShims.schemaEvolutionCapability().ifPresent(capability => caps.add(capability)) + Collections.unmodifiableSet[TableCapability](caps) + } + + private def buildReadOnlyCapabilities(): JSet[TableCapability] = + Collections.unmodifiableSet[TableCapability]( + EnumSet.of(TableCapability.BATCH_READ, TableCapability.MICRO_BATCH_READ)) + + /** + * Helper method to decode URI path handling URL-encoded characters correctly. E.g., converts + * "spark%25dir%25prefix" to "spark%dir%prefix" + * + *

Uses Hadoop's Path class to properly handle all URI schemes (file, s3, abfss, gs, hdfs, + * etc.), not just file:// URIs. + */ + def getDecodedPath(location: URI): String = { + val hadoopPath = new Path(location) + // For local file system paths, return just the path component without the scheme + // to maintain consistency with path-based table construction where tablePath is a + // plain filesystem path string. + if (location.getScheme == null || "file".equals(location.getScheme)) { + return hadoopPath.toUri.getPath + } + hadoopPath.toString + } + + /** + * Resolves a time travel timestamp to the active commit version using the Kernel snapshot + * manager. + * + *

This loads the latest snapshot more than once (here and inside the Kernel lookup), make it + * share a singular load once the snapshot manager exposes it TODO(#5999). + */ + private def resolveTimestampToVersion( + manager: DeltaV2SnapshotManager, + timestampMicros: Long, + queryContext: DeltaV2QueryContext): Long = { + val timeMillis = timestampMicros / 1000 + val kernelCommit: KernelDeltaHistoryManager.Commit = manager.getActiveCommitAtTime( + timeMillis, + /* canReturnLastCommit = */ true, + /* mustBeRecreatable = */ true, + /* canReturnEarliestCommit = */ true, + queryContext) + val latestVersion = manager.loadLatestSnapshot(queryContext).version + if (kernelCommit.getTimestamp > timeMillis) { + // The earliest available commit is younger than the requested time. + throw new TimestampOutOfRangeException(timeMillis, kernelCommit.getTimestamp, false) + } else if (kernelCommit.getVersion == latestVersion && kernelCommit.getTimestamp < timeMillis) { + // The requested time is after the latest commit. + throw new TimestampOutOfRangeException(timeMillis, kernelCommit.getTimestamp, true) + } + kernelCommit.getVersion + } + + + /** + * Private helper class that lazily computes and caches schema-related metadata. + * + *

This class encapsulates all schema computation logic including: + * + *

    + *
  • Raw schema conversion from Kernel to Spark + *
  • Public schema with internal metadata removed + *
  • Data and partition schema derivation + *
  • Column and partition transform creation + *
+ * + *

All schema computations are deferred until first access. + */ + private class SchemaProvider( + sparkSession: SparkSession, + rawSchema: StructType, + partColNamesArg: JList[String]) { + private val partColNames = + Collections.unmodifiableList[String](new ArrayList[String](partColNamesArg)) + + // Lazily computed fields + private var initialized = false + private var publicSchema: StructType = null + private var dataSchema: StructType = null + private var partitionSchema: StructType = null + private var partitionTransforms: Array[Transform] = null + + private def ensureInitialized(): Unit = synchronized { + if (initialized) { + return + } + + // Create public schema by removing internal metadata (for schema() method) + publicSchema = DeltaTableUtils.removeInternalDeltaMetadata( + sparkSession, DeltaTableUtils.removeInternalWriterMetadata(sparkSession, rawSchema)) + + val dataFields = new ArrayList[StructField]() + val partitionFields = new ArrayList[StructField]() + + // Build a map for O(1) field lookups to improve performance + // Use rawSchema (with metadata) for deriving data and partition schemas + val fieldMap = new HashMap[String, StructField]() + rawSchema.fields.foreach { field => + fieldMap.put(field.name, field) + } + + // IMPORTANT: Add partition fields in the exact order specified by partColNames + // This is crucial because the order in partColNames may differ from the order + // in snapshotSchema, and we need to preserve the partColNames order for + // proper partitioning behavior + partColNames.forEach { partColName => + val field = fieldMap.get(partColName) + if (field != null) { + partitionFields.add(field) + } + } + + // Add remaining fields as data fields (non-partition columns) + // These are fields that exist in the schema but are not partition columns + rawSchema.fields.foreach { field => + if (!partColNames.contains(field.name)) { + dataFields.add(field) + } + } + dataSchema = new StructType(dataFields.toArray(new Array[StructField](0))) + partitionSchema = new StructType(partitionFields.toArray(new Array[StructField](0))) + + partitionTransforms = partColNames + .stream() + .map[Transform](partColName => Expressions.identity(partColName)) + .toArray[Transform](length => new Array[Transform](length)) + + initialized = true + } + + private def withInit[T](value: => T): T = { + ensureInitialized() + value + } + + def getPublicSchema(): StructType = withInit(publicSchema) + def getDataSchema(): StructType = withInit(dataSchema) + def getPartitionSchema(): StructType = withInit(partitionSchema) + def getRawSchema(): StructType = withInit(rawSchema) + def getPartitionTransforms(): Array[Transform] = withInit(partitionTransforms) + } +} + +/** + * Table.version() was added in Spark 4.1. + * A direct override would not compile before Spark 4.1. This trait supplies the contract + * independently so the same implementation also overrides the default method in Spark 4.1+. + */ +private[catalog] trait DeltaV2TableVersion { + def version(): String +} From a81a18776480a060c33abb5930dc06fbd035a20f Mon Sep 17 00:00:00 2001 From: Amogh Jahagirdar Date: Tue, 13 Oct 2026 17:16:03 -0600 Subject: [PATCH 14/29] [Spark] Align AMT manifest_info field IDs with the Iceberg spec and drop dv_cardinality (#7816) #### Which Delta project/connector is this regarding? - [x] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [ ] Other (fill in here) ## Description Renumber the AMT V4 manifest_info file/row count field IDs to match the spec PR: replaced_files_count 520 -> 523 modified_files_count 524 -> 525 replaced_rows_count 521 -> 524 modified_rows_count 525 -> 526 Separate DV cardinality Field id 523 is dropped now, readers have to read the cardinality from the bitmap itself. This field ID is now used for replaced_files_count. dv/dv_cardinality consistency invariant is now moot since there's only a single place so that logic is all dropped. format_version moves from top level field 157 into manifest_info as field 521, matching the spec PR. ## How was this patch tested? Existing unit tests, dropped no longer relevant assertions/tests for the fields that were removed. ## Does this PR introduce _any_ user-facing changes? --------- Co-authored-by: Isaac --- .../spark/sql/delta/amt/AMTUsageLogs.scala | 3 - .../spark/sql/delta/amt/AMTWriteHelper.scala | 16 ++-- .../sql/delta/amt/AMTWriterManager.scala | 2 +- .../sql/delta/amt/IncrementalAMTWriter.scala | 2 +- .../apache/spark/sql/delta/amt/actions.scala | 84 +++++++------------ .../sql/delta/LastCheckpointInfoSuite.scala | 4 +- .../delta/amt/AMTCheckpointWriteSuite.scala | 4 +- .../amt/AMTIncrementalWriteCoreSuite.scala | 2 +- .../AMTIncrementalWriteInvariantSuite.scala | 3 +- .../amt/AMTIncrementalWriteTestBase.scala | 11 +-- .../AMTIncrementalWriteTrackingMDVSuite.scala | 2 +- .../delta/amt/AMTInheritanceReadSuite.scala | 4 +- .../sql/delta/amt/AMTInheritanceSuite.scala | 4 +- .../amt/AMTSingleActionSerializerSuite.scala | 11 +-- .../sql/delta/amt/AMTSnapshotSuite.scala | 31 +------ 15 files changed, 63 insertions(+), 120 deletions(-) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/amt/AMTUsageLogs.scala b/spark/src/main/scala/org/apache/spark/sql/delta/amt/AMTUsageLogs.scala index 1b3d2b0aa6b..da46ae5a325 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/amt/AMTUsageLogs.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/amt/AMTUsageLogs.scala @@ -77,9 +77,6 @@ object AMTUsageLogs { val ALERT_UNEXPECTED_LEAF_TRACKING_STATUS = "amt.unexpectedLeafTrackingStatus" - // A manifest deletion vector has inconsistent bytes and cardinality. - val ALERT_MALFORMED_MANIFEST_DV = "amt.malformedManifestDV" - // Alert raised when a base-preserving winning commit's Add/Remove file carries a new commit // sequence number (defaultRowCommitVersion newer than the losing full checkpoint's version) // yet a non-empty back reference into the base tree -- contradictory signals. diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/amt/AMTWriteHelper.scala b/spark/src/main/scala/org/apache/spark/sql/delta/amt/AMTWriteHelper.scala index cc816d27116..5f7ff66d041 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/amt/AMTWriteHelper.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/amt/AMTWriteHelper.scala @@ -531,9 +531,9 @@ object AMTWriteHelper extends DeltaLogging { status = status, deleted_positions = bitmapOf(deletedPositions), replaced_positions = bitmapOf(replacedPositions)) - // manifest_info file/row counts are immutable; only the MDV (dv/dv_cardinality) grows to mask - // the newly superseded positions. Live count is record_count - dv_cardinality, and this - // commit's CDF positions live on the tracking, not in the counts. + // manifest_info file/row counts are immutable; only the MDV (`dv`) grows to mask the newly + // superseded positions. Live count is record_count - dvCardinality, and this commit's CDF + // positions live on the tracking, not in the counts. val manifestInfo = withUpdatedMdv(oldEntry.manifest_info, cumulativeMdv) (newTracking, manifestInfo) } @@ -599,16 +599,15 @@ object AMTWriteHelper extends DeltaLogging { // Deletion Vector. private[amt] def withUpdatedMdv(base: ManifestInfo, mdv: ManifestBitmap): ManifestInfo = { if (mdv.isEmpty) { - base.copy(dv = None, dv_cardinality = None) + base.copy(dv = None) } else { - base.copy( - dv = Some(AMTUtils.serializeMdv(mdv)), - dv_cardinality = Some(mdv.cardinality)) + base.copy(dv = Some(AMTUtils.serializeMdv(mdv))) } } private def emptyManifestInfo: ManifestInfo = ManifestInfo( + format_version = AMTSingleAction.FormatVersionV4, added_files_count = 0, existing_files_count = 0, deleted_files_count = 0, @@ -621,8 +620,7 @@ object AMTWriteHelper extends DeltaLogging { modified_rows_count = 0L, // Placeholder overridden before a newly written manifest is emitted. min_sequence_number = 0L, - dv = None, - dv_cardinality = None) + dv = None) /** * Writes a sequence of AMTSingleActions to a Parquet file. diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/amt/AMTWriterManager.scala b/spark/src/main/scala/org/apache/spark/sql/delta/amt/AMTWriterManager.scala index 46969a2d9e3..f257e3b9fc8 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/amt/AMTWriterManager.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/amt/AMTWriterManager.scala @@ -224,7 +224,7 @@ object SingleAMTWriteMetrics { val liveLeaves = leaves.filter(leaf => Tracking.Status.liveEntryStatuses.contains(leaf.tracking.status)) def maskedEntries(leaf: DataManifestEntry): Long = - leaf.manifest_info.dv_cardinality.getOrElse(0L) + leaf.manifest_info.dvCardinality SingleAMTWriteMetrics( trigger = trigger, incremental = incremental, diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/amt/IncrementalAMTWriter.scala b/spark/src/main/scala/org/apache/spark/sql/delta/amt/IncrementalAMTWriter.scala index bf402fd151d..b77dab233a9 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/amt/IncrementalAMTWriter.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/amt/IncrementalAMTWriter.scala @@ -370,7 +370,7 @@ class IncrementalAMTWriter(spark: SparkSession, deltaLog: DeltaLog) { // The MDV grew this commit: MODIFIED, or DELETED once every live entry is masked. AMTWriteHelper.modifiedOrDeletedTrackingForLeaf( pointer, newMdvPositions, deletedPositions, replacedPositions) - } else if (pointer.manifest_info.dv_cardinality.getOrElse(0L) == pointer.record_count) { + } else if (pointer.manifest_info.dvCardinality == pointer.record_count) { AMTWriteHelper.deletedTrackingForCarriedLeaf(pointer) } else { // Untouched leaf (no new MDVs) that still holds live files: carry it forward EXISTING. diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/amt/actions.scala b/spark/src/main/scala/org/apache/spark/sql/delta/amt/actions.scala index 6c2d2bdf97b..e1bb183441a 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/amt/actions.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/amt/actions.scala @@ -38,7 +38,6 @@ import org.apache.spark.sql.types.{MetadataBuilder, StructField, StructType} * of entry (data file or data-manifest pointer). * * @param content_type Entry-kind discriminator (0=DATA, 3=DATA_MANIFEST). - * @param format_version Iceberg writer format version; 4 for V4. * @param location Path of the file relative to the table root, or an absolute URI. * @param file_format Physical format of `location` (currently always "parquet"). * @param tracking Lineage envelope: status plus snapshot/sequence numbers and DV positions. @@ -60,7 +59,6 @@ import org.apache.spark.sql.types.{MetadataBuilder, StructField, StructType} */ case class AMTSingleAction( content_type: Int, // ID: 134, required. - format_version: Int, // ID: 157, required. location: String, // ID: 100, required. file_format: String, // ID: 101, required ("parquet" for now). tracking: Tracking, // ID: 147, required. @@ -98,8 +96,7 @@ case class AMTSingleAction( content_stats = content_stats, key_metadata = key_metadata, split_offsets = split_offsets, - tags = tags, - format_version = format_version) + tags = tags) case AMTSingleAction.ContentType.Type.DataManifest => DataManifestEntry( location = location, @@ -112,8 +109,7 @@ case class AMTSingleAction( spec_id = spec_id, content_stats = content_stats, key_metadata = key_metadata, - split_offsets = split_offsets, - format_version = format_version) + split_offsets = split_offsets) case other => throw new IllegalStateException(s"Unsupported content_type: $other.") } @@ -220,7 +216,6 @@ object AMTSingleAction { /** Iceberg V4 field specs for the top-level [[AMTSingleAction]] fields. */ private val topLevelFields: Seq[AMTFieldSpec] = Seq( required(134L, "content_type"), - required(157L, "format_version"), required(100L, "location"), required(101L, "file_format"), required(147L, "tracking"), @@ -256,19 +251,19 @@ object AMTSingleAction { /** Iceberg V4 field specs for the scalar fields of the nested [[ManifestInfo]] struct. */ private val manifestInfoFields: Seq[AMTFieldSpec] = Seq( + required(521L, "format_version"), required(504L, "added_files_count"), required(505L, "existing_files_count"), required(506L, "deleted_files_count"), - required(520L, "replaced_files_count"), - required(524L, "modified_files_count"), + required(523L, "replaced_files_count"), + required(525L, "modified_files_count"), required(512L, "added_rows_count"), required(513L, "existing_rows_count"), required(514L, "deleted_rows_count"), - required(521L, "replaced_rows_count"), - required(525L, "modified_rows_count"), + required(524L, "replaced_rows_count"), + required(526L, "modified_rows_count"), required(516L, "min_sequence_number"), - optional(522L, "dv"), - optional(523L, "dv_cardinality")) + optional(522L, "dv")) /** Nested-struct field specs, keyed by the top-level field name that carries the struct. */ private val nestedStructFields: Map[String, Seq[AMTFieldSpec]] = Map( @@ -448,7 +443,6 @@ sealed trait AMTAction { * consume these, but the field is kept in the schema to carry it forward * for tables written by both Delta and Iceberg. * @param tags Delta [[AddFile]] tags, preserved verbatim across the manifest round trip. - * @param format_version Iceberg writer format version; 4 for V4. */ case class DataEntry( location: String, @@ -463,15 +457,13 @@ case class DataEntry( content_stats: Option[String] = None, key_metadata: Option[Array[Byte]] = None, split_offsets: Option[Seq[Long]] = None, - tags: Option[Map[String, String]] = None, - format_version: Int = AMTSingleAction.FormatVersionV4) + tags: Option[Map[String, String]] = None) extends AMTAction { override protected def content_type: Int = AMTSingleAction.ContentType.Type.Data override def wrap: AMTSingleAction = AMTSingleAction( content_type = content_type, - format_version = format_version, location = location, file_format = file_format, tracking = tracking, @@ -670,10 +662,8 @@ object AMTPassthrough { */ def fromDataEntry(entry: DataEntry): Option[AMTPassthrough] = { require( - entry.file_format == AMTSingleAction.FileFormatParquet && - entry.format_version == AMTSingleAction.FormatVersionV4, - s"amtPassthrough only supports parquet/v4. got " + - s"${entry.file_format}/${entry.format_version}.") + entry.file_format == AMTSingleAction.FileFormatParquet, + s"amtPassthrough only supports parquet. got ${entry.file_format}.") val passthrough = AMTPassthrough( spec_id = entry.spec_id, sort_order_id = entry.sort_order_id, @@ -761,7 +751,6 @@ object AMTSnapshotId { * @param split_offsets Row-group split offsets (Iceberg field 132); Delta does not generate or * consume these, but the field is kept in the schema to carry it forward * for tables written by both Delta and Iceberg. - * @param format_version Iceberg writer format version; 4 for V4. */ case class DataManifestEntry( location: String, @@ -774,15 +763,13 @@ case class DataManifestEntry( spec_id: Option[Int] = None, content_stats: Option[String] = None, key_metadata: Option[Array[Byte]] = None, - split_offsets: Option[Seq[Long]] = None, - format_version: Int = AMTSingleAction.FormatVersionV4) + split_offsets: Option[Seq[Long]] = None) extends AMTAction { override protected def content_type: Int = AMTSingleAction.ContentType.Type.DataManifest override def wrap: AMTSingleAction = AMTSingleAction( content_type = content_type, - format_version = format_version, location = location, file_format = file_format, tracking = tracking, @@ -818,26 +805,11 @@ case class DataManifestEntry( /** * The inline manifest deletion vector on this leaf, if any, as (bitmap bytes, cardinality). - * Per the V4 spec, `dv` and `dv_cardinality` must both be set or both unset; a partially - * populated pair is malformed and rejected. + * The cardinality is derived by decoding the bitmap. */ @JsonIgnore - def manifestDV: Option[(Array[Byte], Long)] = { - AMTUtils.invariantCheckWithLogging( - checkInvariant = manifest_info.dv.isDefined == manifest_info.dv_cardinality.isDefined, - opTypeSuffix = AMTUsageLogs.ALERT_MALFORMED_MANIFEST_DV, - message = - s"Malformed manifest DV on leaf $location: dv and dv_cardinality must both be set or " + - s"both unset (dv.isDefined=${manifest_info.dv.isDefined}, " + - s"dv_cardinality=${manifest_info.dv_cardinality}).", - data = Map( - "leafLocation" -> location, - "dvIsDefined" -> manifest_info.dv.isDefined, - "dvCardinalityIsDefined" -> manifest_info.dv_cardinality.isDefined)) - manifest_info.dv.flatMap { dv => - manifest_info.dv_cardinality.map(cardinality => (dv, cardinality)) - } - } + def manifestDV: Option[(Array[Byte], Long)] = + manifest_info.dv.map(dv => (dv, manifest_info.dvCardinality)) } /** @@ -1099,10 +1071,9 @@ object DeletionVector { * Field IDs and required/optional match the V4 `manifest_info` struct verbatim. * * `dv` carries the inline manifest deletion-vector bitmap over leaf row - * positions; `dv_cardinality` is its count. Manifest DVs live INSIDE - * `manifest_info`, not as separate root rows -- a deliberate choice in - * the Combined Data + DV Entry model. + * positions; its cardinality is derived on demand via [[ManifestInfo.dvCardinality]] * + * @param format_version Format version of the referenced manifest (0=PRE-V4, 4=V4). * @param added_files_count Count of ADDED file entries in the referenced manifest. * @param existing_files_count Count of EXISTING file entries. * @param deleted_files_count Count of DELETED file entries. @@ -1115,26 +1086,33 @@ object DeletionVector { * @param modified_rows_count Rows across MODIFIED files. * @param min_sequence_number Minimum data sequence number across the manifest's live entries. * @param dv Inline manifest deletion-vector bitmap over leaf row positions. - * @param dv_cardinality Number of positions the inline manifest DV marks. */ case class ManifestInfo( + format_version: Int, // ID: 521, required. added_files_count: Int, // ID: 504, required. existing_files_count: Int, // ID: 505, required. deleted_files_count: Int, // ID: 506, required. - replaced_files_count: Int, // ID: 520, required. - modified_files_count: Int, // ID: 524, required. + replaced_files_count: Int, // ID: 523, required. + modified_files_count: Int, // ID: 525, required. added_rows_count: Long, // ID: 512, required. existing_rows_count: Long, // ID: 513, required. deleted_rows_count: Long, // ID: 514, required. - replaced_rows_count: Long, // ID: 521, required. - modified_rows_count: Long, // ID: 525, required. + replaced_rows_count: Long, // ID: 524, required. + modified_rows_count: Long, // ID: 526, required. min_sequence_number: Long, // ID: 516, required. - dv: Option[Array[Byte]], // ID: 522, optional (inline manifest DV bitmap). - dv_cardinality: Option[Long]) { // ID: 523, optional. + dv: Option[Array[Byte]]) { // ID: 522, optional (inline manifest DV bitmap). /** Live (non-tombstone) file entries: ADDED, EXISTING and MODIFIED are all live. */ def liveFilesCount: Int = added_files_count + existing_files_count + modified_files_count /** Tombstone file entries: DELETED and REPLACED. */ def tombstoneFilesCount: Int = deleted_files_count + replaced_files_count + + /** + * Cardinality of the inline manifest DV, derived by decoding `dv` (0 when absent). + * + * TODO: A follow-on may deserialize MDVs more selectively, possibly eliminating this function. + */ + @JsonIgnore + def dvCardinality: Long = dv.map(AMTUtils.deserializeMdv(_).cardinality).getOrElse(0L) } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/LastCheckpointInfoSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/LastCheckpointInfoSuite.scala index dd0b1975c7c..7db3e45eeb5 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/LastCheckpointInfoSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/LastCheckpointInfoSuite.scala @@ -296,6 +296,7 @@ class LastCheckpointInfoSuite extends SharedSparkSession record_count = recordCount, file_size_in_bytes = sizeInBytes, manifest_info = ManifestInfo( + format_version = AMTSingleAction.FormatVersionV4, added_files_count = recordCount.toInt, existing_files_count = 2, deleted_files_count = 3, @@ -307,8 +308,7 @@ class LastCheckpointInfoSuite extends SharedSparkSession replaced_rows_count = 400L, modified_rows_count = 500L, min_sequence_number = 7L, - dv = Some(Array[Byte](9, 8, 7)), - dv_cardinality = Some(5L)), + dv = Some(Array[Byte](9, 8, 7))), partition = Some(Map("part_col" -> "part_val")), spec_id = Some(1), content_stats = Some("""{"minValues":{"col-1":1}}"""), diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTCheckpointWriteSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTCheckpointWriteSuite.scala index 76f4dae57da..af958046e5e 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTCheckpointWriteSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTCheckpointWriteSuite.scala @@ -1002,6 +1002,7 @@ class AMTCheckpointWriteSuite extends AMTCheckpointTestBase { /** A zeroed [[ManifestInfo]]; tests set only the counts they assert on via `copy`. */ private def emptyManifestInfo: ManifestInfo = ManifestInfo( + format_version = AMTSingleAction.FormatVersionV4, added_files_count = 0, existing_files_count = 0, deleted_files_count = 0, @@ -1013,6 +1014,5 @@ class AMTCheckpointWriteSuite extends AMTCheckpointTestBase { replaced_rows_count = 0L, modified_rows_count = 0L, min_sequence_number = 0L, - dv = None, - dv_cardinality = None) + dv = None) } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTIncrementalWriteCoreSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTIncrementalWriteCoreSuite.scala index 40539feb5cd..e74b39ea6b8 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTIncrementalWriteCoreSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTIncrementalWriteCoreSuite.scala @@ -265,7 +265,7 @@ class AMTIncrementalWriteCoreSuite extends AMTIncrementalWriteTestBase { // The two removes share one (leaf, position), which the writer holds as a set, so the write // reports a single MDV bit, matching what the leaf's bitmap actually gains. That // agreement is what lets this go through the shared validator, whose second check - // derives the bits from the on-disk dv_cardinality delta. + // derives the bits from the on-disk MDV bitmap. createIncrementalAMTAndValidate( baselineDeltaLog, amtDeltaLog, diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTIncrementalWriteInvariantSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTIncrementalWriteInvariantSuite.scala index 31cb921cea4..6f579918551 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTIncrementalWriteInvariantSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTIncrementalWriteInvariantSuite.scala @@ -184,12 +184,13 @@ class AMTIncrementalWriteInvariantSuite extends AMTIncrementalWriteTestBase { (addedFiles + existingFiles + deletedFiles + replacedFiles + modifiedFiles).toLong, file_size_in_bytes = 100L, manifest_info = ManifestInfo( + format_version = AMTSingleAction.FormatVersionV4, added_files_count = addedFiles, existing_files_count = existingFiles, deleted_files_count = deletedFiles, replaced_files_count = replacedFiles, modified_files_count = modifiedFiles, added_rows_count = 0L, existing_rows_count = 0L, deleted_rows_count = 0L, replaced_rows_count = 0L, modified_rows_count = 0L, - min_sequence_number = 0L, dv = None, dv_cardinality = None)) + min_sequence_number = 0L, dv = None)) test("carryForwardOneLeaf rejects a carried leaf mixing live files and tombstones") { withTables() { (_, amtDeltaLog) => diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTIncrementalWriteTestBase.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTIncrementalWriteTestBase.scala index 0b4df38c75b..8fcc1449e1f 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTIncrementalWriteTestBase.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTIncrementalWriteTestBase.scala @@ -204,7 +204,7 @@ abstract class AMTIncrementalWriteTestBase extends AMTCheckpointTestBase { protected def leafToLeafMDVCardinalityMap(amtDeltaLog: DeltaLog): Map[String, Long] = { val provider = amtProvider(amtDeltaLog.update()) .getOrElse(fail("AMT table must be checkpoint-provider-backed.")) - provider.leaves.map(l => l.location -> l.manifest_info.dv_cardinality.getOrElse(0L)).toMap + provider.leaves.map(l => l.location -> l.manifest_info.dvCardinality).toMap } /** Root-resident DATA-entry counts (live adds, tombstones) read straight off the new root. */ @@ -241,7 +241,7 @@ abstract class AMTIncrementalWriteTestBase extends AMTCheckpointTestBase { /** The MDV cardinality numLeafCountBefore on a leaf pointer, 0 when it has none. */ protected def mdvCardinality(leaf: DataManifestEntry): Long = - leaf.manifest_info.dv_cardinality.getOrElse(0L) + leaf.manifest_info.dvCardinality /** * Emits one incremental AMT and cross-checks its metrics three ways: the hand-specified @@ -425,7 +425,7 @@ abstract class AMTIncrementalWriteTestBase extends AMTCheckpointTestBase { /** * Performs following validations on the underlying AMT (from latest snapshot): - * - each leaf pointer's `dv_cardinality` equals its decoded MDV bitmap size, and every MDV + * - each leaf pointer's MDV cardinality is within its physical entry count, and every MDV * position is in range of that leaf's physical entry count (no stale / out-of-range mask); * - conservation: the reader's live count equals root-resident live adds plus, over every leaf, * (physical leaf entries - MDV cardinality) @@ -457,12 +457,9 @@ abstract class AMTIncrementalWriteTestBase extends AMTCheckpointTestBase { val livePhysicalEntries = statusToCountMapForLeaf .filter { case (status, _) => Tracking.Status.liveEntryStatuses.contains(status) } .values.sum - val mdvDeclaredCardinality = leaf.manifest_info.dv_cardinality.getOrElse(0L) + val mdvDeclaredCardinality = leaf.manifest_info.dvCardinality val decoded = leaf.manifest_info.dv.map(ManifestBitmap.fromSerializedByteArray) .getOrElse(ManifestBitmap.fromPositions(Seq.empty)) - assert(decoded.cardinality == mdvDeclaredCardinality, - s"Leaf ${leaf.location}: dv_cardinality=$mdvDeclaredCardinality but the decoded MDV " + - s"has ${decoded.cardinality} bits.") assert(mdvDeclaredCardinality <= physicalEntries, s"Leaf ${leaf.location}: MDV cardinality ($mdvDeclaredCardinality) exceeds physical " + s"entries " + diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTIncrementalWriteTrackingMDVSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTIncrementalWriteTrackingMDVSuite.scala index 2fa2b19c1dd..d184e21f400 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTIncrementalWriteTrackingMDVSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTIncrementalWriteTrackingMDVSuite.scala @@ -270,7 +270,7 @@ class AMTIncrementalWriteTrackingMDVSuite extends AMTIncrementalWriteTestBase { numLeavesExistingStatus = leafToAddFileMapping.size - 1)) val provider = amtProvider(amtDeltaLog.update()).getOrElse(fail("expected AMT")) provider.leaves.foreach { leaf => - val card = leaf.manifest_info.dv_cardinality.getOrElse(0L) + val card = leaf.manifest_info.dvCardinality if (leaf.location == victimLeaf) { assert(card == 1L, s"The victim leaf must carry exactly one MDV bit; got $card.") } else { diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTInheritanceReadSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTInheritanceReadSuite.scala index 5554a51ee32..76ac0d6df27 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTInheritanceReadSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTInheritanceReadSuite.scala @@ -175,6 +175,7 @@ class AMTInheritanceReadSuite extends AMTCheckpointTestBase { record_count = leaf.entries.size.toLong, file_size_in_bytes = leaf.sizeInBytes, manifest_info = ManifestInfo( + format_version = AMTSingleAction.FormatVersionV4, added_files_count = leaf.entries.size, existing_files_count = 0, deleted_files_count = 0, @@ -187,8 +188,7 @@ class AMTInheritanceReadSuite extends AMTCheckpointTestBase { modified_rows_count = 0L, min_sequence_number = 0L, dv = mdv.map(positions => - AMTUtils.serializeMdv(ManifestBitmap.fromPositions(positions.map(_.toInt)))), - dv_cardinality = mdv.map(_.size.toLong))) + AMTUtils.serializeMdv(ManifestBitmap.fromPositions(positions.map(_.toInt)))))) } /** The messages of `error` and of every exception it wraps. */ diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTInheritanceSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTInheritanceSuite.scala index 05e85db1ac1..9661c43c7fc 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTInheritanceSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTInheritanceSuite.scala @@ -66,6 +66,7 @@ class AMTInheritanceSuite extends SparkFunSuite { record_count = 1L, file_size_in_bytes = 1L, manifest_info = ManifestInfo( + format_version = AMTSingleAction.FormatVersionV4, added_files_count = 1, existing_files_count = 0, deleted_files_count = 0, @@ -77,8 +78,7 @@ class AMTInheritanceSuite extends SparkFunSuite { replaced_rows_count = 0L, modified_rows_count = 0L, min_sequence_number = 0L, - dv = None, - dv_cardinality = None)) + dv = None)) test("an ADDED entry with null tracking inherits every inheritable field") { val resolved = resolve(childTracking()) diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTSingleActionSerializerSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTSingleActionSerializerSuite.scala index cb216ab6625..140719774f9 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTSingleActionSerializerSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTSingleActionSerializerSuite.scala @@ -73,6 +73,7 @@ class AMTSingleActionSerializerSuite extends QueryTest with SharedSparkSession { stats = """{"numRecords":42}""") private def sampleManifestInfo: ManifestInfo = ManifestInfo( + format_version = AMTSingleAction.FormatVersionV4, added_files_count = 2, existing_files_count = 0, deleted_files_count = 0, @@ -84,12 +85,11 @@ class AMTSingleActionSerializerSuite extends QueryTest with SharedSparkSession { replaced_rows_count = 0L, modified_rows_count = 7L, min_sequence_number = 3L, - dv = None, - dv_cardinality = None) + dv = None) test("encoder schema has the expected V4 column names in order") { assert(spark.emptyDataset[AMTSingleAction].schema.fieldNames.toSeq == Seq( - "content_type", "format_version", "location", "file_format", "tracking", + "content_type", "location", "file_format", "tracking", "deletion_vector", "spec_id", "partition", "sort_order_id", "record_count", "file_size_in_bytes", "content_stats", "manifest_info", "key_metadata", "split_offsets", "tags")) @@ -191,7 +191,6 @@ class AMTSingleActionSerializerSuite extends QueryTest with SharedSparkSession { tags: Option[Map[String, String]] = None, tracking: Tracking = addedTracking): AMTSingleAction = AMTSingleAction( content_type = content_type, - format_version = AMTSingleAction.FormatVersionV4, location = "f.parquet", file_format = AMTSingleAction.FileFormatParquet, tracking = tracking, @@ -294,7 +293,7 @@ class AMTSingleActionSerializerSuite extends QueryTest with SharedSparkSession { deleted_positions = Some(deletedPos), replaced_positions = Some(replacedPos)), record_count = 1L, file_size_in_bytes = 1L, - manifest_info = sampleManifestInfo.copy(dv = Some(manifestDv), dv_cardinality = Some(2L)), + manifest_info = sampleManifestInfo.copy(dv = Some(manifestDv)), key_metadata = Some(keyMeta)).wrap val path = new java.io.File(dir, "binary").getCanonicalPath spark.createDataset(Seq(entry)).write.parquet(path) @@ -444,14 +443,12 @@ class AMTSingleActionSerializerSuite extends QueryTest with SharedSparkSession { "modified_snapshot_id must be null when the entry has no modification metadata") // file_format / format_version are not carried; they are reconstructed at the M1 defaults. assert(restored.file_format == AMTSingleAction.FileFormatParquet) - assert(restored.format_version == AMTSingleAction.FormatVersionV4) } test("fromAddFile falls back to defaults with no passthrough") { val restored = DataEntry.fromAddFile(sampleAddFile, addedTracking, tableRoot) assert(restored.file_format == AMTSingleAction.FileFormatParquet) assert(restored.spec_id.isEmpty) - assert(restored.format_version == AMTSingleAction.FormatVersionV4) } test("fromAddFile maps snapshot IDs only from top-level AddFile fields") { diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTSnapshotSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTSnapshotSuite.scala index 5b67735e5a8..d1582c049c5 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTSnapshotSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTSnapshotSuite.scala @@ -1269,7 +1269,7 @@ class AMTSnapshotSuite extends AMTCheckpointTestBase with DeletionVectorsTestUti // Patch the first leaf's pointer with an MDV marking `deletedPos` deleted, then reconstruct. val patchedLeaves = leaf.copy(manifest_info = - leaf.manifest_info.copy(dv = Some(mdvBytesFor(deletedPos)), dv_cardinality = Some(1L))) +: + leaf.manifest_info.copy(dv = Some(mdvBytesFor(deletedPos)))) +: base.leaves.tail val provider = new AMTCheckpointProvider( base.manifestCommitVersion, base.checkpointAction, patchedLeaves, base.tableRoot) @@ -1304,7 +1304,7 @@ class AMTSnapshotSuite extends AMTCheckpointTestBase with DeletionVectorsTestUti def withMdv(idx: Int, positions: Seq[Long]): DataManifestEntry = { val leaf = base.leaves(idx) leaf.copy(manifest_info = leaf.manifest_info.copy( - dv = Some(mdvBytesFor(positions: _*)), dv_cardinality = Some(positions.size.toLong))) + dv = Some(mdvBytesFor(positions: _*)))) } // Exercise every MDV shape in one tree (positions are leaf-local row indices): // - one leaf: drop a single position (pos 0), @@ -1401,7 +1401,7 @@ class AMTSnapshotSuite extends AMTCheckpointTestBase with DeletionVectorsTestUti val patched = base.leaves.map { leaf => if (leaf eq bLeafAndPos._1) { leaf.copy(manifest_info = leaf.manifest_info.copy( - dv = Some(mdvBytesFor(bLeafAndPos._2)), dv_cardinality = Some(1L))) + dv = Some(mdvBytesFor(bLeafAndPos._2)))) } else leaf } val provider = new AMTCheckpointProvider( @@ -1426,31 +1426,6 @@ class AMTSnapshotSuite extends AMTCheckpointTestBase with DeletionVectorsTestUti "The surviving entry's stats must be unchanged by MDV filtering.") } - testAcrossAMTCheckpointScenarios( - "a manifest DV with only one of dv/dv_cardinality set is rejected", - "amt_mdv_malformed", - sqlConfs = leafPackingConfs)( - setup = name => appendRowsAsSeparateFiles(name, numFiles = leafPackedFiles - 1), - inlineCheckpointTriggerActionsOrSQL = Some(name => Right( - s"INSERT INTO $name VALUES (${leafPackedFiles - 1})"))) { context => - val base = context.provider - - // `dv` set but `dv_cardinality` missing: the AMT spec requires both or neither. - val patched = - base.leaves.head.copy(manifest_info = base.leaves.head.manifest_info.copy( - dv = Some(mdvBytesFor(0L)), dv_cardinality = None)) +: - base.leaves.tail - val provider = new AMTCheckpointProvider( - base.manifestCommitVersion, base.checkpointAction, patched, base.tableRoot) - - val e = intercept[IllegalStateException] { - provider.loadActionsForStateReconstruction( - spark, context.postCheckpointSnapshot.deltaLog) - } - assert(e.getMessage.contains("dv and dv_cardinality must both be set or both unset"), - s"Unexpected message: ${e.getMessage}") - } - /** Reconstructed live `add.path`s from the (possibly MDV-patched) provider. */ private def reconstructedPaths( provider: AMTCheckpointProvider, deltaLog: DeltaLog): Set[String] = From f1a1e1bedba8e3b3e920803064944c98d745a51e Mon Sep 17 00:00:00 2001 From: You Zhou <98635051+PorridgeSwim@users.noreply.github.com> Date: Tue, 13 Oct 2026 19:58:02 -0700 Subject: [PATCH 15/29] [Spark] Log streaming source events through snapshots (#7859) ## Description Use `snapshotAtSourceInit` to provide table tags for four existing streaming source events: the unsafe column-mapping read warning, missing trailing commits, schema changes, and the `ignoreFileDeletion` deprecation warning. This keeps event attribution tied to the snapshot the source was initialized to read instead of the latest snapshot cached by `DeltaLog`. Event names, payloads, exception handling, and streaming behavior are unchanged. No additional snapshot fetches are introduced. ## Why this is safe `recordDeltaEvent` uses its provider only for table-context tags. Snapshots delegate to the same `DeltaLog.getCommonTags` helper; only the source of the table identifier tag changes. This affects attribution, not logging mechanics or streaming decisions. The table identifier is the table's `metadata.id`, not its version. Ordinary data writes and schema updates preserve it. Table replacement or drop-and-recreation can create a new table identity and change the ID, even at the same path. - **Source construction:** `snapshotAtSourceInit` is the snapshot used to construct the source. Initialization and deprecation warnings should identify that source, regardless of later table changes. - **Commit-processing diagnostics:** events describe the existing source and should carry its pinned `tableId`, obtained from `snapshotAtSourceInit.metadata.id`. When commit metadata validation runs, an ID mismatch fails before that commit's files are processed; schema-change logging also follows the ID check. Existing ID and schema-tracking checks remain unchanged. ## Testing - `git diff --check` - Verified that `SnapshotDescriptor` implements `DeltaLoggingProvider`, the provider type accepted by `recordDeltaEvent`. - Builds and tests have not been run locally. --- .../org/apache/spark/sql/delta/sources/DeltaSource.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSource.scala b/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSource.scala index 5fe7e6b1ce5..9debaa9fba9 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSource.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSource.scala @@ -125,7 +125,7 @@ trait DeltaSourceBase extends Source snapshotAtSourceInit.metadata.columnMappingMode != NoMapping, isTypeWideningSupportedInProtocol = TypeWidening.isSupported(snapshotAtSourceInit.protocol)) if (schemaReadOptions.allowUnsafeStreamingReadOnColumnMappingSchemaChanges) recordDeltaEvent( - deltaLog, + snapshotAtSourceInit, "delta.unsafe.streaming.readOnColumnMappingSchemaChanges" ) schemaReadOptions @@ -355,7 +355,7 @@ trait DeltaSourceBase extends Source maxVersionSeen: Long, isStreamingCDC: Boolean): Unit = { recordDeltaEvent( - deltaLog, + snapshotAtSourceInit, opType = "delta.exceptions.streamingTrailingCommitMissing", data = Map( "stackTrace" -> Thread.currentThread().getStackTrace.tail.mkString("\n\t"), @@ -658,7 +658,7 @@ trait DeltaSourceBase extends Source val isRetryable = DeltaStreamUtils.SchemaCompatibilityResult.isRetryableIncompatible(checkResult) recordDeltaEvent( - deltaLog, + snapshotAtSourceInit, "delta.streaming.source.schemaChanged", data = Map( "currentVersion" -> snapshotAtSourceInit.version, @@ -764,7 +764,7 @@ case class DeltaSource( private val ignoreFileDeletion = { if (options.ignoreFileDeletion) { logConsole(DeltaErrors.ignoreStreamingUpdatesAndDeletesWarning(spark)) - recordDeltaEvent(deltaLog, "delta.deprecation.ignoreFileDeletion") + recordDeltaEvent(snapshotAtSourceInit, "delta.deprecation.ignoreFileDeletion") } options.ignoreFileDeletion } From 589c1254affad8d6887e80d1c7c66a12f8bd54c0 Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Tue, 13 Oct 2026 23:26:18 -0700 Subject: [PATCH 16/29] [Spark] Decode CDC actions from Kernel commit batches (#7857) #### Which Delta project/connector is this regarding? - [x] Spark ## Description Decode CDC actions from Kernel commit batches into `AddCDCFile` actions, preserving the path, partition values, size, and optional tags. These actions previously caused an `UnsupportedOperationException` during commit-batch conversion. Rename the shared string-map converter to reflect its use for both partition values and tags. ## How was this patch tested? Added coverage in `KernelActionUtilsSuite` for two CDC cases in one test method: an action without tags and an action with all supported fields populated. Local tests have not been run in this checkout; CI validation is pending. ## Does this PR introduce _any_ user-facing changes? Kernel commit-batch conversion now supports CDC actions. Signed-off-by: Hao Sun <86338940+HaoSun1993@users.noreply.github.com> --- .../v2/kernel/KernelActionUtilsSuite.scala | 30 +++++++++++----- .../delta/v2/kernel/KernelActionUtils.scala | 36 ++++++++++++++----- 2 files changed, 49 insertions(+), 17 deletions(-) diff --git a/spark-unified/src/test/scala/org/apache/spark/sql/delta/v2/kernel/KernelActionUtilsSuite.scala b/spark-unified/src/test/scala/org/apache/spark/sql/delta/v2/kernel/KernelActionUtilsSuite.scala index dbc109cb7a3..897e8252743 100644 --- a/spark-unified/src/test/scala/org/apache/spark/sql/delta/v2/kernel/KernelActionUtilsSuite.scala +++ b/spark-unified/src/test/scala/org/apache/spark/sql/delta/v2/kernel/KernelActionUtilsSuite.scala @@ -20,7 +20,7 @@ import java.util.Optional import scala.collection.JavaConverters._ -import org.apache.spark.sql.delta.actions.{Action, AddFile, DeletionVectorDescriptor, DomainMetadata} +import org.apache.spark.sql.delta.actions.{Action, AddCDCFile, AddFile, DeletionVectorDescriptor, DomainMetadata} import io.delta.kernel.{CommitActions => KernelCommitActions} import io.delta.kernel.data.{ColumnarBatch => KernelColumnarBatch} import io.delta.kernel.data.{ColumnVector => KernelColumnVector} @@ -28,6 +28,7 @@ import io.delta.kernel.data.{Row => KernelRow} import io.delta.kernel.data.MapValue import io.delta.kernel.defaults.internal.json.{JsonUtils => KernelJsonUtils} import io.delta.kernel.internal.DeltaLogActionUtils.{DeltaAction => KernelDeltaAction} +import io.delta.kernel.internal.actions.{AddCDCFile => KernelAddCDCFile} import io.delta.kernel.internal.actions.{AddFile => KernelAddFile} import io.delta.kernel.internal.actions.{CommitInfo => KernelCommitInfo} import io.delta.kernel.internal.actions.{ @@ -514,15 +515,26 @@ class KernelActionUtilsSuite extends SparkFunSuite { DomainMetadata("test.userDomain", "{}", removed = true))) } - test("readActions fails loud on an action type with no decoder (hand-built commit batch)") { - val presentColumn = new GenericColumnVector( - java.util.Collections.singletonList("present"), StringType.STRING) - val action = KernelDeltaAction.CDC - val e = intercept[UnsupportedOperationException] { - KernelActionUtils.readActions(singleColumnCommitActions(action.colName, presentColumn)) + test("readActions decodes CDC files with and without tags") { + val testCases = Seq( + """{"path":"_change_data/no-tags.parquet","partitionValues":{"p":"1"},"size":10}""" -> + AddCDCFile("_change_data/no-tags.parquet", Map("p" -> "1"), 10L), + """{"path":"_change_data/with-tags.parquet","partitionValues":{"p":"2"},"size":20, + |"tags":{"tag-a":"value-a"}}""".stripMargin -> + AddCDCFile( + "_change_data/with-tags.parquet", + Map("p" -> "2"), + 20L, + tags = Map("tag-a" -> "value-a"))) + + testCases.foreach { case (json, expected) => + val kernelRow = KernelJsonUtils.rowFromJson(json, KernelAddCDCFile.FULL_SCHEMA) + val kernelColumn = new GenericColumnVector( + java.util.Collections.singletonList(kernelRow), KernelAddCDCFile.FULL_SCHEMA) + + assert(KernelActionUtils.readActions( + singleColumnCommitActions(KernelDeltaAction.CDC.colName, kernelColumn)) === Seq(expected)) } - assert(e.getMessage.contains("No V1 action from Kernel decoder")) - assert(e.getMessage.contains(action.colName)) } } diff --git a/spark/v2/src/main/scala/org/apache/spark/sql/delta/v2/kernel/KernelActionUtils.scala b/spark/v2/src/main/scala/org/apache/spark/sql/delta/v2/kernel/KernelActionUtils.scala index 91dbcd50db8..1b7473dc805 100644 --- a/spark/v2/src/main/scala/org/apache/spark/sql/delta/v2/kernel/KernelActionUtils.scala +++ b/spark/v2/src/main/scala/org/apache/spark/sql/delta/v2/kernel/KernelActionUtils.scala @@ -23,6 +23,7 @@ import scala.jdk.OptionConverters._ import org.apache.spark.sql.delta.actions.{ Action, + AddCDCFile, AddFile, CommitInfo, DeletionVectorDescriptor, @@ -143,6 +144,8 @@ private[v2] object KernelActionUtils { case KernelDeltaAction.COMMITINFO => commitInfoFromKernel( KernelCommitInfo.fromColumnVector(columnVector, rowId)) + case KernelDeltaAction.CDC => + addCDCFileFromKernel(KernelStructRow.fromStructVector(columnVector, rowId)) case KernelDeltaAction.TXN => setTransactionFromKernel( KernelSetTransaction.fromColumnVector(columnVector, rowId)) @@ -160,7 +163,7 @@ private[v2] object KernelActionUtils { def addFileFromKernel(addFile: KernelAddFile): AddFile = { AddFile( path = addFile.getPath, - partitionValues = partitionValuesFromKernel(addFile.getPartitionValues), + partitionValues = stringMapFromKernel(addFile.getPartitionValues), size = addFile.getSize, modificationTime = addFile.getModificationTime, dataChange = addFile.getDataChange, @@ -212,7 +215,7 @@ private[v2] object KernelActionUtils { deletionTimestamp = removeFile.getDeletionTimestamp.toScala.map(_.longValue()), dataChange = removeFile.getDataChange, extendedFileMetadata = removeFile.getExtendedFileMetadata.toScala.map(_.booleanValue()), - partitionValues = removeFile.getPartitionValues.toScala.map(partitionValuesFromKernel).orNull, + partitionValues = removeFile.getPartitionValues.toScala.map(stringMapFromKernel).orNull, size = removeFile.getSize.toScala.map(_.longValue()), tags = tagsFromKernel(removeFile.getTags), deletionVector = deletionVectorFromKernel(removeFile.getDeletionVector), @@ -222,6 +225,23 @@ private[v2] object KernelActionUtils { // scalastyle:on removeFile } + /** + * Converts a Kernel CDC action row into a V1 [[AddCDCFile]]. + */ + private def addCDCFileFromKernel(kernelRow: KernelRow): AddCDCFile = { + // Kernel exposes the CDC schema without an action wrapper, so read its fields by name. + val kernelSchema = kernelRow.getSchema + val tagsOrdinal = kernelSchema.indexOf("tags") + AddCDCFile( + path = kernelRow.getString(kernelSchema.indexOf("path")), + partitionValues = stringMapFromKernel( + kernelRow.getMap(kernelSchema.indexOf("partitionValues"))), + size = kernelRow.getLong(kernelSchema.indexOf("size")), + tags = + if (kernelRow.isNullAt(tagsOrdinal)) null + else stringMapFromKernel(kernelRow.getMap(tagsOrdinal))) + } + /** * Converts a Kernel [[KernelMetadata]] into a V1 [[Metadata]]. */ @@ -318,16 +338,16 @@ private[v2] object KernelActionUtils { cardinality = dv.getCardinality) }.orNull - private def partitionValuesFromKernel(partitionValues: KernelMapValue): Map[String, String] = { - if (partitionValues == null) return Map.empty - val keys = partitionValues.getKeys - val values = partitionValues.getValues - (0 until partitionValues.getSize).map { index => + private def stringMapFromKernel(kernelMap: KernelMapValue): Map[String, String] = { + if (kernelMap == null) return Map.empty + val keys = kernelMap.getKeys + val values = kernelMap.getValues + (0 until kernelMap.getSize).map { index => val value = if (values.isNullAt(index)) null else values.getString(index) keys.getString(index) -> value }.toMap } private def tagsFromKernel(tags: java.util.Optional[KernelMapValue]): Map[String, String] = - tags.toScala.map(partitionValuesFromKernel).orNull + tags.toScala.map(stringMapFromKernel).orNull } From 85ad0c39aeb001ce4b3e208f1ec4a080dd29d41e Mon Sep 17 00:00:00 2001 From: Johan Lasperas Date: Wed, 14 Oct 2026 15:19:22 +0200 Subject: [PATCH 17/29] [Spark] Block CDF batch reads across incompatible partition changes (#7848) ## Description Update CDF schema change check to reject partition column changes, which can otherwise result in `null` values in older versions for columns upgraded to a partition column. This also adds logging for incompatible schema changes encountered during CDF reads. ## How was this patch tested? Added tests in `CDCReaderSuite` (`CDCSchemaChangeTests`) that cover: - A partition column change, with the check disabled (the usage event is logged and the read succeeds) and enabled (the read fails). - An intermediate partition change. - A partition change followed by `RESTORE`. - Additional test coverage for other schema changes: column dropped, renamed, or changes type after the start version. Co-authored-by: Isaac --- .../sql/delta/commands/cdc/CDCReader.scala | 40 +- .../delta/commands/cdc/CDCReaderBase.scala | 72 ++- .../sql/delta/sources/DeltaSQLConf.scala | 10 + .../spark/sql/delta/cdc/CDCReaderSuite.scala | 438 +++++++++++++++++- 4 files changed, 549 insertions(+), 11 deletions(-) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/commands/cdc/CDCReader.scala b/spark/src/main/scala/org/apache/spark/sql/delta/commands/cdc/CDCReader.scala index e67f9c072ba..fe199357aab 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/commands/cdc/CDCReader.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/commands/cdc/CDCReader.scala @@ -27,7 +27,7 @@ import org.apache.spark.sql.delta.actions._ import org.apache.spark.sql.delta.commands.DeletionVectorUtils import org.apache.spark.sql.delta.deletionvectors.{RoaringBitmapArray, RoaringBitmapArrayFormat} import org.apache.spark.sql.delta.files.{CdcAddFileIndex, TahoeChangeFileIndex, TahoeFileIndexWithSnapshotDescriptor, TahoeRemoveFileIndex} -import org.apache.spark.sql.delta.sources.DeltaDataSource +import org.apache.spark.sql.delta.sources.{DeltaDataSource, DeltaSQLConf} import org.apache.spark.sql.delta.storage.dv.DeletionVectorStore import org.apache.spark.sql.util.ScalaExtensions.OptionExt @@ -254,9 +254,10 @@ trait CDCReaderImpl extends CDCReaderBase { val addFiles = ListBuffer[CDCDataSpec[AddFile]]() val removeFiles = ListBuffer[CDCDataSpec[RemoveFile]]() - val startVersionMetadata = startVersionSnapshot.map(_.metadata).getOrElse { - deltaLog.getSnapshotAt(start, catalogTableOpt = catalogTableOpt).metadata - } + lazy val loadedStartVersionSnapshot = + deltaLog.getSnapshotAt(start, catalogTableOpt = catalogTableOpt) + val startVersionMetadata = startVersionSnapshot.map(_.metadata) + .getOrElse(loadedStartVersionSnapshot.metadata) if (!useCoarseGrainedCDC && !isCDCEnabledOnTable(startVersionMetadata, spark)) { throw DeltaErrors.changeDataNotRecordedException(start, start, end) } @@ -266,6 +267,22 @@ trait CDCReaderImpl extends CDCReaderBase { deltaLog, isStreaming ) + val blockReadsAcrossPartitionSchemaChanges = spark.sessionState.conf.getConf( + DeltaSQLConf.DELTA_CDF_BLOCK_READS_ACROSS_PARTITION_SCHEMA_CHANGES) + + // Track whether a partition schema change happens on a non-empty table. We could blindly + // reject all partition changes, but setting up an empty table and changing partitioning in a + // later commit is a valid pattern that we can support. + val partitionSchemaTracker = if ( + checkSchemaToBlockRead && blockReadsAcrossPartitionSchemaChanges) { + val startSnapshot = startVersionSnapshot.collect { case snapshot: Snapshot => snapshot } + .getOrElse(loadedStartVersionSnapshot) + Some(new CdfPartitionSchemaTracker( + startVersionMetadata.partitionColumns, + startSnapshotHasData = startSnapshot.numOfFiles > 0)) + } else { + None + } var totalBytes = 0L var numAddFiles, numRemoveFiles, numAddCDCFiles = 0L @@ -292,8 +309,9 @@ trait CDCReaderImpl extends CDCReaderBase { // 2. If it's indeed a schema change, as we won't be reading any data prior to it that // falls back to the previous (possibly incorrect) schema, we will be safe. Also if there // are any data file residing in the same commit, it will follow the new schema as well. - if (v > start) { - actions.collect { case a: Metadata => a }.foreach { metadata => + actions.collect { case a: Metadata => a }.foreach { metadata => + partitionSchemaTracker.ifDefined(_.processMetadata(metadata)) + if (v > start) { // Verify with start snapshot to check for any read-incompatible changes // This also detects the corner case in that there's only one schema change between // start and end, which looks exactly like the end schema. @@ -302,6 +320,7 @@ trait CDCReaderImpl extends CDCReaderBase { metadata, v, isSchemaChange = true) } } + partitionSchemaTracker.ifDefined(_.processActions(actions)) // Set up buffers for all action types to avoid multiple passes. val cdcActions = ListBuffer[AddCDCFile]() @@ -377,9 +396,16 @@ trait CDCReaderImpl extends CDCReaderBase { // the range, BUT time-travel is used so the read schema could also be arbitrary. // It is sufficient to just verify with the start version schema because we have already // verified that all data being queried is read-compatible with start schema. + val blockOnIncompatiblePartitionSchema = partitionSchemaTracker match { + case Some(tracker) => tracker.blockStartVersionPartitionSchemaCheck + case None => true + } checkBatchCdfReadSchemaIncompatibility( readSchemaSnapshot, start, end, checkSchemaToBlockRead, - startVersionMetadata, start, isSchemaChange = false) + startVersionMetadata, + start, + isSchemaChange = false, + blockOnIncompatiblePartitionSchema = blockOnIncompatiblePartitionSchema) val dfs = ListBuffer[DataFrame]() if (changeFiles.nonEmpty) { diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/commands/cdc/CDCReaderBase.scala b/spark/src/main/scala/org/apache/spark/sql/delta/commands/cdc/CDCReaderBase.scala index 686424ce65d..df29351116d 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/commands/cdc/CDCReaderBase.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/commands/cdc/CDCReaderBase.scala @@ -385,7 +385,9 @@ trait CDCReaderBase extends DeltaLogging { shouldCheckSchemaToBlockBatchRead: Boolean, metadata: Metadata, metadataVer: Long, - isSchemaChange: Boolean): Unit = { + isSchemaChange: Boolean, + blockOnIncompatibleSchema: Boolean = true, + blockOnIncompatiblePartitionSchema: Boolean = true): Unit = { // We do not check for any incompatibility if the global "I don't care" flag is turned on if (shouldCheckSchemaToBlockBatchRead) { // Column mapping incompatibilities @@ -428,19 +430,83 @@ trait CDCReaderBase extends DeltaLogging { } } - if (!compatible) { + val newPartitionColumns = readSchemaSnapshot.metadata.partitionColumns + val oldPartitionColumns = metadata.partitionColumns + val hasPartitionSchemaChange = !SchemaUtils.isPartitionCompatible( + newPartitionColumns = newPartitionColumns, + oldPartitionColumns = oldPartitionColumns) + val partitionSchemaChangeCheckEnabled = SQLConf.get.getConf( + DeltaSQLConf.DELTA_CDF_BLOCK_READS_ACROSS_PARTITION_SCHEMA_CHANGES) + val blockOnPartitionSchemaChange = + blockOnIncompatiblePartitionSchema && partitionSchemaChangeCheckEnabled + + if (!compatible || hasPartitionSchemaChange) { + recordDeltaEvent( + readSchemaSnapshot.deltaLog, + "delta.changeDataFeed.schemaIncompatibility", + data = Map( + "startVersion" -> start, + "endVersion" -> end, + "readVersion" -> readSchemaSnapshot.version, + "incompatibleVersion" -> metadataVer, + "isStartVersionCheck" -> !isSchemaChange, + "isSchemaCompatible" -> compatible, + "hasPartitionSchemaChange" -> hasPartitionSchemaChange, + "partitionSchemaCheckEnabled" -> partitionSchemaChangeCheckEnabled, + "partitionSchemaBlockEnabled" -> blockOnPartitionSchemaChange, + "schemaCheckEnabled" -> blockOnIncompatibleSchema)) + } + if ((!compatible && blockOnIncompatibleSchema) || + (hasPartitionSchemaChange && blockOnPartitionSchemaChange)) { throw DeltaErrors.blockBatchCdfReadWithIncompatibleSchemaChange( start, end, // The consistent read schema readSchemaSnapshot.metadata.schema, readSchemaSnapshot.version, // The conflicting schema or schema change version metadataVer, - isSchemaChange + isSchemaChange || hasPartitionSchemaChange ) } } } + /** Tracks whether a partition schema change happens on a non-empty table. */ + protected class CdfPartitionSchemaTracker( + startPartitionColumns: Seq[String], + startSnapshotHasData: Boolean) { + private var hasSeenData = startSnapshotHasData + private var hasSeenPartitionSchemaChange = false + private var hadDataBeforeFirstPartitionSchemaChange = startSnapshotHasData + + def processMetadata(metadata: Metadata): Unit = { + if (!hasSeenPartitionSchemaChange && !SchemaUtils.isPartitionCompatible( + newPartitionColumns = metadata.partitionColumns, + oldPartitionColumns = startPartitionColumns)) { + hasSeenPartitionSchemaChange = true + hadDataBeforeFirstPartitionSchemaChange = hasSeenData + } + } + + def processActions(actions: Seq[Action]): Unit = { + val commitInfo = actions.collectFirst { case ci: CommitInfo => ci } + val changesData = CommitInfo.fileActionChangesData(commitInfo) + hasSeenData ||= actions.exists { + case _: AddCDCFile => true + case add: AddFile => changesData(add) + case remove: RemoveFile => changesData(remove) + case _ => false + } + } + + def blockStartVersionPartitionSchemaCheck: Boolean = { + if (hasSeenPartitionSchemaChange) { + hadDataBeforeFirstPartitionSchemaChange + } else { + hasSeenData + } + } + } + def shouldCheckSchemaToBlockBatchRead( spark: SparkSession, deltaLog: DeltaLog, diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala b/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala index b70c5942779..21da1e0209e 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala @@ -2854,6 +2854,16 @@ trait DeltaSQLConfBase extends DeltaSQLConfUtils { .booleanConf .createWithDefault(false) + val DELTA_CDF_BLOCK_READS_ACROSS_PARTITION_SCHEMA_CHANGES = + buildConf("changeDataFeed.blockReadsAcrossPartitionSchemaChanges.enabled") + .internal() + .doc( + "When enabled, batch CDF reads fail if their read range crosses a partition schema " + + "change. The read must be split at each partition schema change boundary. This also " + + "enables the read-time CDF start version schema compatibility check.") + .booleanConf + .createWithDefault(true) + val DELTA_CDF_DEFAULT_SCHEMA_MODE_FOR_COLUMN_MAPPING_TABLE = buildConf("changeDataFeed.defaultSchemaModeForColumnMappingTable") .doc( diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/cdc/CDCReaderSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/cdc/CDCReaderSuite.scala index bde3bbb09d7..1b87b3a1953 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/cdc/CDCReaderSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/cdc/CDCReaderSuite.scala @@ -46,7 +46,8 @@ class CDCReaderSuite with CheckCDCAnswer with SharedSparkSession with DeltaSQLCommandTest - with DeltaColumnMappingTestUtils { + with DeltaColumnMappingTestUtils + with CDCSchemaChangeTests { override protected def sparkConf: SparkConf = super.sparkConf .set(DeltaConfigs.CHANGE_DATA_FEED.defaultTablePropertyKey, "true") @@ -586,3 +587,438 @@ class CDCReaderSuite } } +trait CDCSchemaChangeTests { + self: QueryTest with DeltaSQLCommandTest => + + protected def columnMappingMode: String + + test("CDF fails when a column is dropped after the start version") { + val tableName = "cdf_drop_column_after_start" + withTable(tableName) { + sql( + s""" + |CREATE TABLE $tableName (a INT, b INT) USING DELTA + |TBLPROPERTIES ('delta.columnMapping.mode' = 'name') + |""".stripMargin) + sql(s"INSERT INTO $tableName VALUES (1, 10)") + sql(s"ALTER TABLE $tableName DROP COLUMN b") + + withSQLConf( + DeltaSQLConf.DELTA_CDF_BLOCK_READS_ACROSS_PARTITION_SCHEMA_CHANGES.key -> "true") { + checkError( + exception = intercept[DeltaUnsupportedOperationException] { + sql(s"SELECT * FROM table_changes('$tableName', 1, 2)").collect() + }, + condition = "DELTA_CHANGE_DATA_FEED_INCOMPATIBLE_DATA_SCHEMA", + sqlState = Some("0AKDC"), + matchPVals = true, + parameters = Map( + "start" -> "1", + "end" -> "2", + "readSchema" -> ".*", + "readVersion" -> "2", + "incompatibleVersion" -> "1", + "config" -> ".*defaultSchemaModeForColumnMappingTable")) + } + } + } + + test("CDF fails when a column is renamed after the start version") { + val tableName = "cdf_rename_column_after_start" + withTable(tableName) { + sql( + s""" + |CREATE TABLE $tableName (a INT, b INT) USING DELTA + |TBLPROPERTIES ('delta.columnMapping.mode' = 'name') + |""".stripMargin) + sql(s"INSERT INTO $tableName VALUES (1, 10)") + sql(s"ALTER TABLE $tableName RENAME COLUMN b TO c") + + withSQLConf( + DeltaSQLConf.DELTA_CDF_BLOCK_READS_ACROSS_PARTITION_SCHEMA_CHANGES.key -> "true") { + checkError( + exception = intercept[DeltaUnsupportedOperationException] { + sql(s"SELECT * FROM table_changes('$tableName', 1, 2)").collect() + }, + condition = "DELTA_CHANGE_DATA_FEED_INCOMPATIBLE_DATA_SCHEMA", + sqlState = Some("0AKDC"), + matchPVals = true, + parameters = Map( + "start" -> "1", + "end" -> "2", + "readSchema" -> ".*", + "readVersion" -> "2", + "incompatibleVersion" -> "1", + "config" -> ".*defaultSchemaModeForColumnMappingTable")) + } + } + } + + test("CDF fails when a column type changes after the start version") { + val tableName = "cdf_change_column_type_after_start" + withTable(tableName) { + sql( + s""" + |CREATE TABLE $tableName (a INT) USING DELTA + |TBLPROPERTIES ('delta.enableTypeWidening' = 'true') + |""".stripMargin) + sql(s"INSERT INTO $tableName VALUES (1)") + sql(s"ALTER TABLE $tableName ALTER COLUMN a TYPE DOUBLE") + + withSQLConf( + DeltaSQLConf.DELTA_CDF_BLOCK_READS_ACROSS_PARTITION_SCHEMA_CHANGES.key -> "true") { + checkError( + exception = intercept[DeltaUnsupportedOperationException] { + sql(s"SELECT * FROM table_changes('$tableName', 1, 2)").collect() + }, + condition = "DELTA_CHANGE_DATA_FEED_INCOMPATIBLE_DATA_SCHEMA", + sqlState = Some("0AKDC"), + matchPVals = true, + parameters = Map( + "start" -> "1", + "end" -> "2", + "readSchema" -> ".*", + "readVersion" -> "2", + "incompatibleVersion" -> "1", + "config" -> ".*defaultSchemaModeForColumnMappingTable")) + } + } + } + + // The tests below only really make sense when column mapping is disabled, otherwise every + // REPLACE TABLE creates a brand-new set of columns and reads are always rejected. + if (columnMappingMode == "none") { + test("partition schema change check can be rolled out") { + val tableName = "cdf_partition_schema_evolution" + withTable(tableName) { + sql(s""" + |CREATE TABLE $tableName (a INT, b INT) USING DELTA PARTITIONED BY (a) + |""".stripMargin) + sql(s"INSERT INTO $tableName VALUES (1, 10)") + sql( + s""" + |CREATE OR REPLACE TABLE $tableName (a INT, b INT) USING DELTA + |PARTITIONED BY (b) + |""".stripMargin) + + val events = withSQLConf( + DeltaSQLConf.DELTA_CDF_BLOCK_READS_ACROSS_PARTITION_SCHEMA_CHANGES.key -> "false") { + Log4jUsageLogger.track { + sql(s"SELECT * FROM table_changes('$tableName', 1, 2)").collect() + } + } + val event = events.find(event => event.metric == "tahoeEvent" && + event.tags.get("opType").contains("delta.changeDataFeed.schemaIncompatibility")) + assert(event.isDefined) + val eventData = JsonUtils.fromJson[Map[String, Any]](event.get.blob) + assert(eventData("isSchemaCompatible") === true) + assert(eventData("hasPartitionSchemaChange") === true) + assert(eventData("partitionSchemaCheckEnabled") === false) + + withSQLConf( + DeltaSQLConf.DELTA_CDF_BLOCK_READS_ACROSS_PARTITION_SCHEMA_CHANGES.key -> "true") { + checkError( + exception = intercept[DeltaUnsupportedOperationException] { + sql(s"SELECT * FROM table_changes('$tableName', 1, 2)").collect() + }, + condition = "DELTA_CHANGE_DATA_FEED_INCOMPATIBLE_SCHEMA_CHANGE", + sqlState = Some("0AKDC"), + matchPVals = true, + parameters = Map( + "start" -> "1", + "end" -> "2", + "readSchema" -> ".*", + "readVersion" -> "2", + "incompatibleVersion" -> "1")) + } + } + } + + test("CDF allows column reordering when the partition schema is unchanged") { + val tableName = "cdf_reordered_columns_same_partition_schema" + withTable(tableName) { + sql(s""" + |CREATE TABLE $tableName (a INT, b INT) USING DELTA PARTITIONED BY (a) + |""".stripMargin) + sql( + s""" + |CREATE OR REPLACE TABLE $tableName (b INT, a INT) USING DELTA + |PARTITIONED BY (a) + |""".stripMargin) + + withSQLConf( + DeltaSQLConf.DELTA_CDF_BLOCK_READS_ACROSS_PARTITION_SCHEMA_CHANGES.key -> "true") { + checkAnswer( + sql(s"SELECT a, b FROM table_changes('$tableName', 0, 1)"), + Nil) + } + } + } + + test("CDF allows initial partition setup after an empty version 0") { + val tableName = "cdf_initial_partition_setup" + withTable(tableName) { + sql(s""" + |CREATE TABLE $tableName (a INT, b INT) USING DELTA + |""".stripMargin) + sql( + s""" + |CREATE OR REPLACE TABLE $tableName (a INT, b INT) USING DELTA + |PARTITIONED BY (b) + |""".stripMargin) + sql(s"INSERT INTO $tableName VALUES (1, 10)") + + withSQLConf( + DeltaSQLConf.DELTA_CDF_BLOCK_READS_ACROSS_PARTITION_SCHEMA_CHANGES.key -> "true") { + val events = Log4jUsageLogger.track { + checkAnswer( + sql(s"SELECT a, b FROM table_changes('$tableName', 0, 2)"), + Row(1, 10)) + } + val event = events.find(event => event.metric == "tahoeEvent" && + event.tags.get("opType").contains("delta.changeDataFeed.schemaIncompatibility")) + assert(event.isDefined) + val eventData = JsonUtils.fromJson[Map[String, Any]](event.get.blob) + assert(eventData("isSchemaCompatible") === true) + assert(eventData("hasPartitionSchemaChange") === true) + assert(eventData("partitionSchemaBlockEnabled") === false) + } + } + } + + test("CDF still rejects a partition change after data in an initially empty table") { + val tableName = "cdf_partition_change_after_initial_data" + withTable(tableName) { + sql(s""" + |CREATE TABLE $tableName (a INT, b INT) USING DELTA + |""".stripMargin) + sql(s"INSERT INTO $tableName VALUES (1, 10)") + sql( + s""" + |CREATE OR REPLACE TABLE $tableName (a INT, b INT) USING DELTA + |PARTITIONED BY (b) + |""".stripMargin) + sql(s"INSERT INTO $tableName VALUES (2, 20)") + + withSQLConf( + DeltaSQLConf.DELTA_CDF_BLOCK_READS_ACROSS_PARTITION_SCHEMA_CHANGES.key -> "true") { + checkError( + exception = intercept[DeltaUnsupportedOperationException] { + sql(s"SELECT * FROM table_changes('$tableName', 0, 3)").collect() + }, + condition = "DELTA_CHANGE_DATA_FEED_INCOMPATIBLE_SCHEMA_CHANGE", + sqlState = Some("0AKDC"), + matchPVals = true, + parameters = Map( + "start" -> "0", + "end" -> "3", + "readSchema" -> ".*", + "readVersion" -> "3", + "incompatibleVersion" -> "0")) + } + } + } + + test("CDF fails for an intermediate partition schema when endpoints agree") { + val tableName = "cdf_multiple_partition_changes" + withTable(tableName) { + sql(s""" + |CREATE TABLE $tableName (a INT, b INT) USING DELTA PARTITIONED BY (a) + |""".stripMargin) + sql(s"INSERT INTO $tableName VALUES (1, 10)") + sql( + s""" + |CREATE OR REPLACE TABLE $tableName (a INT, b INT) USING DELTA + |PARTITIONED BY (b) + |""".stripMargin) + sql( + s""" + |CREATE OR REPLACE TABLE $tableName (a INT, b INT) USING DELTA + |PARTITIONED BY (a) + |""".stripMargin) + + withSQLConf( + DeltaSQLConf.DELTA_CDF_BLOCK_READS_ACROSS_PARTITION_SCHEMA_CHANGES.key -> "true") { + checkError( + exception = intercept[DeltaUnsupportedOperationException] { + sql(s"SELECT * FROM table_changes('$tableName', 1, 3)").collect() + }, + condition = "DELTA_CHANGE_DATA_FEED_INCOMPATIBLE_SCHEMA_CHANGE", + sqlState = Some("0AKDC"), + matchPVals = true, + parameters = Map( + "start" -> "1", + "end" -> "3", + "readSchema" -> ".*", + "readVersion" -> "3", + "incompatibleVersion" -> "2")) + } + } + } + + test("CDF fails for multiple partition changes on an empty table") { + val tableName = "cdf_multiple_empty_partition_changes" + withTable(tableName) { + sql(s""" + |CREATE TABLE $tableName (a INT, b INT) USING DELTA PARTITIONED BY (a) + |""".stripMargin) + sql( + s""" + |CREATE OR REPLACE TABLE $tableName (a INT, b INT) USING DELTA + |PARTITIONED BY (b) + |""".stripMargin) + sql( + s""" + |CREATE OR REPLACE TABLE $tableName (a INT, b INT) USING DELTA + |PARTITIONED BY (a) + |""".stripMargin) + + withSQLConf( + DeltaSQLConf.DELTA_CDF_BLOCK_READS_ACROSS_PARTITION_SCHEMA_CHANGES.key -> "true") { + checkError( + exception = intercept[DeltaUnsupportedOperationException] { + sql(s"SELECT * FROM table_changes('$tableName', 0, 2)").collect() + }, + condition = "DELTA_CHANGE_DATA_FEED_INCOMPATIBLE_SCHEMA_CHANGE", + sqlState = Some("0AKDC"), + matchPVals = true, + parameters = Map( + "start" -> "0", + "end" -> "2", + "readSchema" -> ".*", + "readVersion" -> "2", + "incompatibleVersion" -> "1")) + } + } + } + } + + test("CDF fails for an intermediate dropped column when endpoints agree") { + val tableName = "cdf_drop_add_column" + withTable(tableName) { + sql( + s""" + |CREATE TABLE $tableName (a INT, b INT) USING DELTA + |TBLPROPERTIES ('delta.columnMapping.mode' = 'name') + |""".stripMargin) + sql(s"INSERT INTO $tableName VALUES (1, 10)") + sql(s"ALTER TABLE $tableName DROP COLUMN b") + sql(s"ALTER TABLE $tableName ADD COLUMN b INT") + + withSQLConf( + DeltaSQLConf.DELTA_CDF_BLOCK_READS_ACROSS_PARTITION_SCHEMA_CHANGES.key -> "true") { + checkError( + exception = intercept[DeltaUnsupportedOperationException] { + sql(s"SELECT * FROM table_changes('$tableName', 1, 3)").collect() + }, + condition = "DELTA_CHANGE_DATA_FEED_INCOMPATIBLE_DATA_SCHEMA", + sqlState = Some("0AKDC"), + matchPVals = true, + parameters = Map( + "start" -> "1", + "end" -> "3", + "readSchema" -> ".*", + "readVersion" -> "3", + "incompatibleVersion" -> "1", + "config" -> ".*defaultSchemaModeForColumnMappingTable")) + } + } + } + + test("CDF fails for an intermediate renamed column when endpoints agree") { + val tableName = "cdf_rename_column_back" + withTable(tableName) { + sql( + s""" + |CREATE TABLE $tableName (a INT, b INT) USING DELTA + |TBLPROPERTIES ('delta.columnMapping.mode' = 'name') + |""".stripMargin) + sql(s"INSERT INTO $tableName VALUES (1, 10)") + sql(s"ALTER TABLE $tableName RENAME COLUMN b TO c") + sql(s"ALTER TABLE $tableName RENAME COLUMN c TO b") + + checkError( + exception = intercept[DeltaUnsupportedOperationException] { + sql(s"SELECT * FROM table_changes('$tableName', 1, 3)").collect() + }, + condition = "DELTA_CHANGE_DATA_FEED_INCOMPATIBLE_SCHEMA_CHANGE", + sqlState = Some("0AKDC"), + matchPVals = true, + parameters = Map( + "start" -> "1", + "end" -> "3", + "readSchema" -> ".*", + "readVersion" -> "3", + "incompatibleVersion" -> "2")) + } + } + + if (columnMappingMode == "none") { + test("CDF fails for a partition schema change followed by RESTORE") { + val tableName = "cdf_partition_change_restore" + withTable(tableName) { + sql(s""" + |CREATE TABLE $tableName (a INT, b INT) USING DELTA PARTITIONED BY (a) + |""".stripMargin) + sql(s"INSERT INTO $tableName VALUES (1, 10)") + sql( + s""" + |CREATE OR REPLACE TABLE $tableName (a INT, b INT) USING DELTA + |PARTITIONED BY (b) + |""".stripMargin) + sql(s"RESTORE TABLE $tableName TO VERSION AS OF 1") + + withSQLConf( + DeltaSQLConf.DELTA_CDF_BLOCK_READS_ACROSS_PARTITION_SCHEMA_CHANGES.key -> "true") { + checkError( + exception = intercept[DeltaUnsupportedOperationException] { + sql(s"SELECT * FROM table_changes('$tableName', 1, 3)").collect() + }, + condition = "DELTA_CHANGE_DATA_FEED_INCOMPATIBLE_SCHEMA_CHANGE", + sqlState = Some("0AKDC"), + matchPVals = true, + parameters = Map( + "start" -> "1", + "end" -> "3", + "readSchema" -> ".*", + "readVersion" -> "3", + "incompatibleVersion" -> "2")) + } + } + } + } + + if (columnMappingMode != "none") { + test(s"CDF fails for a partition schema change with " + + s"$columnMappingMode column mapping") { + val tableName = "cdf_partition_change_with_column_mapping" + withTable(tableName) { + sql(s"CREATE TABLE $tableName (a INT, b INT) USING DELTA PARTITIONED BY (a)") + sql(s"INSERT INTO $tableName VALUES (1, 10)") + sql( + s""" + |CREATE OR REPLACE TABLE $tableName (a INT, b INT) USING DELTA + |PARTITIONED BY (b) + |""".stripMargin) + + withSQLConf( + DeltaSQLConf.DELTA_CDF_BLOCK_READS_ACROSS_PARTITION_SCHEMA_CHANGES.key -> "true") { + checkError( + exception = intercept[DeltaUnsupportedOperationException] { + sql(s"SELECT * FROM table_changes('$tableName', 1, 2)").collect() + }, + condition = "DELTA_CHANGE_DATA_FEED_INCOMPATIBLE_SCHEMA_CHANGE", + sqlState = Some("0AKDC"), + matchPVals = true, + parameters = Map( + "start" -> "1", + "end" -> "2", + "readSchema" -> ".*", + "readVersion" -> "2", + "incompatibleVersion" -> "1")) + } + } + } + } +} + From 05d41c9897d142b41cb9063f2905550ac5c432b1 Mon Sep 17 00:00:00 2001 From: Rakesh Veeramacheneni <173086727+raveeram-db@users.noreply.github.com> Date: Wed, 14 Oct 2026 07:32:05 -0700 Subject: [PATCH 18/29] [Spark] [DSv2 Connector] Convert TableNotFoundException to Scala (#7860) #### Which Delta project/connector is this regarding? - [x] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [ ] Other ## Description Converts `TableNotFoundException` to Scala as part of the V2 connector migration. Preserves its constructor, getter, error message, subclassability, and field initialization order. The path field initializes after the superclass constructor so a getter called from an overridden `fillInStackTrace()` sees the same default value as before. ## How was this patch tested? No new test cases. Validation in this repository is pending; existing tests will exercise the converted class through its existing callers. ## Does this PR introduce any user-facing changes? No. Signed-off-by: Rakesh Veeramacheneni --- .../v2/exception/TableNotFoundException.scala} | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) rename spark/v2/src/main/{java/io/delta/spark/internal/v2/exception/TableNotFoundException.java => scala/io/delta/spark/internal/v2/exception/TableNotFoundException.scala} (65%) diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/exception/TableNotFoundException.java b/spark/v2/src/main/scala/io/delta/spark/internal/v2/exception/TableNotFoundException.scala similarity index 65% rename from spark/v2/src/main/java/io/delta/spark/internal/v2/exception/TableNotFoundException.java rename to spark/v2/src/main/scala/io/delta/spark/internal/v2/exception/TableNotFoundException.scala index 471fd4bbc55..3ceb3108c55 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/exception/TableNotFoundException.java +++ b/spark/v2/src/main/scala/io/delta/spark/internal/v2/exception/TableNotFoundException.scala @@ -13,19 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.delta.spark.internal.v2.exception; -/** Exception thrown when no Delta table exists at the requested path. */ -public class TableNotFoundException extends RuntimeException { +package io.delta.spark.internal.v2.exception - private final String tablePath; +/** Exception thrown when no Delta table exists at the requested path. */ +class TableNotFoundException(path: String) + extends RuntimeException(String.format("Delta table does not exist at path: %s.", path)) { - public TableNotFoundException(String tablePath) { - super(String.format("Delta table does not exist at path: %s.", tablePath)); - this.tablePath = tablePath; - } + // Preserve Java initialization order: getTablePath returns null during super construction. + private[this] val tablePath: String = path - public String getTablePath() { - return tablePath; - } + def getTablePath: String = tablePath } From 4e0d160671f7ff441ebfda094c7732259015e63e Mon Sep 17 00:00:00 2001 From: Robert <93660664+Rob2U@users.noreply.github.com> Date: Wed, 14 Oct 2026 17:40:09 +0200 Subject: [PATCH 19/29] [Spark] Foundation for Concurrent Identity Columns: metadata key, error classes (#7618) #### Which Delta project/connector is this regarding? - [x] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [ ] Other (fill in here) ## Description This PR is the first out of a stack of seven PRs in total. The combined stack adds support for the Concurrent Identity Column (CIC) feature following the RFC [here](https://github.com/delta-io/delta/pull/7272). The TLDR for this RFC/feature: It allows for two concurrent 'inserters' to both succeed by adding an identity sequence service that hands out disjoint ranges of id values. This way the two writers do not conflict and can both commit. **_Stack overview:_** 1. Foundation <- this PR (#7618) 2. Service abstraction + In-Memory one (#7619) 3. Transport and buffering for the service requests (#7620) 4. Table feature and the insert path (#7621) 5. DDL integration (#7622) 6. Merge support (#7623) 7. UC Client addition (#7785) This PR introduces no behaviour changes, just scaffolding. It adds: - `ConcurrentIdentityColumnSchema`: the `delta.identity.concurrent.sequenceId` column-metadata key that marks a column as a CIC. - `ConcurrentIdentityColumnErrors` + 8 DELTA_CONCURRENT_IDENTITY_COLUMN_* error classes (disabled, too-many-columns, sequence-not-found, metadata-mismatch, empty-reserve-range, service-timeout, conversion-incomplete, using-wrong-generator). - Config `spark.databricks.delta.identityColumn.concurrent.maxColumnsPerTable` ## How was this patch tested? The ConcurrentIdentityColumnErrorsSuite adds three tests for the `EMPTY_RESERVE_RANGE`, `METADATA_MISMATCH`, and `USING_WRONG_GENERATOR` errors. The tests are only added for these errors because these errors can not occur 'naturally' in the code and are therefore tested this way. ## Does this PR introduce _any_ user-facing changes? No --------- Signed-off-by: Robert Weeke Co-authored-by: mkroll-db --- .../resources/error/delta-error-classes.json | 48 +++++ .../ConcurrentIdentityColumnErrors.scala | 189 ++++++++++++++++++ .../ConcurrentIdentityColumnSchema.scala | 139 +++++++++++++ .../sql/delta/sources/DeltaSQLConf.scala | 11 + .../ConcurrentIdentityColumnErrorsSuite.scala | 62 ++++++ 5 files changed, 449 insertions(+) create mode 100644 spark/src/main/scala/org/apache/spark/sql/delta/ConcurrentIdentityColumnErrors.scala create mode 100644 spark/src/main/scala/org/apache/spark/sql/delta/ConcurrentIdentityColumnSchema.scala create mode 100644 spark/src/test/scala/org/apache/spark/sql/delta/ConcurrentIdentityColumnErrorsSuite.scala diff --git a/spark/src/main/resources/error/delta-error-classes.json b/spark/src/main/resources/error/delta-error-classes.json index 22302558785..b511705704c 100644 --- a/spark/src/main/resources/error/delta-error-classes.json +++ b/spark/src/main/resources/error/delta-error-classes.json @@ -819,6 +819,54 @@ }, "sqlState" : "2D521" }, + "DELTA_CONCURRENT_IDENTITY_COLUMN_CONVERSION_INCOMPLETE" : { + "message" : [ + "Identity column on table is missing its sequence service pointer. Re-run the feature opt-in conversion to repair it." + ], + "sqlState" : "XXKDS" + }, + "DELTA_CONCURRENT_IDENTITY_COLUMN_DISABLED" : { + "message" : [ + "Cannot table : concurrent identity columns are disabled." + ], + "sqlState" : "0A000" + }, + "DELTA_CONCURRENT_IDENTITY_COLUMN_EMPTY_RESERVE_RANGE" : { + "message" : [ + "Concurrent identity column reservation returned an empty range for sequence on table ." + ], + "sqlState" : "XXKDS" + }, + "DELTA_CONCURRENT_IDENTITY_COLUMN_METADATA_MISMATCH" : { + "message" : [ + "Identity column mismatch: service value (sequence ) vs schema value ." + ], + "sqlState" : "XXKDS" + }, + "DELTA_CONCURRENT_IDENTITY_COLUMN_SEQUENCE_NOT_FOUND" : { + "message" : [ + "Identity column on table references sequence , which no longer exists. Run SYNC IDENTITY to repair the column." + ], + "sqlState" : "XXKDS" + }, + "DELTA_CONCURRENT_IDENTITY_COLUMN_SERVICE_TIMEOUT" : { + "message" : [ + "The identity sequence operation for table timed out after ms." + ], + "sqlState" : "XXKDS" + }, + "DELTA_CONCURRENT_IDENTITY_COLUMN_TOO_MANY_COLUMNS" : { + "message" : [ + "Table would have concurrent identity columns, which exceeds the maximum of . Reduce the number of identity columns." + ], + "sqlState" : "54011" + }, + "DELTA_CONCURRENT_IDENTITY_COLUMN_USING_WRONG_GENERATOR" : { + "message" : [ + "Write failed due to an identity column version mismatch for column(s) on table ." + ], + "sqlState" : "XXKDS" + }, "DELTA_CONCURRENT_TRANSACTION" : { "message" : [ "ConcurrentTransactionException: This error occurs when multiple streaming queries are using the same checkpoint to write into this table. Did you run multiple instances of the same streaming query at the same time?\nRefer to for more details." diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/ConcurrentIdentityColumnErrors.scala b/spark/src/main/scala/org/apache/spark/sql/delta/ConcurrentIdentityColumnErrors.scala new file mode 100644 index 00000000000..4063a1a4a57 --- /dev/null +++ b/spark/src/main/scala/org/apache/spark/sql/delta/ConcurrentIdentityColumnErrors.scala @@ -0,0 +1,189 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta + +import org.apache.spark.sql.catalyst.TableIdentifier + +/** + * Thrown by the Concurrent Identity Column (CIC) service-backed reservation guards when a + * write would emit identity values outside a reserved range. It is a [[DeltaThrowable]] carrying + * one of the `DELTA_CONCURRENT_IDENTITY_COLUMN_*` error classes, so driver call sites and tests + * can match on the type or the error class instead of message prose. + * + * The executor-side generator guard throws this across the Spark task boundary, so it reaches the + * driver wrapped in a `SparkException`; tests exercising that path match the error-class token on + * the cause chain rather than the type. + */ +class ConcurrentIdentityColumnReservationException( + errorClass: String, + messageParameters: Array[String] = Array.empty) + extends IllegalStateException( + DeltaThrowableHelper.getMessage(errorClass, messageParameters)) + with DeltaThrowable { + override def getErrorClass: String = errorClass + + override def getMessageParameters: java.util.Map[String, String] = + DeltaThrowableHelper.getMessageParameters(errorClass, errorSubClass = null, messageParameters) +} + +/** + * Builders for the CIC reservation-guard errors. The message text lives in the Delta + * error-classes JSON under the `DELTA_CONCURRENT_IDENTITY_COLUMN_*` classes; each + * builder only names its error class and supplies the parameters, in the order the placeholders + * appear in the template. + */ +object ConcurrentIdentityColumnErrors { + // Builders are ordered to match the alphabetical class order in the Delta error-classes JSON. + + /** The column is missing its sequence service pointer; re-run the feature opt-in conversion. */ + def conversionIncomplete( + columnName: String, + tableId: String): ConcurrentIdentityColumnReservationException = + new ConcurrentIdentityColumnReservationException( + "DELTA_CONCURRENT_IDENTITY_COLUMN_CONVERSION_INCOMPLETE", + Array(columnName, tableId)) + + /** + * The CIC feature is disabled, blocking the attempted operation. Covers identity-generating + * writes to service-backed tables, CREATE TABLE with CIC, the feature opt-in conversion and + * SYNC IDENTITY repair. + */ + def concurrentIdentityColumnsDisabled( + operation: String, + tableId: String): ConcurrentIdentityColumnReservationException = + new ConcurrentIdentityColumnReservationException( + "DELTA_CONCURRENT_IDENTITY_COLUMN_DISABLED", + Array(operation, tableId)) + + /** + * A CIC pull granted an empty range. The driver should never return zero values; this guards + * the executor generator against a no-progress retry spin. + */ + def emptyReserveRange( + sequenceId: String, + tableId: String): ConcurrentIdentityColumnReservationException = + new ConcurrentIdentityColumnReservationException( + "DELTA_CONCURRENT_IDENTITY_COLUMN_EMPTY_RESERVE_RANGE", + Array(sequenceId, tableId)) + + /** + * A column carries both a service-sequence pointer and a stock `delta.identity.highWaterMark`. + * The two name rival generators, so a writer must reject this state rather than fall back to + * high-water-mark generation for the column; SYNC IDENTITY repairs it. + */ + def highWaterMarkConflict( + columnName: String, + tableId: String): ConcurrentIdentityColumnReservationException = + new ConcurrentIdentityColumnReservationException( + "DELTA_CONCURRENT_IDENTITY_COLUMN_HIGH_WATER_MARK_CONFLICT", + Array(columnName, tableId)) + + /** + * Service-returned metadata disagrees with the schema-declared metadata. Generalized from the + * old STEP_MISMATCH so the same class covers start/other metadata drift later: the `property` + * param names which one drifted (value "step" today). + */ + def metadataMismatch( + grantedStep: Long, + sequenceId: String, + columnStep: Long): ConcurrentIdentityColumnReservationException = + new ConcurrentIdentityColumnReservationException( + "DELTA_CONCURRENT_IDENTITY_COLUMN_METADATA_MISMATCH", + Array("step", grantedStep.toString, sequenceId, columnStep.toString)) + + def requiresCatalogManaged(tableId: String): Throwable = + DeltaErrors.operationNotSupportedException( + "Enabling concurrent identity columns on a table that is not catalog-managed", + TableIdentifier(tableId)) + + /** + * A CREATE OR REPLACE TABLE would drop [[ConcurrentIdentityColumnsTableFeature]] from a table + * that supports it. REPLACE can only reach this point under + * `REPLACE_TABLE_PROTOCOL_DOWNGRADE_ALLOWED`. CIC removal is not supported that way because it + * bypasses the sequence-retiring pre-downgrade path. + */ + def replaceCannotDropFeature(tableId: String): Throwable = + DeltaErrors.operationNotSupportedException( + "Removing concurrent identity columns with CREATE OR REPLACE TABLE", + TableIdentifier(tableId)) + + /** + * A column still carries a service-sequence pointer but the Concurrent Identity Columns feature + * is no longer supported. The orphaned "needs repair" state created by `ALTER TABLE ... UNSET + * MANAGED` combined with a `RESTORE` to a CIC-era version. + */ + def orphanedSequencePointerRequiresSync(tableId: String): Throwable = + DeltaErrors.operationNotSupportedException( + "Writing to an identity column with an orphaned concurrent identity columns sequence " + + "pointer (the feature is no longer supported); run " + + "ALTER TABLE ... ALTER COLUMN ... SYNC IDENTITY to repair the column first", + TableIdentifier(tableId)) + + /** + * A schema-stamped sequenceId is gone from the service. Not recoverable on the write path: + * reseeding a fresh counter could hand out values colliding with already-written rows, so + * the table needs deliberate repair. + */ + def sequenceNotFound( + sequenceId: String, + columnName: String, + tableId: String): ConcurrentIdentityColumnReservationException = + new ConcurrentIdentityColumnReservationException( + "DELTA_CONCURRENT_IDENTITY_COLUMN_SEQUENCE_NOT_FOUND", + Array(columnName, tableId, sequenceId)) + + /** + * A single call to the UC identity-sequence service exceeded the bounded wait + * (`spark.databricks.delta.identityColumn.concurrent.uc.timeoutMs`) before it returned. The + * service is likely temporarily unavailable (e.g. disabled), which the managed-catalog client + * would otherwise retry for minutes; we fail fast instead so the caller is not blocked. + */ + def serviceTimeout( + tableId: String, + timeoutMs: Long): ConcurrentIdentityColumnReservationException = + new ConcurrentIdentityColumnReservationException( + "DELTA_CONCURRENT_IDENTITY_COLUMN_SERVICE_TIMEOUT", + Array(tableId, timeoutMs.toString)) + + /** + * A CIC table would carry more service-backed identity columns than the configured maximum + * (`spark.databricks.delta.identityColumn.concurrent.maxColumnsPerTable`). Thrown by the + * CREATE/REPLACE stamping hook and the feature opt-in conversion before any sequence is minted. + */ + def tooManyColumns( + tableId: String, + numColumns: Int, + maxColumns: Int): ConcurrentIdentityColumnReservationException = + new ConcurrentIdentityColumnReservationException( + "DELTA_CONCURRENT_IDENTITY_COLUMN_TOO_MANY_COLUMNS", + Array(tableId, numColumns.toString, maxColumns.toString)) + + /** + * Defends against an unexpected write path that reaches the legacy high-water-mark + * generator on a service-backed (sequence-stamped) table. Every wired write path + * (INSERT, all three MERGE variants) reserves from the service before writing, so this + * is a backstop: if a future or otherwise-unhandled path skipped the reservation, its + * legacy generator would silently emit duplicate identity values, so we abort before + * any data is written rather than corrupt the table. + */ + def usingWrongGenerator( + columnNames: Seq[String], + tableId: String): ConcurrentIdentityColumnReservationException = + new ConcurrentIdentityColumnReservationException( + "DELTA_CONCURRENT_IDENTITY_COLUMN_USING_WRONG_GENERATOR", + Array(columnNames.mkString(", "), tableId)) +} diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/ConcurrentIdentityColumnSchema.scala b/spark/src/main/scala/org/apache/spark/sql/delta/ConcurrentIdentityColumnSchema.scala new file mode 100644 index 00000000000..73b492bd939 --- /dev/null +++ b/spark/src/main/scala/org/apache/spark/sql/delta/ConcurrentIdentityColumnSchema.scala @@ -0,0 +1,139 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta + +import org.apache.spark.sql.delta.actions.{Metadata, Protocol} +import org.apache.spark.sql.delta.sources.{DeltaSourceUtils, DeltaSQLConf} +import io.delta.storage.commit.uccommitcoordinator.UCCommitCoordinatorClient + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.types.{MetadataBuilder, StructField, StructType} + +/** + * Schema-metadata for Concurrent Identity Columns (CIC), persisted on identity column + * StructFields. + * + * A CIC column carries a single extra key, the sequence ID + * `delta.identity.concurrent.sequenceId`, alongside the standard + * `delta.identity.{start, step, allowExplicitInsert}` keys. Those standard keys remain the + * source of truth for start/step (read via [[IdentityColumn.getIdentityInfo]]); we do not + * duplicate them here. The key name is the cross-system contract with Unity Catalog: UC's + * registration hook extracts this exact key from the schema it processes, so it must match + * the catalog side verbatim. + * + * Presence of [[SEQUENCE_ID]] is the canonical signal that a column is backed by a + * service-allocated sequence. The reservation path reads it from here. + */ +object ConcurrentIdentityColumnSchema { + val SEQUENCE_ID = "delta.identity.concurrent.sequenceId" + + /** + * The table id under which this table's sequences live in the sequence service. For a + * catalog-owned (UC coordinated-commits) table this is the UC table id, stamped into the + * table configuration at creation ([[UCCommitCoordinatorClient.UC_TABLE_ID_KEY]]). + * The `metadata.id` fallback below is test-only, as CIC requires a table to be managed. + */ + def sequenceServiceTableId(metadata: Metadata): String = + metadata.configuration.getOrElse(UCCommitCoordinatorClient.UC_TABLE_ID_KEY, metadata.id) + + /** + * Rejects a CIC table with more identity columns than + * [[DeltaSQLConf.CONCURRENT_IDENTITY_COLUMN_MAX_COLUMNS_PER_TABLE]]. Called by both producers of + * service-backed columns (the CREATE/REPLACE stamping hook and the feature opt-in conversion) + * before any sequence is minted, so the bound holds regardless of which path adds the columns. + */ + def checkColumnLimit( + sparkSession: SparkSession, + metadata: Metadata, + identityColumnCount: Int): Unit = { + val maxCicColumnsPerTable = + sparkSession.conf.get(DeltaSQLConf.CONCURRENT_IDENTITY_COLUMN_MAX_COLUMNS_PER_TABLE) + if (identityColumnCount > maxCicColumnsPerTable) { + throw ConcurrentIdentityColumnErrors.tooManyColumns( + metadata.id, identityColumnCount, maxCicColumnsPerTable) + } + } + + /** True iff this column carries the sequence ID. */ + def hasConcurrentSequenceMetadata(field: StructField): Boolean = + field.metadata.contains(SEQUENCE_ID) + + /** True iff any column in `schema` carries the sequence ID. */ + def hasConcurrentSequenceMetadata(schema: StructType): Boolean = + schema.exists(hasConcurrentSequenceMetadata) + + /** The driver-generated sequence id for this column, if present. */ + def getSequenceId(field: StructField): Option[String] = + if (hasConcurrentSequenceMetadata(field)) Some(field.metadata.getString(SEQUENCE_ID)) else None + + /** + * Names of the columns in `schema` that carry the sequence ID. Non-empty only + * on a table stamped with sequence IDs; empty on a stock-identity table. Used by + * the reservation downgrade guard to refuse a flag-off write that would otherwise pick the + * wrong backend. + */ + def stampedSequenceColumnNames(schema: StructType): Seq[String] = + schema.filter(hasConcurrentSequenceMetadata).map(_.name) + + /** + * Returns a copy of `metadata` with the sequence ID stripped from every + * identity column. Use this before invoking the create-table stamping hook on a path + * where the input schema may carry over a stale sequence ID (e.g. REPLACE TABLE), so the + * hook's idempotency short-circuit cannot reuse a sequenceId minted for a different + * table id. SYNC's downgrade path also uses it to drop a dangling sequence ID. + */ + def stripConcurrentSequenceMetadata(metadata: Metadata): Metadata = { + val schema = metadata.schema + val stripped = stripConcurrentSequenceMetadata(schema) + if (stripped eq schema) metadata else metadata.copy(schemaString = stripped.json) + } + + /** + * Returns a copy of `schema` with the sequence ID stripped from every identity column, or + * `schema` itself when no column carries one. + */ + def stripConcurrentSequenceMetadata(schema: StructType): StructType = { + val stripped = schema.map { field => + if (!hasConcurrentSequenceMetadata(field)) { + field + } else { + withoutConcurrentSequenceMetadata(field) + } + } + val anyStripped = stripped.zip(schema).exists { case (out, in) => out ne in } + if (!anyStripped) schema else schema.copy(fields = stripped.toArray) + } + + /** Returns a copy of `field` with the sequence ID removed. */ + def withoutConcurrentSequenceMetadata(field: StructField): StructField = + field.copy(metadata = new MetadataBuilder() + .withMetadata(field.metadata) + .remove(SEQUENCE_ID) + .build()) + + /** + * Returns a copy of `field` with the sequence ID attached. The standard `delta.identity.*` + * start/step/allowExplicitInsert keys are preserved, but the stock `delta.identity.highWaterMark` + * is removed. + */ + def withConcurrentSequenceMetadata(field: StructField, sequenceId: String): StructField = + field.copy(metadata = new MetadataBuilder() + .withMetadata(field.metadata) + .remove(DeltaSourceUtils.IDENTITY_INFO_HIGHWATERMARK) + .putString(SEQUENCE_ID, sequenceId) + .build()) +} diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala b/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala index 21da1e0209e..fe239deff8a 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala @@ -3596,6 +3596,17 @@ trait DeltaSQLConfBase extends DeltaSQLConfUtils { .booleanConf .createWithDefault(false) + val CONCURRENT_IDENTITY_COLUMN_MAX_COLUMNS_PER_TABLE = + buildConf("identityColumn.concurrent.maxColumnsPerTable") + .internal() + .doc("Maximum number of concurrent (service-backed) identity columns allowed on a single " + + "table. Enforced when the table opts into the feature: CREATE/REPLACE with the CIC " + + "feature and the feature opt-in conversion reject a table whose identity-column count " + + "exceeds this, before any sequence is minted.") + .intConf + .checkValue(_ > 0, "maxColumnsPerTable must be positive") + .createWithDefault(50) + ////////////////// // GeoSpatial ////////////////// diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/ConcurrentIdentityColumnErrorsSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/ConcurrentIdentityColumnErrorsSuite.scala new file mode 100644 index 00000000000..06a46481ef4 --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/ConcurrentIdentityColumnErrorsSuite.scala @@ -0,0 +1,62 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta + +import org.apache.spark.SparkFunSuite + +/** + * Unit tests for the internal CIC reservation-guard error builders in + * [[ConcurrentIdentityColumnErrors]]. EMPTY_RESERVE_RANGE, METADATA_MISMATCH, and + * USING_WRONG_GENERATOR are invariants/backstops that a wired write path is not expected to hit, + * so they are verified directly on the builder output with `checkError` (error class, SQLSTATE, + * and parameters , never the message prose). The remaining user-triggerable variants + * (CONVERSION_INCOMPLETE, DISABLED, SEQUENCE_NOT_FOUND) are exercised + * through their real paths in [[ConcurrentIdentityColumnServiceBackendSuite]] and + * [[ConcurrentIdentityColumnConversionSuite]]. + */ +class ConcurrentIdentityColumnErrorsSuite extends SparkFunSuite { + + test("emptyReserveRange reports EMPTY_RESERVE_RANGE with its parameters") { + checkError( + ConcurrentIdentityColumnErrors.emptyReserveRange(sequenceId = "seq-1", tableId = "tbl-1"), + condition = "DELTA_CONCURRENT_IDENTITY_COLUMN_EMPTY_RESERVE_RANGE", + sqlState = "XXKDS", + parameters = Map("sequenceId" -> "seq-1", "tableId" -> "tbl-1")) + } + + test("metadataMismatch reports METADATA_MISMATCH with its parameters") { + checkError( + ConcurrentIdentityColumnErrors.metadataMismatch( + grantedStep = 7L, sequenceId = "seq-1", columnStep = 3L), + condition = "DELTA_CONCURRENT_IDENTITY_COLUMN_METADATA_MISMATCH", + sqlState = "XXKDS", + parameters = Map( + "property" -> "step", + "grantedStep" -> "7", + "sequenceId" -> "seq-1", + "columnStep" -> "3")) + } + + test("usingWrongGenerator reports USING_WRONG_GENERATOR with its parameters") { + checkError( + ConcurrentIdentityColumnErrors.usingWrongGenerator( + columnNames = Seq("c1", "c2"), tableId = "tbl-1"), + condition = "DELTA_CONCURRENT_IDENTITY_COLUMN_USING_WRONG_GENERATOR", + sqlState = "XXKDS", + parameters = Map("columnNames" -> "c1, c2", "tableId" -> "tbl-1")) + } +} From b573e0024d5428efe436a5529673c11be36fb056 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthis=20G=C3=B6rdel?= Date: Wed, 14 Oct 2026 18:37:37 +0200 Subject: [PATCH 20/29] [Spark] Refactor DeleteSuiteBase: replace SharedSparkSession with DeltaSQLCommandTest (#7863) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #### Which Delta project/connector is this regarding? - [x] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [ ] Other (fill in here) ## Description In `DeleteSuiteBase`, replaces the `SharedSparkSession` test util with `DeltaSQLCommandTest` as the latter is the 'delta wrapper' of the former, to slightly decouple from Spark OSS. Also replaces `withSQLConf` with the more generic `withConf` to set confs via public APIs. ## How was this patch tested? test-only change. ## Does this PR introduce _any_ user-facing changes? No Co-authored-by: Matthis Gördel --- .../spark/sql/delta/DeleteSuiteBase.scala | 25 ++++++++++--------- .../spark/sql/delta/DeltaTestUtils.scala | 2 +- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeleteSuiteBase.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeleteSuiteBase.scala index abf18b521a1..38664e212af 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeleteSuiteBase.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeleteSuiteBase.scala @@ -18,6 +18,7 @@ package org.apache.spark.sql.delta // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta.sources.DeltaSQLConf +import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.{SparkThrowable, SparkUnsupportedOperationException} import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest, Row} @@ -28,7 +29,7 @@ import org.apache.spark.sql.types.StructType trait DeleteBaseMixin extends QueryTest - with SharedSparkSession + with DeltaSQLCommandTest with DeltaDMLTestUtils with DeltaTestUtilsForTempViews { @@ -219,7 +220,7 @@ trait DeleteBaseTests extends DeleteBaseMixin { Seq(true, false).foreach { skippingEnabled => Seq(true, false).foreach { isPartitioned => test(s"data and partition columns - Partition=$isPartitioned Skipping=$skippingEnabled") { - withSQLConf(DeltaSQLConf.DELTA_STATS_SKIPPING.key -> skippingEnabled.toString) { + withConf(DeltaSQLConf.DELTA_STATS_SKIPPING.key -> skippingEnabled.toString) { val partitions = if (isPartitioned) "key" :: Nil else Nil val input = Seq((2, 2), (1, 4), (1, 1), (0, 3)).toDF("key", "value") append(input, partitions) @@ -239,7 +240,7 @@ trait DeleteBaseTests extends DeleteBaseMixin { test("basic case with NullType") { assume(DeltaTestUtilsBase.nullTypeColumnsSupported) - withSQLConf(DeltaSQLConf.DELTA_CREATE_DATAFRAME_DROP_NULL_COLUMNS.key -> "false") { + withConf(DeltaSQLConf.DELTA_CREATE_DATAFRAME_DROP_NULL_COLUMNS.key -> "false") { append(Seq((null, 2), (null, 4), (null, 1), (null, 3)).toDF("key", "value")) checkDelete(condition = None, Nil) } @@ -247,7 +248,7 @@ trait DeleteBaseTests extends DeleteBaseMixin { test("basic case with condition on NullType") { assume(DeltaTestUtilsBase.nullTypeColumnsSupported) - withSQLConf(DeltaSQLConf.DELTA_CREATE_DATAFRAME_DROP_NULL_COLUMNS.key -> "false") { + withConf(DeltaSQLConf.DELTA_CREATE_DATAFRAME_DROP_NULL_COLUMNS.key -> "false") { append(Seq((null, 2), (null, 4), (null, 1), (null, 3)).toDF("key", "value")) checkDelete(condition = Some("key is null"), Nil) } @@ -255,7 +256,7 @@ trait DeleteBaseTests extends DeleteBaseMixin { test("basic case with nested NullType") { assume(DeltaTestUtilsBase.nullTypeColumnsSupported) - withSQLConf( + withConf( DeltaSQLConf.DELTA_CREATE_DATAFRAME_DROP_NULL_COLUMNS.key -> "false" ) { append(Seq(Tuple1((null, 2)), Tuple1((null, 4)), Tuple1(null), Tuple1((null, 1))).toDF("s")) @@ -430,7 +431,7 @@ trait DeleteBaseTests extends DeleteBaseMixin { assert(e4.contains("Subqueries are not supported")) // EXISTS mixed with IN: the IN subquery should still be blocked - withSQLConf(DeltaSQLConf.ALLOW_EXISTS_SUBQUERY_IN_DELETE.key -> "true") { + withConf(DeltaSQLConf.ALLOW_EXISTS_SUBQUERY_IN_DELETE.key -> "true") { checkError( intercept[DeltaAnalysisException] { executeDelete( @@ -448,7 +449,7 @@ trait DeleteBaseTests extends DeleteBaseMixin { append(Seq((2, 2), (1, 4), (1, 1), (0, 3)).toDF("key", "value")) Seq((2, 2), (1, 4)).toDF("c", "d").createOrReplaceTempView("source") - withSQLConf(DeltaSQLConf.ALLOW_EXISTS_SUBQUERY_IN_DELETE.key -> "true") { + withConf(DeltaSQLConf.ALLOW_EXISTS_SUBQUERY_IN_DELETE.key -> "true") { executeDelete( target = tableSQLIdentifier, where = "EXISTS (SELECT 1 FROM source WHERE key = c)") @@ -462,7 +463,7 @@ trait DeleteBaseTests extends DeleteBaseMixin { append(Seq((2, 2), (1, 4), (1, 1), (0, 3)).toDF("key", "value")) Seq((2, 2), (1, 4)).toDF("c", "d").createOrReplaceTempView("source") - withSQLConf(DeltaSQLConf.ALLOW_EXISTS_SUBQUERY_IN_DELETE.key -> "true") { + withConf(DeltaSQLConf.ALLOW_EXISTS_SUBQUERY_IN_DELETE.key -> "true") { executeDelete( target = tableSQLIdentifier, where = "NOT EXISTS (SELECT 1 FROM source WHERE key = c)") @@ -477,7 +478,7 @@ trait DeleteBaseTests extends DeleteBaseMixin { append(Seq((2, 2), (1, 4)).toDF("key", "value")) Seq((2, 2)).toDF("c", "d").createOrReplaceTempView("source") - withSQLConf( + withConf( DeltaSQLConf.ALLOW_EXISTS_SUBQUERY_IN_DELETE.key -> "false") { checkError( intercept[DeltaAnalysisException] { @@ -693,7 +694,7 @@ trait DeleteSubqueryBaseMixin extends DeleteBaseMixin { .toDF("c", "s2") .createOrReplaceTempView("source2") } - withSQLConf( + withConf( DeltaSQLConf.ALLOW_EXISTS_SUBQUERY_IN_DELETE.key -> "true" ) { executeDelete(target = "target t", where = deleteWhere) @@ -841,7 +842,7 @@ trait DeleteSubqueryExistsTests extends DeleteSubqueryBaseMixin { .toDF("c", "s1") .createOrReplaceTempView("source") - withSQLConf( + withConf( DeltaSQLConf.ALLOW_EXISTS_SUBQUERY_IN_DELETE.key -> "true" ) { executeDelete(target = "target t", @@ -866,7 +867,7 @@ trait DeleteSubqueryExistsTests extends DeleteSubqueryBaseMixin { .toDF("c", "s1") .createOrReplaceTempView("source") - withSQLConf( + withConf( DeltaSQLConf.ALLOW_EXISTS_SUBQUERY_IN_DELETE.key -> "true" ) { executeDelete(target = "target t", diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaTestUtils.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaTestUtils.scala index e1c0a5e65de..d9a2f45acbd 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaTestUtils.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaTestUtils.scala @@ -744,7 +744,7 @@ object DeltaTestUtils extends DeltaTestUtilsBase { } trait DeltaTestUtilsForTempViews - extends SharedSparkSession + extends DeltaSQLCommandTest with DeltaTestUtilsBase { def testWithTempView(testName: String)(testFun: Boolean => Any): Unit = { From 91fd083ff10c3646490890e3a147d0d158ea6609 Mon Sep 17 00:00:00 2001 From: Zhipeng Mao Date: Wed, 14 Oct 2026 18:53:13 +0200 Subject: [PATCH 21/29] [Spark] Preserve numeric precision when widening file statistics (#7854) #### Which Delta project/connector is this regarding? - [x] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [ ] Other (fill in here) ## Description Problem: Widening file statistics through the shared AddFile.withoutTightBoundStats helper can round DECIMAL bounds inward, causing data skipping to discard matching rows. The SQL below demonstrates this through DELETE rebasing over a concurrent OPTIMIZE; the fix also affects other callers of the shared helper. ``` CREATE TABLE t (id BIGINT, amount DECIMAL(18, 2)) USING DELTA TBLPROPERTIES ('delta.enableDeletionVectors' = 'true', 'delta.autoOptimize.autoCompact' = 'false'); INSERT INTO t SELECT /*+ COALESCE(1) */ * FROM VALUES (1L, 0.64), (2L, 3647609576981584.64), (3L, 1.00), (4L, 2.00), (5L, -3647609576981584.64) AS test_data(id, amount); DELETE FROM t WHERE id = 1; INSERT INTO t VALUES (6L, 3.00); -- Session 1: the test harness pauses this DELETE before commit. DELETE FROM t WHERE id = 3; -- Session 2: OPTIMIZE commits while Session 1 is paused. OPTIMIZE t; -- Resume Session 1 and let DELETE commit before running these queries. -- With the fix disabled and data skipping enabled, both incorrectly return no rows. SELECT id, amount FROM t WHERE amount = -3647609576981584.64; -- Expected: id=5 SELECT id, amount FROM t WHERE amount = 3647609576981584.64; -- Expected: id=2 ``` Root cause: AddFile.withoutTightBoundStats parses fractional numbers as Double in its JSON-tree rewrite while setting tightBounds=false. In the example, DV conflict rebasing reaches this helper through removeRows(updateStats=true). The amount bounds round inward from [-3647609576981584.64, 3647609576981584.64] to [-3647609576981584.50, 3647609576981584.50]. Both surviving endpoints then fall outside the recorded bounds, so data skipping discards their file. The precision loss is in the shared helper and can affect any caller that rewrites numeric bounds. Fix: Reuse Jackson's `copyCurrentEventExact` in `withoutTightBoundStats` to preserve numeric precision while replacing tightBounds. ## How was this patch tested? UT ## Does this PR introduce _any_ user-facing changes? No Signed-off-by: zhipeng.mao --- .../spark/sql/delta/actions/actions.scala | 71 ++++++++++++++++--- .../sql/delta/sources/DeltaSQLConf.scala | 11 +++ 2 files changed, 74 insertions(+), 8 deletions(-) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/actions/actions.scala b/spark/src/main/scala/org/apache/spark/sql/delta/actions/actions.scala index 1bef2086090..ba6b9b2836e 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/actions/actions.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/actions/actions.scala @@ -17,6 +17,7 @@ package org.apache.spark.sql.delta.actions // scalastyle:off import.ordering.noEmptyLine +import java.io.StringWriter import java.net.URI import java.sql.Timestamp import java.util.Locale @@ -36,12 +37,13 @@ import org.apache.spark.sql.delta.amt.{AMTPassthrough, AMTUtils} import org.apache.spark.sql.delta.commands.DeletionVectorUtils import org.apache.spark.sql.delta.metering.{DeltaLogging, DeltaLoggingProvider} import org.apache.spark.sql.delta.sources.DeltaSQLConf +import org.apache.spark.sql.delta.stats.DeltaStatistics.TIGHT_BOUNDS import org.apache.spark.sql.delta.util.{DeltaEncoder, DeltaFileOperations, JsonUtils, Utils => DeltaUtils} import org.apache.spark.sql.delta.util.FileNames import org.apache.spark.sql.delta.util.PartitionUtils import com.fasterxml.jackson.annotation._ import com.fasterxml.jackson.annotation.JsonInclude.Include -import com.fasterxml.jackson.core.{JsonGenerator, JsonParser} +import com.fasterxml.jackson.core.{JsonGenerator, JsonParser, JsonToken} import com.fasterxml.jackson.databind._ import com.fasterxml.jackson.databind.annotation.{JsonDeserialize, JsonSerialize} import com.fasterxml.jackson.databind.node.ObjectNode @@ -1092,16 +1094,69 @@ case class AddFile( if (stats == null || stats.isEmpty) { this } else { - val node = JsonUtils.mapper.readTree(stats).asInstanceOf[ObjectNode] - if (node.has("tightBounds") && - !node.get("tightBounds").asBoolean(true)) { - this + val precisionConf = DeltaSQLConf.DELTA_STATS_WIDEN_BOUNDS_PRESERVE_NUMERIC_PRECISION_ENABLED + val preserveStatsPrecision = SQLConf.get.getConf(precisionConf) + if (preserveStatsPrecision) { + withoutTightBoundStatsPreservingPrecision } else { - node.put("tightBounds", false) - val newStatsString = JsonUtils.mapper.writer.writeValueAsString(node) - this.copy(stats = newStatsString) + val node = JsonUtils.mapper.readTree(stats).asInstanceOf[ObjectNode] + if (node.has("tightBounds") && + !node.get("tightBounds").asBoolean(true)) { + this + } else { + node.put("tightBounds", false) + val newStatsString = JsonUtils.mapper.writer.writeValueAsString(node) + this.copy(stats = newStatsString) + } + } + } + } + + /** + * Set the top-level `tightBounds` field to false in the stats JSON while preserving + * numeric precision. + * Return `this` if bounds are already wide. + */ + private[delta] def withoutTightBoundStatsPreservingPrecision: AddFile = { + val writer = new StringWriter() + val factory = JsonUtils.mapper.getFactory + var needsUpdate = true + Utils.tryWithResource(factory.createParser(stats)) { parser => + require(parser.nextToken() == JsonToken.START_OBJECT, + "Expected file statistics to be a JSON object") + Utils.tryWithResource(factory.createGenerator(writer)) { generator => + generator.writeStartObject() + while (needsUpdate && parser.nextToken() != JsonToken.END_OBJECT) { + if (parser.currentName() == TIGHT_BOUNDS) { + parser.nextToken() + // `true` is the default if Jackson cannot convert the value to a boolean. + // In that case, widen the bounds as in the legacy path. + needsUpdate = parser.readValueAsTree[JsonNode]().asBoolean(true) + } else { + // Preserve decimal bounds without rounding through Double. + // Copy the field name and its value while preserving numeric precision. + generator.copyCurrentEventExact(parser) + parser.nextToken() + var depth = 0 + do { + val token = parser.currentToken() + generator.copyCurrentEventExact(parser) + if (token.isStructStart) { + depth += 1 + } else if (token.isStructEnd) { + depth -= 1 + } + if (depth > 0) { + parser.nextToken() + } + } while (depth > 0) + } + } + generator.writeBooleanField(TIGHT_BOUNDS, false) + generator.writeEndObject() } } + if (needsUpdate) this.copy(stats = writer.toString) else this } /** diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala b/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala index fe239deff8a..bb44fde6ad0 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala @@ -127,6 +127,17 @@ trait DeltaSQLConfBase extends DeltaSQLConfUtils { .booleanConf .createWithDefault(true) + val DELTA_STATS_WIDEN_BOUNDS_PRESERVE_NUMERIC_PRECISION_ENABLED = + buildConf("stats.widenBounds.preserveNumericPrecision.enabled") + .internal() + .doc( + """When enabled, AddFile.withoutTightBoundStats preserves numeric precision when + |updating file statistics to set tightBounds to false. When disabled, the legacy + |JSON update can round decimal bounds through Double and cause incorrect data + |skipping.""".stripMargin) + .booleanConf + .createWithDefault(false) + val DELTA_USER_METADATA = buildConf("commitInfo.userMetadata") .doc("Arbitrary user-defined metadata to include in CommitInfo.") From 6256b6a1d3fd29ebae3da90ea097c122add47889 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthis=20G=C3=B6rdel?= Date: Wed, 14 Oct 2026 19:39:19 +0200 Subject: [PATCH 22/29] [Spark] Refactor DVTestUtils: replace SharedSparkSession with DeltaSQLCommandTest (#7864) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #### Which Delta project/connector is this regarding? - [x] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [ ] Other (fill in here) ## Description Replaces `SharedSparkSession` with `DeltaSQLCommandTest`, which is a 'delta wrapper' of the former. Also replaces `withSQLConf` with the more generic `withConf`. ## How was this patch tested? test-only change. ## Does this PR introduce _any_ user-facing changes? No Co-authored-by: Matthis Gördel --- .../sql/delta/DeletionVectorsTestUtils.scala | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeletionVectorsTestUtils.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeletionVectorsTestUtils.scala index 1cd78a75d04..b98c64681f7 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeletionVectorsTestUtils.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeletionVectorsTestUtils.scala @@ -26,7 +26,7 @@ import org.apache.spark.sql.delta.commands.{AlterTableDropFeatureDeltaCommand, D import org.apache.spark.sql.delta.deletionvectors.{RoaringBitmapArray, RoaringBitmapArrayFormat} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.storage.dv.DeletionVectorStore -import org.apache.spark.sql.delta.test.DeltaSQLTestUtils +import org.apache.spark.sql.delta.test.{DeltaSQLCommandTest, DeltaSQLTestUtils} import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.util.PathWithFileSystem import org.apache.commons.io.FileUtils @@ -37,14 +37,13 @@ import org.apache.spark.internal.config.ConfigEntry import org.apache.spark.sql.{DataFrame, QueryTest, RuntimeConfig, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.functions.{col, lit} -import org.apache.spark.sql.test.SharedSparkSession -trait MergePersistentDVDisabled extends SharedSparkSession { +trait MergePersistentDVDisabled extends DeltaSQLCommandTest { override protected def sparkConf: SparkConf = super.sparkConf .set(DeltaSQLConf.MERGE_USE_PERSISTENT_DELETION_VECTORS.key, "false") } -trait PersistentDVDisabled extends SharedSparkSession { +trait PersistentDVDisabled extends DeltaSQLCommandTest { override protected def sparkConf: SparkConf = super.sparkConf .set(DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.defaultTablePropertyKey, "false") } @@ -56,14 +55,14 @@ trait PersistentDVEnabled extends DeletionVectorsTestUtils { } } -trait PredicatePushdownDisabled extends SharedSparkSession { +trait PredicatePushdownDisabled extends DeltaSQLCommandTest { override def beforeAll(): Unit = { super.beforeAll() spark.conf.set(DeltaSQLConf.DELETION_VECTORS_USE_METADATA_ROW_INDEX.key, "false") } } -trait PredicatePushdownEnabled extends SharedSparkSession { +trait PredicatePushdownEnabled extends DeltaSQLCommandTest { override def beforeAll(): Unit = { super.beforeAll() spark.conf.set(DeltaSQLConf.DELETION_VECTORS_USE_METADATA_ROW_INDEX.key, "true") @@ -71,7 +70,7 @@ trait PredicatePushdownEnabled extends SharedSparkSession { } /** Collection of test utilities related with persistent Deletion Vectors. */ -trait DeletionVectorsTestUtils extends QueryTest with SharedSparkSession with DeltaSQLTestUtils { +trait DeletionVectorsTestUtils extends QueryTest with DeltaSQLCommandTest with DeltaSQLTestUtils { def enableDeletionVectors( spark: SparkSession, @@ -131,7 +130,7 @@ trait DeletionVectorsTestUtils extends QueryTest with SharedSparkSession with De /** Run a thunk with Deletion Vectors enabled/disabled. */ def withDeletionVectorsEnabled(enabled: Boolean = true)(thunk: => Unit): Unit = { val enabledStr = enabled.toString - withSQLConf( + withConf( DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.defaultTablePropertyKey -> enabledStr, DeltaSQLConf.DELETE_USE_PERSISTENT_DELETION_VECTORS.key -> enabledStr, DeltaSQLConf.UPDATE_USE_PERSISTENT_DELETION_VECTORS.key -> enabledStr, @@ -160,7 +159,7 @@ trait DeletionVectorsTestUtils extends QueryTest with SharedSparkSession with De def createTable(tableNameOpt: Option[String], tablePathOpt: Option[Path]): Unit = { assert((tableNameOpt.isDefined && tablePathOpt.isEmpty) || (tableNameOpt.isEmpty && tablePathOpt.isDefined)) - withSQLConf(conf: _*) { + withConf(conf: _*) { val df = dataDF.write .option(DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.key, enableDVs.toString) .partitionBy(partitionBy: _*) @@ -406,7 +405,7 @@ trait DeletionVectorsTestUtils extends QueryTest with SharedSparkSession with De var numFiles: Option[Int] = None // This is needed to make the manual commit work correctly, since we are not actually // running a command that produces metrics. - withSQLConf(DeltaSQLConf.DELTA_HISTORY_METRICS_ENABLED.key -> "false") { + withConf(DeltaSQLConf.DELTA_HISTORY_METRICS_ENABLED.key -> "false") { val txn = log.startTransaction() val allAddFiles = txn.snapshot.allFiles.collect() numFiles = Some(allAddFiles.length) From 42777ea34a6ac7341d1c7e0425a4b34b4f17e68f Mon Sep 17 00:00:00 2001 From: Dhruv Arya Date: Wed, 14 Oct 2026 16:19:06 -0700 Subject: [PATCH 23/29] [Spark] Block users from setting ICT enablement table properties (#7865) #### Which Delta project/connector is this regarding? - [x] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [ ] Other (fill in here) ## Description `delta.inCommitTimestampEnablementVersion` and `delta.inCommitTimestampEnablementTimestamp` record the version and timestamp at which In-Commit Timestamps (ICT) were enabled on a table. They are written by the commit that enables ICT and describe that table's log. A user-specified value, for example one copied from another table, can point to a version that is not in the table's log. That breaks later commits and ICT-based time travel. This PR: - Marks both properties as not user-configurable, so CREATE, REPLACE, CTAS, RTAS, ALTER TABLE SET TBLPROPERTIES, DataFrameWriter options, session defaults and the `DeltaTable` builder API fail with `DELTA_CANNOT_MODIFY_TABLE_PROPERTY`. - Stops CREATE TABLE LIKE and CLONE from copying them from the source table. The target still gets `delta.enableInCommitTimestamps` from the source, and the commit derives the target's own enablement info when needed. - Adds `InCommitTimestampUtils.ENABLEMENT_INFO_PROPERTY_KEYS` so the filtering and the tests share one list of keys. ## How was this patch tested? New tests in `InCommitTimestampSuite`: - For each property, setting it fails through CREATE/REPLACE/CTAS/RTAS TBLPROPERTIES, ALTER TABLE SET TBLPROPERTIES, DataFrameWriter options, session defaults and the `DeltaTable` builder API. - CREATE TABLE LIKE from a table whose ICT enablement info is set produces a target that has ICT enabled but no copied enablement info. ## Does this PR introduce _any_ user-facing changes? Yes. Setting `delta.inCommitTimestampEnablementVersion` or `delta.inCommitTimestampEnablementTimestamp` now fails with `DELTA_CANNOT_MODIFY_TABLE_PROPERTY`; before, the value was accepted. CREATE TABLE LIKE and CLONE no longer copy these properties from the source table. Tables that already have these properties are unaffected. This pull request and its description were written by Isaac. --------- Co-authored-by: Murali Ramanujam --- .../spark-4.2/AbstractDeltaCatalogShims.scala | 3 + .../spark/sql/delta/DeltaAnalysis.scala | 2 + .../apache/spark/sql/delta/DeltaConfig.scala | 13 +- .../sql/delta/commands/CloneTableBase.scala | 3 + .../delta/util/InCommitTimestampUtils.scala | 8 ++ .../sql/delta/InCommitTimestampSuite.scala | 112 ++++++++++++++++++ .../delta/InCommitTimestampTestUtils.scala | 20 +++- 7 files changed, 156 insertions(+), 5 deletions(-) diff --git a/spark/src/main/scala-shims/spark-4.2/AbstractDeltaCatalogShims.scala b/spark/src/main/scala-shims/spark-4.2/AbstractDeltaCatalogShims.scala index 96d4d2c0182..69f978cd72e 100644 --- a/spark/src/main/scala-shims/spark-4.2/AbstractDeltaCatalogShims.scala +++ b/spark/src/main/scala-shims/spark-4.2/AbstractDeltaCatalogShims.scala @@ -34,6 +34,7 @@ import org.apache.spark.sql.delta.{ CatalogOwnedTableFeature, DeltaErrors, DeltaV2Mode, + InCommitTimestampUtils, MaterializedRowCommitVersion, MaterializedRowId } @@ -138,6 +139,8 @@ trait AbstractDeltaCatalogShims { self: AbstractDeltaCatalog => key != MaterializedRowId.MATERIALIZED_COLUMN_NAME_PROP && key != MaterializedRowCommitVersion.MATERIALIZED_COLUMN_NAME_PROP && !CoordinatedCommitsUtils.TABLE_PROPERTY_KEYS.contains(key) && + // ICT enablement provenance info should not be copied from the source + !InCommitTimestampUtils.ENABLEMENT_INFO_PROPERTY_KEYS.contains(key) && key != UC_TABLE_ID_KEY } val sourceProtocol = diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaAnalysis.scala b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaAnalysis.scala index 7d72da75afa..b73063c8d2a 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaAnalysis.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaAnalysis.scala @@ -206,6 +206,8 @@ class DeltaAnalysis(protected val session: SparkSession) .-(MaterializedRowId.MATERIALIZED_COLUMN_NAME_PROP) .-(MaterializedRowCommitVersion.MATERIALIZED_COLUMN_NAME_PROP) .filterKeys(!CoordinatedCommitsUtils.TABLE_PROPERTY_KEYS.contains(_)).toMap + // ICT enablement provenance info should not be copied from the source + .filterKeys(!InCommitTimestampUtils.ENABLEMENT_INFO_PROPERTY_KEYS.contains(_)).toMap // Catalog-Owned: Do not copy table UUID from source table .filterKeys(_ != UCCommitCoordinatorClient.UC_TABLE_ID_KEY).toMap diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaConfig.scala b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaConfig.scala index 2ccb2adb24b..150af1eab8d 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaConfig.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaConfig.scala @@ -981,27 +981,32 @@ trait DeltaConfigsBase extends DeltaLogging { /** * This table property is used to track the version of the table at which - * inCommitTimestamps were enabled. + * inCommitTimestamps were enabled. Internal property that cannot be set by users: a + * user-specified value (e.g. copied from another table) can point to a version that is not in + * this table's log and break later commits and ICT-based time travel. */ val IN_COMMIT_TIMESTAMP_ENABLEMENT_VERSION = buildConfig[Option[Long]]( "inCommitTimestampEnablementVersion", null, v => Option(v).map(_.toLong), validationFunction = _ => true, - "needs to be a long." + "needs to be a long.", + userConfigurable = false ) /** * This table property is used to track the timestamp at which inCommitTimestamps * were enabled. More specifically, it is the inCommitTimestamp of the commit with - * the version specified in [[IN_COMMIT_TIMESTAMP_ENABLEMENT_VERSION]]. + * the version specified in [[IN_COMMIT_TIMESTAMP_ENABLEMENT_VERSION]]. Internal property that + * cannot be set by users. */ val IN_COMMIT_TIMESTAMP_ENABLEMENT_TIMESTAMP = buildConfig[Option[Long]]( "inCommitTimestampEnablementTimestamp", null, v => Option(v).map(_.toLong), validationFunction = _ => true, - "needs to be a long.") + "needs to be a long.", + userConfigurable = false) /** * This property is used by CheckpointProtectionTableFeature and denotes the diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/commands/CloneTableBase.scala b/spark/src/main/scala/org/apache/spark/sql/delta/commands/CloneTableBase.scala index 2aa5afde29b..5fa4b505c81 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/commands/CloneTableBase.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/commands/CloneTableBase.scala @@ -315,6 +315,9 @@ abstract class CloneTableBase( val filteredConfiguration = clonedMetadata.configuration // Coordinated Commit configurations are never copied over to the target table. .filterKeys(!CoordinatedCommitsUtils.TABLE_PROPERTY_KEYS.contains(_)) + // ICT enablement provenance info refers to the source's log. The commit re-derives it for + // the target (see InCommitTimestampUtils.getUpdatedMetadataWithICTEnablementInfo). + .filterKeys(!InCommitTimestampUtils.ENABLEMENT_INFO_PROPERTY_KEYS.contains(_)) // Catalog-Owned enabled table's UC table ID property is never copied over to the // target table. .filterKeys(_ != UCCommitCoordinatorClient.UC_TABLE_ID_KEY) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/util/InCommitTimestampUtils.scala b/spark/src/main/scala/org/apache/spark/sql/delta/util/InCommitTimestampUtils.scala index dc40bea4a0c..f472894620f 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/util/InCommitTimestampUtils.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/util/InCommitTimestampUtils.scala @@ -34,6 +34,14 @@ object InCommitTimestampUtils { final val TABLE_PROPERTY_KEYS: Seq[String] = TABLE_PROPERTY_CONFS.map(_.key) + /** + * ICT enablement provenance properties. They describe the log they were written to, so they + * must never be set by users or copied to another table. + */ + final val ENABLEMENT_INFO_PROPERTY_KEYS: Seq[String] = Seq( + DeltaConfigs.IN_COMMIT_TIMESTAMP_ENABLEMENT_VERSION.key, + DeltaConfigs.IN_COMMIT_TIMESTAMP_ENABLEMENT_TIMESTAMP.key) + /** * Returns true if ICT was newly enabled by the commit at [[currentMetadataWithVersion.version]], * i.e. it is enabled there but was not enabled in the immediately preceding version. diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/InCommitTimestampSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/InCommitTimestampSuite.scala index b6fda123b64..1ae4fc5818e 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/InCommitTimestampSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/InCommitTimestampSuite.scala @@ -32,6 +32,7 @@ import org.apache.spark.sql.delta.test.{DeltaSQLCommandTest, DeltaSQLTestUtils} import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.util.{DateTimeUtils, DeltaCommitFileProvider, FileNames, JsonUtils, TimestampFormatter} +import io.delta.tables.{DeltaTable => IODeltaTable} import org.apache.hadoop.fs.Path import org.apache.spark.sql.QueryTest @@ -1810,6 +1811,117 @@ class InCommitTimestampSuite sql(s"REPLACE TABLE $tableName USING delta AS SELECT * FROM range(3, 4)") } } + private def checkICTEnablementInfoBlocked(key: String)(f: => Unit): Unit = { + checkError( + intercept[DeltaUnsupportedOperationException](f), + "DELTA_CANNOT_MODIFY_TABLE_PROPERTY", + parameters = Map("prop" -> key)) + } + + // Enables ICT at version `numCommitsBeforeEnablement + 1` so the table carries enablement info. + // Tests using this helper run via `testWithDefaultCommitCoordinatorUnset` because + // catalog-managed tables have ICT from creation and block ALTERing ICT properties. + // CatalogOwnedPropertySuite covers the catalog-managed combinations by upgrading a table. + private def createTableWithICTEnablementInfo( + tableName: String, + numCommitsBeforeEnablement: Int = 0, + numCommitsAfterEnablement: Int = 0): (DeltaLog, Map[String, String]) = { + withSQLConf(DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.defaultTablePropertyKey -> "false") { + spark.range(1).write.format("delta").saveAsTable(tableName) + (1 to numCommitsBeforeEnablement).foreach { _ => + spark.range(1).write.format("delta").mode("append").saveAsTable(tableName) + } + } + sql(s"ALTER TABLE $tableName " + + s"SET TBLPROPERTIES ('${DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.key}' = 'true')") + (1 to numCommitsAfterEnablement).foreach { _ => + spark.range(1).write.format("delta").mode("append").saveAsTable(tableName) + } + val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) + val expectedEnablementInfo = + expectedICTEnablementInfo(deltaLog, version = numCommitsBeforeEnablement + 1) + assert(getICTEnablementInfo(deltaLog.update().metadata) == expectedEnablementInfo) + (deltaLog, expectedEnablementInfo) + } + + for (key <- InCommitTimestampUtils.ENABLEMENT_INFO_PROPERTY_KEYS) { + test(s"user cannot set $key via CREATE/REPLACE/CTAS/RTAS TBLPROPERTIES") { + withTempTable(createTable = false) { tableName => + Seq( + s"CREATE TABLE $tableName (id LONG) USING delta TBLPROPERTIES ('$key' = '1170')", + s"CREATE OR REPLACE TABLE $tableName (id LONG) USING delta " + + s"TBLPROPERTIES ('$key' = '1170')", + s"CREATE TABLE $tableName USING delta TBLPROPERTIES ('$key' = '1170') " + + "AS SELECT * FROM range(1)" + ).foreach { command => + checkICTEnablementInfoBlocked(key) { sql(command) } + } + spark.range(1).write.format("delta").saveAsTable(tableName) + Seq( + s"REPLACE TABLE $tableName (id LONG) USING delta TBLPROPERTIES ('$key' = '1170')", + s"REPLACE TABLE $tableName USING delta TBLPROPERTIES ('$key' = '1170') " + + "AS SELECT * FROM range(1)", + s"CREATE OR REPLACE TABLE $tableName USING delta TBLPROPERTIES ('$key' = '1170') " + + "AS SELECT * FROM range(1)" + ).foreach { command => + checkICTEnablementInfoBlocked(key) { sql(command) } + } + } + } + + test(s"user cannot set $key via ALTER TABLE SET TBLPROPERTIES") { + withTempTable(createTable = false) { tableName => + spark.range(1).write.format("delta").saveAsTable(tableName) + checkICTEnablementInfoBlocked(key) { + sql(s"ALTER TABLE $tableName SET TBLPROPERTIES ('$key' = '1170')") + } + assert(!DeltaLog.forTable(spark, TableIdentifier(tableName)) + .update().metadata.configuration.contains(key)) + } + } + + test(s"user cannot set $key via DataFrameWriter options or session defaults") { + withTempDir { dir => + checkICTEnablementInfoBlocked(key) { + spark.range(1).write.format("delta").option(key, "1170").save(dir.getCanonicalPath) + } + } + // Session defaults are merged into every new table's properties. + withSQLConf(DeltaConfigs.sqlConfPrefix + key.stripPrefix("delta.") -> "1170") { + withTempTable(createTable = false) { tableName => + checkICTEnablementInfoBlocked(key) { + sql(s"CREATE TABLE $tableName (id LONG) USING delta") + } + } + } + } + + test(s"user cannot set $key via the DeltaTable builder API") { + withTempTable(createTable = false) { tableName => + checkICTEnablementInfoBlocked(key) { + IODeltaTable.createOrReplace(spark) + .tableName(tableName) + .addColumn("id", "LONG") + .property(key, "1170") + .execute() + } + } + } + } + + testWithDefaultCommitCoordinatorUnset( + "CREATE TABLE LIKE does not copy ICT enablement info from the source table") { + withTempTable(createTable = false) { sourceTableName => + withTempTable(createTable = false) { targetTableName => + createTableWithICTEnablementInfo(sourceTableName) + + sql(s"CREATE TABLE $targetTableName LIKE $sourceTableName") + assertICTEnabledWithoutEnablementInfo( + DeltaLog.forTable(spark, TableIdentifier(targetTableName)).update().metadata) + } + } + } + } class InCommitTimestampWithCatalogOwnedBatch1Suite extends InCommitTimestampSuite { diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/InCommitTimestampTestUtils.scala b/spark/src/test/scala/org/apache/spark/sql/delta/InCommitTimestampTestUtils.scala index 03fd2bb44f7..f59a8f5b21d 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/InCommitTimestampTestUtils.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/InCommitTimestampTestUtils.scala @@ -16,11 +16,29 @@ package org.apache.spark.sql.delta -import org.apache.spark.sql.delta.actions.{Action, CommitInfo} +import org.apache.spark.sql.delta.actions.{Action, CommitInfo, Metadata} import org.apache.spark.sql.delta.util.{DeltaCommitFileProvider, FileNames, JsonUtils} import org.apache.hadoop.fs.Path object InCommitTimestampTestUtils { + def getICTEnablementInfo(metadata: Metadata): Map[String, String] = + metadata.configuration + .filterKeys(InCommitTimestampUtils.ENABLEMENT_INFO_PROPERTY_KEYS.contains).toMap + + def assertICTEnabledWithoutEnablementInfo(metadata: Metadata): Unit = { + assert(DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.fromMetaData(metadata)) + assert(getICTEnablementInfo(metadata).isEmpty) + } + + /** The ICT enablement info a table should carry if ICT was enabled by commit `version`. */ + def expectedICTEnablementInfo(deltaLog: DeltaLog, version: Long): Map[String, String] = { + deltaLog.update() + Map( + DeltaConfigs.IN_COMMIT_TIMESTAMP_ENABLEMENT_VERSION.key -> version.toString, + DeltaConfigs.IN_COMMIT_TIMESTAMP_ENABLEMENT_TIMESTAMP.key -> + getInCommitTimestamp(deltaLog, version).toString) + } + /** * Overwrites the in-commit-timestamp in the delta file with the given timestamp. * It will also set operationParameters to an empty map because operationParameters From b315fbef602a8fd80e724a13737978bce7cfbe2b Mon Sep 17 00:00:00 2001 From: hari <39596845+hariuserx@users.noreply.github.com> Date: Wed, 14 Oct 2026 17:19:54 -0700 Subject: [PATCH 24/29] [Spark][Fix] Keep partition column names as single-part references in DeltaV2Table (#7866) #### Which Delta project/connector is this regarding? - [x] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [ ] Other (fill in here) ## Description DeltaV2Table` built its identity partition transforms with `Expressions.identity(String)`, which re-parses each partition column name as a SQL multipart identifier. Partition column names are exact top-level field names, so this goes wrong for names that need quoting: - A dotted name such as `customer.id` silently becomes the two-part reference `[customer, id]`. Consumers that expect one name part per identity partition column (for example `DataSourceV2Relation.partitionColumns`, which takes `fieldNames().head`) see the wrong column. - Names with spaces or backticks, which column mapping allows (for example `order date`), throw a `ParseException` as soon as `partitioning()` is called. This PR builds each transform from `FieldReference.column(name)`, a single exact name part. Partition-column order, case, and column-mapping logical names are unchanged. ## How was this patch tested? CI and new tests ## Does this PR introduce _any_ user-facing changes? No --- .../v2/read/DeltaV2ReadE2ESuite.scala | 41 ++++++++++ .../internal/v2/catalog/DeltaV2Table.scala | 7 +- .../internal/v2/catalog/DeltaV2TableTest.java | 78 +++++++++++++++++++ 3 files changed, 125 insertions(+), 1 deletion(-) diff --git a/spark-unified/src/test/scala/io/delta/spark/internal/v2/read/DeltaV2ReadE2ESuite.scala b/spark-unified/src/test/scala/io/delta/spark/internal/v2/read/DeltaV2ReadE2ESuite.scala index 433007aec93..267b0f32b29 100644 --- a/spark-unified/src/test/scala/io/delta/spark/internal/v2/read/DeltaV2ReadE2ESuite.scala +++ b/spark-unified/src/test/scala/io/delta/spark/internal/v2/read/DeltaV2ReadE2ESuite.scala @@ -266,6 +266,28 @@ private[read] trait DeltaV2ReadE2ETests { } } + test("partition names with spaces and backticks under column mapping prune on V2") { + val table = "v2_scan_e2e_spaced_partitions" + withTable(table) { + withV1Mode { + sql( + s"CREATE TABLE $table (id LONG, `order date` STRING, `a``b` STRING) " + + s"USING $tableProvider PARTITIONED BY (`order date`, `a``b`) " + + "TBLPROPERTIES ('delta.columnMapping.mode' = 'name')") + Seq("(1, 'd1', 'x')", "(2, 'd2', 'x')", "(3, 'd1', 'y')").foreach { values => + sql(s"INSERT INTO $table VALUES $values") + } + } + // Spark 4.0 keeps the partition filter above the scan, so the scan reads all 3 files. + checkPartitionPrunedReadMatchesV1( + s"SELECT id, `order date`, `a``b` FROM $table " + + "WHERE `order date` = 'd1' AND `a``b` = 'y'", + expectedRows = Seq(Row(3L, "d1", "y")), + expectedSelectedFiles = if (spark.version.startsWith("4.0")) 3 else 1, + context = "space and backtick partition-column filter") + } + } + test("deletion vectors with a middle partition column preserve rows and DDL order") { val table = "v2_scan_e2e_dv_partition_middle" withTable(table) { @@ -422,6 +444,25 @@ private[read] trait DeltaV2ReadE2ETests { } } } + + private def checkPartitionPrunedReadMatchesV1( + query: String, + expectedRows: Seq[Row], + expectedSelectedFiles: Int, + context: String): Unit = { + withV1Mode { + checkAnswer(sql(query), expectedRows) + } + checkRead(context)(sql(query)) { df => + checkAnswer(df, expectedRows) + } + withDeltaV2ScanForSelectedFiles(sql(query), context) { scan => + val numSelectedFiles = scan.preparedScan().files.size + assert( + numSelectedFiles == expectedSelectedFiles, + s"expected $expectedSelectedFiles selected files for $context, got $numSelectedFiles") + } + } } /** Covers read, schema, column-mapping, and deletion-vector semantics shared across Delta V2 diff --git a/spark/v2/src/main/scala/io/delta/spark/internal/v2/catalog/DeltaV2Table.scala b/spark/v2/src/main/scala/io/delta/spark/internal/v2/catalog/DeltaV2Table.scala index fa0ca6bfe68..41dffe1bef2 100644 --- a/spark/v2/src/main/scala/io/delta/spark/internal/v2/catalog/DeltaV2Table.scala +++ b/spark/v2/src/main/scala/io/delta/spark/internal/v2/catalog/DeltaV2Table.scala @@ -80,6 +80,7 @@ import io.delta.kernel.internal.{DeltaHistoryManager => KernelDeltaHistoryManage import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.catalog.{CatalogStatistics, CatalogTable} +import org.apache.spark.sql.catalyst.util.QuotingUtils import org.apache.spark.sql.connector.catalog.{ Column, Identifier, @@ -653,9 +654,13 @@ private[catalog] object DeltaV2Table { dataSchema = new StructType(dataFields.toArray(new Array[StructField](0))) partitionSchema = new StructType(partitionFields.toArray(new Array[StructField](0))) + // Each partition column is a single top-level field whose name may contain dots, spaces, or + // backticks. Expressions.identity parses its argument as a SQL multipart identifier, so + // quote the name to keep it as one exact name part (e.g. "a.b" must not become [a, b]). partitionTransforms = partColNames .stream() - .map[Transform](partColName => Expressions.identity(partColName)) + .map[Transform](partColName => + Expressions.identity(QuotingUtils.quoteIdentifier(partColName))) .toArray[Transform](length => new Array[Transform](length)) initialized = true diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/catalog/DeltaV2TableTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/catalog/DeltaV2TableTest.java index 86e0531d1d4..efd9aa0ce19 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/catalog/DeltaV2TableTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/catalog/DeltaV2TableTest.java @@ -385,6 +385,84 @@ static Stream tableTestCases() { .map(method -> Arguments.of(testCase, method))); } + /** + * Partition columns are top-level fields, so each identity transform must reference the exact + * column name as a single part, in partition-column order, even when the name would need quoting + * in SQL (dots, spaces, backticks, leading digits, mixed case). + */ + @ParameterizedTest(name = "{0} - {4}") + @MethodSource("quotedPartitionNameTestCases") + public void testPartitioningKeepsNamesRequiringQuotingAsSinglePart( + String layout, + String columnsAndPartitioning, + String tableProperties, + String[] expectedPartitionColumns, + ConstructionMethod method, + @TempDir File tempDir) + throws Exception { + String path = tempDir.getAbsolutePath(); + String tableName = "quoted_partitions_" + layout + "_" + method.name().toLowerCase(); + spark.sql( + String.format( + "CREATE TABLE %s %s %s LOCATION '%s'", + tableName, columnsAndPartitioning, tableProperties, path)); + Identifier identifier = Identifier.of(new String[] {"default"}, tableName); + DeltaV2Table table; + switch (method) { + case FROM_PATH: + table = new DeltaV2Table(identifier, path); + break; + case FROM_CATALOG_TABLE: + table = + new DeltaV2Table( + identifier, + spark.sessionState().catalog().getTableMetadata(new TableIdentifier(tableName)), + Collections.emptyMap()); + break; + default: + throw new IllegalArgumentException("Unknown construction method: " + method); + } + + Transform[] partitioning = table.partitioning(); + assertEquals(expectedPartitionColumns.length, partitioning.length); + for (int i = 0; i < expectedPartitionColumns.length; i++) { + assertEquals("identity", partitioning[i].name()); + assertEquals(1, partitioning[i].references().length); + assertArrayEquals( + new String[] {expectedPartitionColumns[i]}, + partitioning[i].references()[0].fieldNames(), + "Partition column reference mismatch at position " + i); + } + } + + static Stream quotedPartitionNameTestCases() { + // Without column mapping, names may still contain dots and start with digits. Spaces and + // backticks are only allowed under column mapping. + List layouts = + Arrays.asList( + Arguments.of( + "no_column_mapping", + "(id INT, `customer.id` STRING, `123_part` INT, Region STRING) USING delta " + + "PARTITIONED BY (Region, `customer.id`, `123_part`)", + "", + new String[] {"Region", "customer.id", "123_part"}), + Arguments.of( + "name_column_mapping", + "(id INT, `order date` STRING, `a``b` STRING, `x.y z` STRING) USING delta " + + "PARTITIONED BY (`order date`, `a``b`, `x.y z`)", + "TBLPROPERTIES ('delta.columnMapping.mode' = 'name')", + new String[] {"order date", "a`b", "x.y z"})); + return layouts.stream() + .flatMap( + layout -> + Stream.of(ConstructionMethod.FROM_PATH, ConstructionMethod.FROM_CATALOG_TABLE) + .map( + method -> { + Object[] args = layout.get(); + return Arguments.of(args[0], args[1], args[2], args[3], method); + })); + } + /** * Test that getDecodedPath handles various URI schemes correctly, not just file:// URIs. This * verifies the fix for supporting cloud storage paths (s3, abfss, gs) and HDFS. From f2eff2a5ac31b20ce7f46ec09c78b286945a27c4 Mon Sep 17 00:00:00 2001 From: Jiayuan Chen Date: Wed, 14 Oct 2026 17:28:25 -0700 Subject: [PATCH 25/29] [Spark] Add AMT variants of the DELETE test suites (#7849) #### Which Delta project/connector is this regarding? - [x] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [ ] Other (fill in here) ## Description Adds `DeleteAMT` and `RowTrackingDeleteAMT` dimensions to the delta-suite-generator, plus a generated `deleteamt` group that runs the DELETE tests against AMT (`adaptiveMetadata-preview`) tables, following the UPDATE AMT suites. - `DeleteAMTMixin` brackets each `executeDelete` with the `AMTDMLTestUtils` checkpoints. `RowTrackingDeleteAMTMixin` brackets each test table's body instead, since its row tracking checks inspect the latest commit. - AMT requires deletion vectors, so `AMTDMLTestUtils` now unsets a suite's `delta.enableDeletionVectors = false` default for new tables for all AMT DML suites (moved from the UPDATE AMT base), and keeps DELETE, UPDATE and MERGE from writing deletion vectors where the tests assert file rewrites. - The DELETE schema pruning tests only collect scans over a `TahoeFileIndex`, so they skip the checkpoint's tree read. - Excludes the tests the row tracking and deletion vector DELETE suites already exclude, and the row tracking tests that create tables with row tracking disabled, which AMT rejects. `RowTrackingDeleteDvBase` is not generated, since its only test disables deletion vectors on new tables. ## How was this patch tested? Test-only change. The new generated `deleteamt` suites run the existing DELETE tests on AMT tables. ## Does this PR introduce _any_ user-facing changes? No. --- .../suitegenerator/SuiteGeneratorConfig.scala | 22 ++++ .../spark/sql/delta/DeleteSuiteBase.scala | 7 +- .../spark/sql/delta/amt/AMTDMLTestUtils.scala | 34 ++++-- .../spark/sql/delta/amt/DeleteAMTMixin.scala | 107 ++++++++++++++++++ .../spark/sql/delta/amt/UpdateAMTMixin.scala | 10 +- .../deleteamt/DeleteBaseTests.scala | 39 +++++++ .../RowTrackingDeleteSuiteBase.scala | 43 +++++++ 7 files changed, 242 insertions(+), 20 deletions(-) create mode 100644 spark/src/test/scala/org/apache/spark/sql/delta/amt/DeleteAMTMixin.scala create mode 100644 spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/deleteamt/DeleteBaseTests.scala create mode 100644 spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/deleteamt/RowTrackingDeleteSuiteBase.scala diff --git a/spark/delta-suite-generator/src/main/scala/io/delta/suitegenerator/SuiteGeneratorConfig.scala b/spark/delta-suite-generator/src/main/scala/io/delta/suitegenerator/SuiteGeneratorConfig.scala index 56cdfacecad..ba59d70eec7 100644 --- a/spark/delta-suite-generator/src/main/scala/io/delta/suitegenerator/SuiteGeneratorConfig.scala +++ b/spark/delta-suite-generator/src/main/scala/io/delta/suitegenerator/SuiteGeneratorConfig.scala @@ -148,6 +148,8 @@ object SuiteGeneratorConfig { val UPDATE_ROW_TRACKING_DV = DimensionMixin("RowTrackingUpdateDV") val UPDATE_AMT = DimensionMixin("UpdateAMT", alias = Some("AMT")) val ROW_TRACKING_UPDATE_AMT = DimensionMixin("RowTrackingUpdateAMT", alias = Some("AMT")) + val DELETE_AMT = DimensionMixin("DeleteAMT", alias = Some("AMT")) + val ROW_TRACKING_DELETE_AMT = DimensionMixin("RowTrackingDeleteAMT", alias = Some("AMT")) val DELETE_SCALA = DimensionMixin("DeleteScala", alias = Some("Scala")) val DELETE_SQL = DimensionMixin("DeleteSQL", alias = Some("SQL")) val DELETE_WITH_DVS = DimensionMixin("DeleteSQLWithDeletionVectors", alias = Some("DV")) @@ -453,6 +455,26 @@ object SuiteGeneratorConfig { ) ) ), + TestGroup( + packageName = "deleteamt", + imports = List( + importer"org.apache.spark.sql.delta._", + importer"org.apache.spark.sql.delta.amt._", + importer"org.apache.spark.sql.delta.rowid._" + ), + testConfigs = List( + TestConfig( + List("DeleteBaseTests"), + List(List(Dims.DELETE_SQL, Dims.NAME_BASED, Dims.DELETE_AMT)) + ), + // Not RowTrackingDeleteDvBase: its only test disables deletion vectors on new tables, which + // AMT tables require. + TestConfig( + List("RowTrackingDeleteSuiteBase"), + List(List(Dims.PERSISTENT_DV, Dims.ROW_TRACKING_DELETE_AMT)) + ) + ) + ), TestGroup( packageName = "insert", imports = List( diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeleteSuiteBase.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeleteSuiteBase.scala index 38664e212af..92c20c6f4dc 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeleteSuiteBase.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeleteSuiteBase.scala @@ -17,6 +17,7 @@ package org.apache.spark.sql.delta // scalastyle:off import.ordering.noEmptyLine +import org.apache.spark.sql.delta.files.TahoeFileIndex import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest @@ -505,8 +506,9 @@ trait DeleteBaseTests extends DeleteBaseMixin { Row(1, 4) :: Row(1, 1) :: Row(0, 3) :: Nil) } + // Filter out Delta Metadata scans. val scans = executedPlans.flatMap(_.collect { - case f: FileSourceScanLike => f + case f: FileSourceScanLike if f.relation.location.isInstanceOf[TahoeFileIndex] => f }) // The first scan is for finding files to delete. We only are matching against the key @@ -529,8 +531,9 @@ trait DeleteBaseTests extends DeleteBaseMixin { Row(Row(1, 4)) :: Row(Row(1, 1)) :: Row(Row(0, 3)) :: Nil) } + // Filter out Delta Metadata scans. val scans = executedPlans.flatMap(_.collect { - case f: FileSourceScanLike => f + case f: FileSourceScanLike if f.relation.location.isInstanceOf[TahoeFileIndex] => f }) assert(scans.head.schema == StructType.fromDDL("nested STRUCT")) diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTDMLTestUtils.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTDMLTestUtils.scala index 5b0b733478c..401c83b61d9 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTDMLTestUtils.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTDMLTestUtils.scala @@ -37,15 +37,31 @@ import org.apache.spark.sql.functions.col */ trait AMTDMLTestUtils extends AMTCheckpointTestBase { - override protected def sparkConf: SparkConf = super.sparkConf - .set(defaultPropertyKey(AdaptiveMetadataTableFeature), FEATURE_PROP_SUPPORTED) - // Disable periodic checkpoints so the only AMT checkpoints are the ones the harness commits - // around each DML. - .set(DeltaConfigs.CHECKPOINT_INTERVAL.defaultTablePropertyKey, Int.MaxValue.toString) - // With the default cap, a small table's tree is a single root without leaves, and entries in - // the root carry no back reference. One entry per leaf gives any table with more than one file - // real leaves, so the files a DML removes carry back references into them. - .set(DeltaSQLConf.AMT_ENTRIES_PER_LEAF.key, "1") + override protected def sparkConf: SparkConf = { + val conf = super.sparkConf + .set(defaultPropertyKey(AdaptiveMetadataTableFeature), FEATURE_PROP_SUPPORTED) + // Disable periodic checkpoints so the only AMT checkpoints are the ones the harness commits + // around each DML. + .set(DeltaConfigs.CHECKPOINT_INTERVAL.defaultTablePropertyKey, Int.MaxValue.toString) + // With the default cap, a small table's tree is a single root without leaves, and entries + // in the root carry no back reference. One entry per leaf gives any table with more than one + // file real leaves, so the files a DML removes carry back references into them. + .set(DeltaSQLConf.AMT_ENTRIES_PER_LEAF.key, "1") + // Some suites default new tables to `delta.enableDeletionVectors = false`, which AMT rejects at + // CREATE since it requires deletion vectors. Unset it instead of setting it to true: AMT then + // enables deletion vectors on the table itself, while the tests, which read this default to + // predict whether a DML writes deletion vectors, keep seeing it off. Keep the DMLs from writing + // them to match, so they still rewrite files as those tests assert. + val dvsInNewTablesKey = DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.defaultTablePropertyKey + if (conf.getOption(dvsInNewTablesKey).contains("false")) { + Seq( + DeltaSQLConf.DELETE_USE_PERSISTENT_DELETION_VECTORS, + DeltaSQLConf.UPDATE_USE_PERSISTENT_DELETION_VECTORS, + DeltaSQLConf.MERGE_USE_PERSISTENT_DELETION_VECTORS + ).foreach(dmlConf => conf.set(dmlConf.key, "false")) + } + conf.remove(dvsInNewTablesKey) + } // Register the in-memory commit coordinator before super.beforeAll(). Some suites (e.g. // RowTrackingMergeSuiteBase) create catalog-managed tables in their own beforeAll(), which runs diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/DeleteAMTMixin.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/DeleteAMTMixin.scala new file mode 100644 index 00000000000..b98c56d150f --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/DeleteAMTMixin.scala @@ -0,0 +1,107 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.amt + +import org.apache.spark.sql.delta.DeleteBaseMixin +import org.apache.spark.sql.delta.rowid.RowTrackingDeleteTestDimension +import org.apache.spark.sql.delta.sources.DeltaSQLConf + +import org.apache.spark.SparkConf + +/** + * DELETE-specific setup for running the DELETE test suites against AMT tables with the + * [[AMTDMLTestUtils]] harness. + * + * The DELETE AMT mixins must come after all other mixins of a suite, so their dimensions must be + * the last of every generator config they are used in. This is for 2 reasons: + * 1. Visibility: some mixins declare beforeAll public, and a later mixin overriding it as + * protected (which AMTDMLTestUtils does) fails to compile. + * 2. Commit orders: some mixins perform extra commits (e.g. DeleteTableWithDVsMixin). To maintain + * a stable commit order, we prefer to have AMT commits wrap them all. + */ +trait DeleteAMTTestBase extends AMTDMLTestUtils { + + // Override as public to pass compilation + override def beforeAll(): Unit = super.beforeAll() +} + +/** + * Generates AMT variants of the DELETE test suites that run DELETE through + * [[DeleteBaseMixin.executeDelete]]. + * + * Each DELETE is bracketed by the [[AMTDMLTestUtils]] checkpoints. + */ +trait DeleteAMTMixin extends DeleteBaseMixin with DeleteAMTTestBase { + + // The base DELETE suites assert rewrite semantics (copied rows, commit tags, file counts), so + // keep DELETE from writing deletion vectors on the DV-enabled AMT tables. Unlike + // AMTDMLTestUtils, do this whatever the suite's default, since not every build of the base + // suites sets one. Suites that mix in DeleteDvMixin turn them back on at runtime. + override protected def sparkConf: SparkConf = super.sparkConf + .set(DeltaSQLConf.DELETE_USE_PERSISTENT_DELETION_VECTORS.key, "false") + + override def excluded: Seq[String] = super.excluded ++ Seq( + // These tests fail when row tracking or deletion vectors are enabled, and AMT tables always + // have both. The baseline DELETE suites exclude them for the same reason, so keep this list in + // sync with DeleteWithRowTrackingOverrides (row tracking) and DeleteDvOverrides (deletion + // vectors). + "DELETE preserves insertion time and zorder tags with BHJ with ephemeral deletion vectors", + "DELETE preserves insertion time and zorder tags without BHJ with ephemeral deletion vectors", + "DELETE preserves insertion time tags with 2 files per task", + "DELETE preserved tags with 2 files per task metric true", + "DELETE does not preserve insertion time tags without flag", + "DELETE does not preserve insertion time and zorder tags with BHJ", + "DELETE does not preserve insertion time and zorder tags without BHJ", + "Probe table in a broadcast join should not preserve order without AQE", + "SortMergeJoin does not preserve order without AQE", + "partition pruning", + "Deletion vectors are cleaned up with subquery.", + "delete logs error if number of records are missing in stats" + ) + + abstract override protected def executeDelete(target: String, where: String): Unit = { + withAMTCheckpointsAround(target) { + super.executeDelete(target, where) + } + } +} + +/** + * Generates AMT variants of the row tracking DELETE suites, which run DELETE through + * [[RowTrackingDeleteTestDimension.executeDelete]] rather than [[DeleteBaseMixin]]. + * + * The [[AMTDMLTestUtils]] checkpoints bracket each test table's body rather than each DELETE: the + * row tracking checks around each DELETE inspect the latest commit, which must be the DELETE + * itself. + */ +trait RowTrackingDeleteAMTMixin extends RowTrackingDeleteTestDimension with DeleteAMTTestBase { + + override def excluded: Seq[String] = super.excluded ++ Seq( + // These tests create the table with delta.enableRowTracking = false (one to assert row tracking + // stays off, the others to enable it later via backfill). AMT mandates row tracking, so table + // creation fails with DELTA_ADAPTIVE_METADATA_REQUIRES_DEPENDENT_FEATURE_ENABLED. + "Row tracking marked as not preserved when row tracking disabled", + "DELETE preserves Row ID on tables with row IDs enabled using backfill,isPartitioned=false", + "DELETE preserves Row ID on tables with row IDs enabled using backfill,isPartitioned=true" + ) + + override def withRowIdTestTable(isPartitioned: Boolean)(f: => Unit): Unit = { + super.withRowIdTestTable(isPartitioned) { + withAMTCheckpointsAround(testTableName)(f) + } + } +} diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/UpdateAMTMixin.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/UpdateAMTMixin.scala index e7f56691baa..c24be2433cf 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/amt/UpdateAMTMixin.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/UpdateAMTMixin.scala @@ -18,7 +18,7 @@ package org.apache.spark.sql.delta.amt import scala.util.Try -import org.apache.spark.sql.delta.{DeltaConfigs, UpdateBaseMixin} +import org.apache.spark.sql.delta.UpdateBaseMixin import org.apache.spark.sql.delta.coordinatedcommits.{CatalogOwnedCommitCoordinatorProvider, CatalogOwnedTableUtils, InMemoryCommitCoordinator, TrackingCommitCoordinatorClient} import org.apache.spark.sql.delta.rowid.RowTrackingUpdateSuiteBase import org.apache.spark.sql.delta.sources.DeltaSQLConf @@ -38,14 +38,6 @@ import org.apache.spark.SparkConf */ trait UpdateAMTTestBase extends AMTDMLTestUtils { - override protected def sparkConf: SparkConf = super.sparkConf - // The UPDATE suites default new tables to `delta.enableDeletionVectors = false`, which AMT - // rejects at CREATE since it requires deletion vectors. Unset it instead of setting it to true: - // AMT then enables deletion vectors on the table itself, while the tests, which read this - // default to predict whether UPDATE writes deletion vectors, keep seeing the command's - // behavior. Whether UPDATE writes them is a command conf that the mixins below decide. - .remove(DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.defaultTablePropertyKey) - // Override as public to pass compilation override def beforeAll(): Unit = super.beforeAll() } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/deleteamt/DeleteBaseTests.scala b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/deleteamt/DeleteBaseTests.scala new file mode 100644 index 00000000000..753f19dc00e --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/deleteamt/DeleteBaseTests.scala @@ -0,0 +1,39 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// *********************************************************************************** +// * This file is automatically generated. Manual modification is not allowed. * +// * There is a unit test that should prevent merging a manual change. * +// * * +// * To make changes to the suites, modify the generator script config at * +// * SuiteGeneratorConfig.scala and run it. The generator can be run via the * +// * sbt command deltaSuiteGenerator / run. * +// * * +// * DO NOT TOUCH ANYTHING IN THIS FILE! * +// *********************************************************************************** + +// scalastyle:off line.size.limit +package org.apache.spark.sql.delta.generatedsuites.deleteamt + +import org.apache.spark.sql.delta._ +import org.apache.spark.sql.delta.amt._ +import org.apache.spark.sql.delta.rowid._ + +class DeleteBaseSQLNameBasedAMTSuite + extends DeleteBaseTests + with DeleteSQLMixin + with DeltaDMLTestUtilsNameBased + with DeleteAMTMixin diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/deleteamt/RowTrackingDeleteSuiteBase.scala b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/deleteamt/RowTrackingDeleteSuiteBase.scala new file mode 100644 index 00000000000..5a3dd687cac --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/deleteamt/RowTrackingDeleteSuiteBase.scala @@ -0,0 +1,43 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// *********************************************************************************** +// * This file is automatically generated. Manual modification is not allowed. * +// * There is a unit test that should prevent merging a manual change. * +// * * +// * To make changes to the suites, modify the generator script config at * +// * SuiteGeneratorConfig.scala and run it. The generator can be run via the * +// * sbt command deltaSuiteGenerator / run. * +// * * +// * DO NOT TOUCH ANYTHING IN THIS FILE! * +// *********************************************************************************** + +// scalastyle:off line.size.limit +package org.apache.spark.sql.delta.generatedsuites.deleteamt + +import org.apache.spark.sql.delta._ +import org.apache.spark.sql.delta.amt._ +import org.apache.spark.sql.delta.rowid._ + +class RowTrackingDeleteSuiteBaseDVOffAMTSuite + extends RowTrackingDeleteSuiteBase + with PersistentDVDisabled + with RowTrackingDeleteAMTMixin + +class RowTrackingDeleteSuiteBaseDVOnAMTSuite + extends RowTrackingDeleteSuiteBase + with PersistentDVEnabled + with RowTrackingDeleteAMTMixin From b1c78934c7125374e0d3741bca482abfb43aea4f Mon Sep 17 00:00:00 2001 From: hari <39596845+hariuserx@users.noreply.github.com> Date: Wed, 14 Oct 2026 19:26:12 -0700 Subject: [PATCH 26/29] [Spark] Use DELTA_LIMIT_PUSHDOWN_ENABLED in the V2 scan builder (#7870) #### Which Delta project/connector is this regarding? - [x] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [ ] Other (fill in here) ## Description The V1 read path uses a LIMIT to prune files only when DeltaSQLConf.DELTA_LIMIT_PUSHDOWN_ENABLED is on, but DeltaV2ScanBuilder.pushLimit always accepted the limit, so the V2 connector ignored the config. pushLimit now declines the limit and records nothing when the config is off, so Spark applies its own LIMIT to an unlimited scan. Nothing changes when the config is on, which is the default. ## How was this patch tested? Added new tests ## Does this PR introduce _any_ user-facing changes? No --- .../read/DeltaV2LimitPushdownE2ESuite.scala | 23 +++++++++++++++++++ .../internal/v2/read/DeltaV2ScanBuilder.scala | 23 ++++++++++++------- 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/spark-unified/src/test/scala/io/delta/spark/internal/v2/read/DeltaV2LimitPushdownE2ESuite.scala b/spark-unified/src/test/scala/io/delta/spark/internal/v2/read/DeltaV2LimitPushdownE2ESuite.scala index b69e2e0ba10..ed641527d94 100644 --- a/spark-unified/src/test/scala/io/delta/spark/internal/v2/read/DeltaV2LimitPushdownE2ESuite.scala +++ b/spark-unified/src/test/scala/io/delta/spark/internal/v2/read/DeltaV2LimitPushdownE2ESuite.scala @@ -303,6 +303,29 @@ private[read] trait DeltaV2LimitPushdownE2ETests { } } + test("disabling Delta limit pushdown keeps LIMIT off the scan") { + val table = "v2_scan_e2e_limit_pushdown_disabled" + withTable(table) { + withV1Mode { + sql(s"CREATE TABLE $table (id INT) USING $tableProvider") + // One row per file so that a pushed limit would select fewer files than the table has. + (0 until 5).foreach { id => + sql(s"INSERT INTO $table VALUES ($id)") + } + } + + withSQLConf(DeltaSQLConf.DELTA_LIMIT_PUSHDOWN_ENABLED.key -> "false") { + val df = sql(s"SELECT * FROM $table LIMIT 2") + assert(df.collect().length == 2, "LIMIT 2 should return exactly 2 rows") + assertExpectedNoPushedLimit(df, "LIMIT with Delta limit pushdown disabled") + assertRouteSpecificInputFileCount( + df, + expected = 5, + context = "LIMIT with Delta limit pushdown disabled reads every file") + } + } + } + test("a non-deterministic filter prevents LIMIT pushdown") { val table = "v2_scan_e2e_limit_nondeterministic_filter" withTable(table) { diff --git a/spark/v2/src/main/scala/io/delta/spark/internal/v2/read/DeltaV2ScanBuilder.scala b/spark/v2/src/main/scala/io/delta/spark/internal/v2/read/DeltaV2ScanBuilder.scala index 47338223380..572ef061df8 100644 --- a/spark/v2/src/main/scala/io/delta/spark/internal/v2/read/DeltaV2ScanBuilder.scala +++ b/spark/v2/src/main/scala/io/delta/spark/internal/v2/read/DeltaV2ScanBuilder.scala @@ -23,6 +23,7 @@ import io.delta.kernel.engine.Engine import io.delta.spark.internal.v2.read.cdc.CDCSchemaContext import org.apache.spark.sql.delta.Snapshot +import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.stats.DeltaScan import io.delta.spark.internal.v2.DeltaV2Logging import org.apache.spark.sql.delta.v2.interop.DeltaV2Snapshot @@ -133,21 +134,27 @@ private[read] class DeltaV2ScanBuilder( /** * Accepts a LIMIT hint from Spark's optimizer. * - * Always returns `true`: the connector treats the limit as a best-effort hint. When the limit is - * kept (see [[build]]), file selection routes through V1's limit-aware `filesForScan`, which - * stops adding files once enough logical rows have accumulated. Because pruning happens at file - * granularity, the planned scan may still return more rows than requested (for example, a single - * file with 1,000 rows for LIMIT 5), so `isPartiallyPushed()` is left at its default of `true` - * and Spark keeps its limit operators as a backstop. + * The connector treats the limit as a best-effort hint. When the limit is kept (see [[build]]), + * file selection routes through V1's limit-aware `filesForScan`, which stops adding files once + * enough logical rows have accumulated. Because pruning happens at file granularity, the planned + * scan may still return more rows than requested (for example, a single file with 1,000 rows for + * LIMIT 5), so `isPartiallyPushed()` is left at its default of `true` and Spark keeps its limit + * operators as a backstop. * * @param limit the row limit requested by Spark which must be non-negative. + * @return `false`, keeping no limit, when `spark.databricks.delta.stats.limitPushdown.enabled` + * is off; `true` otherwise. */ override def pushLimit(limit: Int): Boolean = { if (limit < 0) { throw new IllegalArgumentException("Pushed limit must be non-negative, but got: " + limit) } - pushedLimit = OptionalInt.of(limit) - true + if (SQLConf.get.getConf(DeltaSQLConf.DELTA_LIMIT_PUSHDOWN_ENABLED)) { + pushedLimit = OptionalInt.of(limit) + true + } else { + false + } } // isPartiallyPushed() intentionally uses the interface default (true). Because pruning happens at From 56f12d4ae9e113ca0cbda43afeceab48d6c432f3 Mon Sep 17 00:00:00 2001 From: Eunjin Song Date: Wed, 14 Oct 2026 22:04:02 -0700 Subject: [PATCH 27/29] Address review: pair each read's data skipping with its own partition probe Fuse the conflict-time data-skipping filter into the per-read partition probe so a file conflicts iff it could match (P_i AND D_i) for some read, instead of the looser (P_1 OR P_2) AND (D_1 OR D_2). Partition pruning now runs first and no extra Spark job is launched. Move the evaluation fallback into ConflictChecker and record it as a delta event; record the noConflict event only after all checks against the winning commit pass. Add multi-read, null-stats, and event tests. Co-Authored-By: Claude Opus 5.5 --- .../spark/sql/delta/ConflictChecker.scala | 148 +++++++++++++----- .../stats/ConflictDataSkippingReader.scala | 106 ++++++------- .../sql/delta/ConflictDataSkippingSuite.scala | 105 ++++++++++++- 3 files changed, 260 insertions(+), 99 deletions(-) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/ConflictChecker.scala b/spark/src/main/scala/org/apache/spark/sql/delta/ConflictChecker.scala index 0201dfe8403..e6d6fb9179c 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/ConflictChecker.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/ConflictChecker.scala @@ -20,6 +20,7 @@ package org.apache.spark.sql.delta import java.util.concurrent.TimeUnit import scala.collection.mutable +import scala.util.control.NonFatal import org.apache.spark.sql.delta.DeltaOperations.{OP_SET_TBLPROPERTIES, ROW_TRACKING_BACKFILL_OPERATION_NAME, ROW_TRACKING_UNBACKFILL_OPERATION_NAME} import org.apache.spark.sql.delta.RowId.RowTrackingMetadataDomain @@ -231,6 +232,12 @@ private[delta] class ConflictChecker( protected def recordSkippedPhase(phase: String): Unit = timingStats += phase -> 0 + /** + * Set when conflict-time data skipping was applied to a checked-appends probe that then found no + * conflicting file. Reported once the whole check against the winning commit has passed. + */ + private var dataSkippingAppliedWithoutConflict = false + /** * This function checks conflict of the `initialCurrentTransactionInfo` against the * `winningCommitVersion` and returns an updated [[CurrentTransactionInfo]] that represents @@ -306,6 +313,12 @@ private[delta] class ConflictChecker( checkForDeletedFilesAgainstCurrentTxnDeletedFiles() resolveTimestampOrderingConflicts() + if (dataSkippingAppliedWithoutConflict) { + recordDeltaEvent(deltaLog, + opType = "delta.conflictDetection.dataSkipping.noConflict", + data = Map("winningCommitVersion" -> winningCommitVersion)) + } + logMetrics() currentTransactionInfo } @@ -994,58 +1007,44 @@ private[delta] class ConflictChecker( return None } - // Reader-side data skipping: narrow the candidate files using the read snapshot's column-stats - // skipping. A concurrently-added file whose stats prove it cannot match the read predicates is - // not a conflict -- this is what lets unpartitioned / liquid-clustered tables avoid conflicting - // on every added file below. Data predicates are applied *per read predicate* and the survivors - // OR-combined, because independent reads have OR, not AND, semantics: a file is a candidate if - // it could match ANY read; it is dropped only if its stats prove it fails EVERY read. All reads - // are evaluated in a single Spark job by filterFilesMatchingAnyReadPredicate. Skipping is not - // applied for a whole-table read, nor when there are no read predicates at all: a transaction - // can be a non-blind-append (e.g. it removed files) yet have no read predicates, and in that - // case every added file must remain a conflict candidate, exactly as when the feature is - // disabled. Guarding on non-empty read predicates avoids an empty survivor set silently - // suppressing a real conflict. - val candidateFiles = - if (spark.conf.get(DeltaSQLConf.DELTA_CONFLICT_DETECTION_DATA_SKIPPING_ENABLED) && - !currentTransactionInfo.readWholeTable && - currentTransactionInfo.readPredicates.nonEmpty) { - val readSnapshot = currentTransactionInfo.readSnapshot - val remaining = readSnapshot.filterFilesMatchingAnyReadPredicate( - files, currentTransactionInfo.readPredicates.map(_.dataPredicates).toSeq) - if (remaining.size < files.size) { - recordDeltaEvent(deltaLog, - opType = "delta.conflictDetection.dataSkipping.filesSkipped", - data = Map( - "candidateFiles" -> files.size, - "skippedFiles" -> (files.size - remaining.size))) - } - remaining - } else { - files - } + // Reader-side data skipping: a concurrently-added file whose column stats prove it cannot match + // the read predicates is not a conflict -- this is what lets unpartitioned / liquid-clustered + // tables avoid conflicting on every added file. Not applied for a whole-table read, nor when + // there are no read predicates: a non-blind-append (e.g. it removed files) can have no read + // predicates, and then every added file must remain a conflict candidate, exactly as when the + // feature is disabled. + val useDataSkipping = + spark.conf.get(DeltaSQLConf.DELTA_CONFLICT_DETECTION_DATA_SKIPPING_ENABLED) && + !currentTransactionInfo.readWholeTable && + currentTransactionInfo.readPredicates.nonEmpty + + import org.apache.spark.sql.delta.implicits._ // There is no reason to filter files if the table is not partitioned. if (currentTransactionInfo.readWholeTable || currentTransactionInfo.readSnapshot.metadata.partitionColumns.isEmpty) { - return candidateFiles.headOption + if (!useDataSkipping || files.isEmpty) return files.headOption + return getFirstFileMatchingDataPredicates( + files.toDF(spark), currentTransactionInfo.readPredicates.map(_.dataPredicates)) } - import org.apache.spark.sql.delta.implicits._ - val filesDf = candidateFiles.toDF(spark) + val filesDf = files.toDF(spark) spark.conf.get(DeltaSQLConf.DELTA_CONFLICT_DETECTION_WIDEN_NONDETERMINISTIC_PREDICATES) match { case DeltaSQLConf.NonDeterministicPredicateWidening.OFF => getFirstFileMatchingPartitionPredicatesInternal( - filesDf, shouldWidenNonDeterministicPredicates = false, shouldWidenAllUdf = false) + filesDf, shouldWidenNonDeterministicPredicates = false, shouldWidenAllUdf = false, + useDataSkipping) case wideningMode => val fileWithWidening = getFirstFileMatchingPartitionPredicatesInternal( - filesDf, shouldWidenNonDeterministicPredicates = true, shouldWidenAllUdf = true) + filesDf, shouldWidenNonDeterministicPredicates = true, shouldWidenAllUdf = true, + useDataSkipping) fileWithWidening.flatMap { fileWithWidening => val fileWithoutWidening = getFirstFileMatchingPartitionPredicatesInternal( - filesDf, shouldWidenNonDeterministicPredicates = false, shouldWidenAllUdf = false) + filesDf, shouldWidenNonDeterministicPredicates = false, shouldWidenAllUdf = false, + useDataSkipping) if (fileWithoutWidening.isEmpty) { // Conflict due to widening of non-deterministic predicate. recordDeltaEvent(deltaLog, @@ -1071,7 +1070,8 @@ private[delta] class ConflictChecker( private def getFirstFileMatchingPartitionPredicatesInternal( filesDf: DataFrame, shouldWidenNonDeterministicPredicates: Boolean, - shouldWidenAllUdf: Boolean): Option[AddFile] = { + shouldWidenAllUdf: Boolean, + useDataSkipping: Boolean): Option[AddFile] = { def rewritePredicateFn( predicate: Expression, @@ -1101,15 +1101,44 @@ private[delta] class ConflictChecker( canonicalRewrites ++ canonicalNonRewrites } + // With data skipping, each probe is a partition predicate plus the data predicates of the reads + // that share it, so a read's data predicates are only combined with that same read's partition + // predicate: a file conflicts iff it could match (P_i AND D_i) for some read i, not the looser + // (P_1 OR P_2) AND (D_1 OR D_2). Like ExpressionSet, non-deterministic predicates are never + // merged. + val probes: Seq[(DeltaTableReadPredicate, Seq[Seq[Expression]])] = if (useDataSkipping) { + val groups = + mutable.LinkedHashMap[Any, (DeltaTableReadPredicate, mutable.Buffer[Seq[Expression]])]() + currentTransactionInfo.readPredicates.zipWithIndex.foreach { case (rp, i) => + val key = if (rp.partitionPredicate.deterministic) { + (rp.shouldRewriteFilter, rp.partitionPredicate.canonicalized) + } else { + i + } + groups.getOrElseUpdate(key, (rp, mutable.ArrayBuffer.empty))._2 += rp.dataPredicates + } + groups.values.toSeq.map { case (rp, dataFiltersPerRead) => + rewritePredicateFn(rp.partitionPredicate, rp.shouldRewriteFilter) -> + dataFiltersPerRead.toSeq + } + } else { + canonicalPredicates.toSeq.map(_ -> Seq.empty[Seq[Expression]]) + } + import org.apache.spark.sql.delta.implicits._ - val filesMatchingPartitionPredicates = canonicalPredicates.iterator - .flatMap { readPredicate => - val matchingFileOpt = DeltaLog.filterFileList( + val filesMatchingPartitionPredicates = probes.iterator + .flatMap { case (readPredicate, dataFiltersPerRead) => + val filesMatchingPartition = DeltaLog.filterFileList( partitionSchema = currentTransactionInfo.partitionSchemaAtReadTime, files = filesDf, partitionFilters = readPredicate.partitionPredicates, shouldRewritePartitionFilters = readPredicate.shouldRewriteFilter - ).as[AddFile].head(1).headOption + ) + val matchingFileOpt = if (useDataSkipping) { + getFirstFileMatchingDataPredicates(filesMatchingPartition, dataFiltersPerRead) + } else { + filesMatchingPartition.as[AddFile].head(1).headOption + } matchingFileOpt.foreach { f => logInfo(log"Partition predicate is matching a file changed by the winning transaction: " + log"predicate=${MDC(DeltaLogKeys.DATA_FILTER, @@ -1122,6 +1151,43 @@ private[delta] class ConflictChecker( filesMatchingPartitionPredicates.headOption } + /** + * Returns the first file in `filesDf` whose column stats do not prove it fails every read in + * `dataFiltersPerRead`, i.e. a file that must still be treated as a conflict. The skipping filter + * is fused into the same `head(1)` job as the caller's partition filter, which is applied first. + * + * Skipping is a pure optimization over conservative conflict detection: if building or + * evaluating the skipping filter fails, this records an event and falls back to checking without + * skipping, exactly as when the feature is disabled. + */ + private def getFirstFileMatchingDataPredicates( + filesDf: DataFrame, + dataFiltersPerRead: Seq[Seq[Expression]]): Option[AddFile] = { + import org.apache.spark.sql.delta.implicits._ + def firstFile(df: DataFrame): Option[AddFile] = df.as[AddFile].head(1).headOption + try { + currentTransactionInfo.readSnapshot + .filterFilesDfMatchingAnyReadPredicate(filesDf, dataFiltersPerRead) match { + case Some(skippedDf) => + val matchingFileOpt = firstFile(skippedDf) + if (matchingFileOpt.isEmpty) dataSkippingAppliedWithoutConflict = true + matchingFileOpt + case None => firstFile(filesDf) + } + } catch { + case NonFatal(e) => + logWarning(log"Conflict-time data skipping failed; falling back to treating all added " + + log"files as conflict candidates", e) + recordDeltaEvent(deltaLog, + opType = "delta.conflictDetection.dataSkipping.fallback", + data = Map( + "winningCommitVersion" -> winningCommitVersion, + "exceptionClass" -> e.getClass.getName, + "exceptionMessage" -> e.getMessage)) + firstFile(filesDf) + } + } + /** * RowTrackingBackfill does not do any data change. If backfill is the winning commit, the * current transaction does not need to read its AddFiles -- the exact same AddFiles have diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/stats/ConflictDataSkippingReader.scala b/spark/src/main/scala/org/apache/spark/sql/delta/stats/ConflictDataSkippingReader.scala index 2305397b7f5..c7e84fda0bb 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/stats/ConflictDataSkippingReader.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/stats/ConflictDataSkippingReader.scala @@ -16,8 +16,6 @@ package org.apache.spark.sql.delta.stats -import scala.util.control.NonFatal - import org.apache.spark.sql.delta.ClassicColumnConversions._ import org.apache.spark.sql.delta.DeltaTableUtils import org.apache.spark.sql.delta.actions.AddFile @@ -26,7 +24,7 @@ import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.schema.SchemaUtils import org.apache.spark.sql.delta.sources.DeltaSQLConf -import org.apache.spark.sql.Column +import org.apache.spark.sql.{Column, DataFrame} import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.functions.{col, from_json} @@ -102,7 +100,7 @@ trait ConflictDataSkippingReader extends DeltaLogging { self: DataSkippingReader * `dataFilters` must come from a single logical read: they are AND-combined via * [[buildDataSkippingPredicate]]. This is the one-read case of * [[filterFilesMatchingAnyReadPredicate]]; callers with multiple independent reads should use - * that method so all reads are evaluated in a single Spark job. + * that method. */ private[delta] def filterFilesByDataSkipping( files: Seq[AddFile], @@ -112,61 +110,61 @@ trait ConflictDataSkippingReader extends DeltaLogging { self: DataSkippingReader /** * Conflict-detection helper (reader-side data skipping) over several INDEPENDENT reads: returns * the subset of `files` that could still match ANY read and therefore must be treated as - * conflicts. Each inner `Seq[Expression]` is one logical read's data filters (AND-combined via - * [[buildDataSkippingPredicate]]); the reads are OR-combined, matching read semantics -- a file - * is a candidate if it could match any one of them. - * - * All reads are evaluated in a SINGLE Spark job: the per-read skipping predicates are OR-ed - * together into one `where` clause, rather than filtering per read and unioning the survivors - * (which launched one job per read). Note we cannot instead flatten every read's filters into one - * predicate -- that would AND them (read1 AND read2), the opposite of the OR we need. - * - * One-way safe: each read contributes `expr || !verifyStatsForFilter(...)` exactly as - * [[getDataSkippedFiles]], so a file is dropped only when its stats *prove* it fails EVERY read - * -- a real conflict is never a false negative. Files with missing/insufficient stats, a - * read with no usable skipping predicate (empty / ineligible filters -> matches everything), or a - * table without stats are all kept. Returns the original [[AddFile]]s (matched by path). - * - * Fail-safe: skipping here is a pure optimization over correct (conservative) conflict detection, - * so if building or evaluating the predicate throws we fall back to the default behavior of - * keeping all `files` as conflict candidates rather than failing the commit. + * conflicts. Collecting wrapper around [[filterFilesDfMatchingAnyReadPredicate]]. */ private[delta] def filterFilesMatchingAnyReadPredicate( files: Seq[AddFile], dataFiltersPerRead: Seq[Seq[Expression]]): Seq[AddFile] = { import org.apache.spark.sql.delta.implicits._ - if (files.isEmpty || dataFiltersPerRead.isEmpty || schema.isEmpty) return files - try { - // One skipping predicate per read. A read with no usable predicate (empty or ineligible - // filters) matches every file -> nothing can be skipped, so keep all files without a job. - val perReadPredicates = dataFiltersPerRead.map { dataFilters => - if (dataFilters.isEmpty) None else buildDataSkippingPredicate(dataFilters) - } - if (perReadPredicates.exists(_.isEmpty)) return files - // Survive if the file could match ANY read. Per read, `expr || !verifyStatsForFilter(...)` - // keeps any file whose referenced stats are missing/NULL (mirrors getDataSkippedFiles): only - // skip when stats prove no match. OR the reads so a match against any one keeps the file. - val survivorCondition = perReadPredicates.flatten - .map(pred => pred.expr || !verifyStatsForFilter(pred.referencedStats)) - .reduce(_ || _) - val survivingPaths = recordFrameProfile( - "Delta", "DataSkippingReader.filterFilesMatchingAnyReadPredicate") { - files.toDF(spark) - .withColumn("stats", parseAndDecodeStats(col("stats"))) - .where(survivorCondition) - .select("path") - .collect() - .map(_.getString(0)) - .toSet - } - files.filter(f => survivingPaths.contains(f.path)) - } catch { - case NonFatal(e) => - // Optimization only: never let a skipping failure abort a commit. Fall back to the default - // (feature-off) behavior of treating every added file as a conflict candidate. - logWarning(log"Conflict-time data skipping failed to evaluate; falling back to treating " + - log"all added files as conflict candidates", e) - files + if (files.isEmpty) return files + filterFilesDfMatchingAnyReadPredicate(files.toDF(spark), dataFiltersPerRead) match { + case None => files + case Some(df) => + val survivingPaths = recordFrameProfile( + "Delta", "DataSkippingReader.filterFilesMatchingAnyReadPredicate") { + df.select("path").collect().map(_.getString(0)).toSet + } + files.filter(f => survivingPaths.contains(f.path)) + } + } + + /** + * Lazily restricts `filesDf` (a DataFrame of [[AddFile]]s with JSON `stats`) to the files that + * could still match ANY of several INDEPENDENT reads. Each inner `Seq[Expression]` is one logical + * read's data filters (AND-combined via [[buildDataSkippingPredicate]]); the reads are + * OR-combined, matching read semantics -- a file is a candidate if it could match any one of + * them. Returns None when nothing can be skipped: no reads, a table without stats, or a read + * with no usable skipping predicate (empty / ineligible filters match every file). + * + * No job is launched here, so callers can fuse the filter into their own query, e.g. after a + * partition filter so stats are only parsed for files in matching partitions. The returned + * DataFrame keeps the input's columns, with `stats` still the original JSON string. + * + * One-way safe: each read contributes `expr || !verifyStatsForFilter(...)` exactly as + * [[getDataSkippedFiles]], so a file is dropped only when its stats *prove* it fails EVERY read + * -- a real conflict is never a false negative. Files with missing/insufficient stats are kept. + */ + private[delta] def filterFilesDfMatchingAnyReadPredicate( + filesDf: DataFrame, + dataFiltersPerRead: Seq[Seq[Expression]]): Option[DataFrame] = { + if (dataFiltersPerRead.isEmpty || schema.isEmpty) return None + val perReadPredicates = dataFiltersPerRead.map { dataFilters => + if (dataFilters.isEmpty) None else buildDataSkippingPredicate(dataFilters) } + if (perReadPredicates.exists(_.isEmpty)) return None + // Per read, `expr || !verifyStatsForFilter(...)` keeps any file whose referenced stats are + // missing/NULL (mirrors getDataSkippedFiles); OR the reads so a match against any one keeps it. + val survivorCondition = perReadPredicates.flatten + .map(pred => pred.expr || !verifyStatsForFilter(pred.referencedStats)) + .reduce(_ || _) + // The skipping expressions reference the parsed `stats` struct; keep the raw JSON aside so the + // result still has the AddFile shape. + val rawStatsCol = "__conflict_skipping_raw_stats" + Some(filesDf + .withColumn(rawStatsCol, col("stats")) + .withColumn("stats", parseAndDecodeStats(col("stats"))) + .where(survivorCondition) + .withColumn("stats", col(rawStatsCol)) + .drop(rawStatsCol)) } } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/ConflictDataSkippingSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/ConflictDataSkippingSuite.scala index bf5648ce97e..9786920e58f 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/ConflictDataSkippingSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/ConflictDataSkippingSuite.scala @@ -19,8 +19,12 @@ package org.apache.spark.sql.delta import java.io.File import scala.concurrent.duration.Duration +import scala.util.Try + +import com.databricks.spark.util.Log4jUsageLogger import org.apache.spark.sql.delta.actions.AddFile +import org.apache.spark.sql.delta.DeltaTestUtils.filterUsageRecords import org.apache.spark.sql.delta.concurrency.PhaseLockingTestMixin import org.apache.spark.sql.delta.concurrency.TransactionExecutionTestMixin import org.apache.spark.sql.delta.sources.DeltaSQLConf @@ -28,10 +32,10 @@ import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.SparkException import org.apache.spark.sql.{QueryTest, Row, SaveMode} -import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression, GreaterThanOrEqual, LessThan, Literal} +import org.apache.spark.sql.catalyst.expressions.{AttributeReference, EqualTo, Expression, GreaterThanOrEqual, LessThan, Literal} import org.apache.spark.sql.functions.lit import org.apache.spark.sql.test.SharedSparkSession -import org.apache.spark.sql.types.LongType +import org.apache.spark.sql.types.{IntegerType, LongType} import org.apache.spark.util.ThreadUtils /** @@ -172,8 +176,11 @@ class ConflictDataSkippingSuite extends QueryTest } } - private def manufacturedAdd(name: String): AddFile = - AddFile(name, Map.empty[String, String], size = 1L, modificationTime = 1L, dataChange = true) + private def manufacturedAdd( + name: String, partitionValues: Map[String, String] = Map.empty): AddFile = + AddFile(name, partitionValues, size = 1L, modificationTime = 1L, dataChange = true) + + private val noConflictEvent = "delta.conflictDetection.dataSkipping.noConflict" test("empty read predicates on a non-blind-append txn: added file still conflicts") { // Soundness of the Serializable blind-append case. A transaction can be a NON-blind append @@ -277,4 +284,94 @@ class ConflictDataSkippingSuite extends QueryTest s"a read with no predicate must keep all files, got ${allKept.size}") } } + + test("files with null stats are kept while files whose stats prove no match are skipped") { + // Exercises the per-file `!verifyStatsForFilter` keep path: the skipping predicate IS built + // (the table has min/max stats), but one candidate file carries no stats at all. + withTempDir { dir => + createTable(dir) + val snapshot = DeltaLog.forTable(spark, dir.getCanonicalPath).update() + val allFiles = snapshot.allFiles.collect().toSeq + val noStats = manufacturedAdd("no-stats.parquet") + assert(noStats.stats == null) + + val id = AttributeReference("id", LongType)() + val survivors = snapshot.filterFilesByDataSkipping(allFiles :+ noStats, + Seq(LessThan(id, Literal(50L)))) + // Only the [0,100) file can match id<50; the other 9 are skipped by their stats. The file + // without stats cannot be proven to miss, so it is kept. + assert(survivors.size == 2, s"got ${survivors.map(_.path).sorted}") + assert(survivors.exists(_.path == noStats.path)) + } + } + + test("concurrently added file with null stats still conflicts") { + withTempDir { dir => + createTable(dir) + val log = DeltaLog.forTable(spark, dir.getCanonicalPath) + val id = AttributeReference("id", LongType)() + withSQLConf(DeltaSQLConf.DELTA_CONFLICT_DETECTION_DATA_SKIPPING_ENABLED.key -> "true") { + val loser = log.startTransaction() + loser.filterFiles(Seq(LessThan(id, Literal(50L)))) + log.startTransaction().commit( + Seq(manufacturedAdd("winner-no-stats.parquet")), DeltaOperations.Write(SaveMode.Append)) + intercept[io.delta.exceptions.ConcurrentAppendException] { + loser.commit( + Seq(manufacturedAdd("loser.parquet")), DeltaOperations.Write(SaveMode.Append)) + } + } + } + } + + test("multiple reads: data predicates are paired with their own partition predicate") { + // Reads `p=1 AND id<50` and `p=2 AND id>=100`. A file in p=1 with id=150 matches neither read, + // but matches (p=1 OR p=2) AND (id<50 OR id>=100), so evaluating partition and data + // predicates independently would report a false conflict. + withTempDir { dir => + spark.range(0, 1000, 1, 4).withColumn("p", lit(1)) + .union(spark.range(0, 1000, 1, 4).withColumn("p", lit(2))) + .write.partitionBy("p").format("delta").save(dir.getAbsolutePath) + sql(s"ALTER TABLE ${tableRef(dir)} SET TBLPROPERTIES " + + s"('${DeltaConfigs.ISOLATION_LEVEL.key}' = 'Serializable')") + val log = DeltaLog.forTable(spark, dir.getCanonicalPath) + val id = AttributeReference("id", LongType)() + val p = AttributeReference("p", IntegerType)() + var loserCount = 0 + + /** Runs the two-read loser against a winner appending `id` in [start, end) to `winnerP`. */ + def run(start: Long, end: Long, winnerP: Int, dataSkipping: Boolean): Try[Long] = { + loserCount += 1 + withSQLConf(DeltaSQLConf.DELTA_CONFLICT_DETECTION_DATA_SKIPPING_ENABLED.key -> + dataSkipping.toString) { + val loser = log.startTransaction() + loser.filterFiles(Seq(EqualTo(p, Literal(1)), LessThan(id, Literal(50L)))) + loser.filterFiles(Seq(EqualTo(p, Literal(2)), GreaterThanOrEqual(id, Literal(100L)))) + appendTxn(dir, start, end, partition = Some(winnerP))() + Try(loser.commit( + Seq(manufacturedAdd(s"loser-$loserCount.parquet", Map("p" -> "1"))), + DeltaOperations.Write(SaveMode.Append))) + } + } + + def assertConflict(result: Try[Long]): Unit = assert( + result.failed.toOption.exists( + _.isInstanceOf[io.delta.exceptions.ConcurrentAppendException]), + s"Expected ConcurrentAppendException, got: $result") + + // Cross terms (P1 AND D2) and (P2 AND D1): no read matches -> no conflict. + val usageLogs = Log4jUsageLogger.track { + assert(run(150, 151, winnerP = 1, dataSkipping = true).isSuccess) + assert(run(10, 11, winnerP = 2, dataSkipping = true).isSuccess) + } + assert(filterUsageRecords(usageLogs, noConflictEvent).size == 2) + // A file matching one of the reads still conflicts, and records no noConflict event. + val conflictLogs = Log4jUsageLogger.track { + assertConflict(run(10, 11, winnerP = 1, dataSkipping = true)) + assertConflict(run(150, 151, winnerP = 2, dataSkipping = true)) + } + assert(filterUsageRecords(conflictLogs, noConflictEvent).isEmpty) + // Feature disabled: partition pruning alone keeps the file -> conflict. + assertConflict(run(150, 151, winnerP = 1, dataSkipping = false)) + } + } } From 536ae8a9a3c6d63e50ffa752f6fa0b71b8dbe70f Mon Sep 17 00:00:00 2001 From: Eunjin Song Date: Thu, 15 Oct 2026 09:24:01 -0700 Subject: [PATCH 28/29] Test the conflict-time data skipping evaluation fallback Co-Authored-By: Claude Opus 5.5 --- .../sql/delta/ConflictDataSkippingSuite.scala | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/ConflictDataSkippingSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/ConflictDataSkippingSuite.scala index 9786920e58f..09d34ff76cd 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/ConflictDataSkippingSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/ConflictDataSkippingSuite.scala @@ -374,4 +374,33 @@ class ConflictDataSkippingSuite extends QueryTest assertConflict(run(150, 151, winnerP = 1, dataSkipping = false)) } } + + test("skipping evaluation failure falls back to treating added files as conflicts") { + // `id < 'abc'` builds a skipping predicate over the long `id` stats that fails once evaluated + // (the string does not cast to a long). The loser records it with stats skipping off so its own + // scan does not fail, then commits with skipping on. + withTempDir { dir => + createTable(dir) + val log = DeltaLog.forTable(spark, dir.getCanonicalPath) + val id = AttributeReference("id", LongType)() + val loser = withSQLConf(DeltaSQLConf.DELTA_STATS_SKIPPING.key -> "false") { + val txn = log.startTransaction() + txn.filterFiles(Seq(LessThan(id, Literal("abc")))) + txn + } + // A stats-bearing winner file that the predicate cannot match if evaluated. + appendTxn(dir, 1000, 1100)() + val usageLogs = Log4jUsageLogger.track { + withSQLConf(DeltaSQLConf.DELTA_CONFLICT_DETECTION_DATA_SKIPPING_ENABLED.key -> "true") { + intercept[io.delta.exceptions.ConcurrentAppendException] { + loser.commit( + Seq(manufacturedAdd("loser.parquet")), DeltaOperations.Write(SaveMode.Append)) + } + } + } + val fallbackEvent = "delta.conflictDetection.dataSkipping.fallback" + assert(filterUsageRecords(usageLogs, fallbackEvent).size == 1) + assert(filterUsageRecords(usageLogs, noConflictEvent).isEmpty) + } + } } From 9f7735fbb4c32050e6711e5beffbf844232a0ecc Mon Sep 17 00:00:00 2001 From: Eunjin Song Date: Thu, 15 Oct 2026 15:34:03 -0700 Subject: [PATCH 29/29] Address review: batch conflict-time skipping, skip toDF when unusable, report on commit - Check whether a skipping condition can be built before constructing any DataFrame; an unpartitioned check with nothing to skip returns files.headOption directly. - Evaluate candidates in bounded batches (internal conf conflictDetection.dataSkipping.batchSize, default 1000) and stop at the first batch holding a conflicting file: a filter over a local relation is evaluated over every row before the limit applies. - Group reads by their partition predicate after widening. - Report delta.conflictDetection.dataSkipping.applied once, from OptimisticTransaction, when the transaction commits. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Eunjin Song --- .../spark/sql/delta/ConflictChecker.scala | 109 +++++++++++------- .../sql/delta/OptimisticTransaction.scala | 12 ++ .../sql/delta/sources/DeltaSQLConf.scala | 11 ++ .../stats/ConflictDataSkippingReader.scala | 50 +++++--- .../sql/delta/ConflictDataSkippingSuite.scala | 40 ++++++- 5 files changed, 157 insertions(+), 65 deletions(-) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/ConflictChecker.scala b/spark/src/main/scala/org/apache/spark/sql/delta/ConflictChecker.scala index 52f0621c50a..959a2f5c9b6 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/ConflictChecker.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/ConflictChecker.scala @@ -287,11 +287,14 @@ private[delta] class ConflictChecker( protected def recordSkippedPhase(phase: String): Unit = timingStats += phase -> 0 + /** Set when conflict-time data skipping was evaluated by a checked-appends probe. */ + private var dataSkippingApplied = false + /** - * Set when conflict-time data skipping was applied to a checked-appends probe that then found no - * conflicting file. Reported once the whole check against the winning commit has passed. + * Whether conflict-time data skipping was applied while checking against the winning commit. + * Read by the transaction, which reports it once the commit succeeds. */ - private var dataSkippingAppliedWithoutConflict = false + private[delta] def conflictDataSkippingApplied: Boolean = dataSkippingApplied /** * This function checks conflict of the `initialCurrentTransactionInfo` against the @@ -365,12 +368,6 @@ private[delta] class ConflictChecker( checkForDeletedFilesAgainstCurrentTxnDeletedFiles() resolveTimestampOrderingConflicts() - if (dataSkippingAppliedWithoutConflict) { - recordDeltaEvent(deltaLog, - opType = "delta.conflictDetection.dataSkipping.noConflict", - data = Map("winningCommitVersion" -> winningCommitVersion)) - } - // Fold the winning commit into our transaction info before the next attempt: this advances // `preCommitLatestAMTCheckpointOpt` to the winner's inline AMT checkpoint (when the winner // wrote one) and clears `currentCommitAttemptAMTCheckpointOpt`. @@ -1078,33 +1075,33 @@ private[delta] class ConflictChecker( !currentTransactionInfo.readWholeTable && currentTransactionInfo.readPredicates.nonEmpty - import org.apache.spark.sql.delta.implicits._ + val candidates = new CandidateFiles( + files, spark.conf.get(DeltaSQLConf.DELTA_CONFLICT_DETECTION_DATA_SKIPPING_BATCH_SIZE)) // There is no reason to filter files if the table is not partitioned. if (currentTransactionInfo.readWholeTable || currentTransactionInfo.readSnapshot.metadata.partitionColumns.isEmpty) { if (!useDataSkipping || files.isEmpty) return files.headOption return getFirstFileMatchingDataPredicates( - files.toDF(spark), currentTransactionInfo.readPredicates.map(_.dataPredicates)) + candidates, currentTransactionInfo.readPredicates.map(_.dataPredicates), + partitionFilter = None) } - val filesDf = files.toDF(spark) - spark.conf.get(DeltaSQLConf.DELTA_CONFLICT_DETECTION_WIDEN_NONDETERMINISTIC_PREDICATES) match { case DeltaSQLConf.NonDeterministicPredicateWidening.OFF => getFirstFileMatchingPartitionPredicatesInternal( - filesDf, shouldWidenNonDeterministicPredicates = false, shouldWidenAllUdf = false, + candidates, shouldWidenNonDeterministicPredicates = false, shouldWidenAllUdf = false, useDataSkipping) case wideningMode => val fileWithWidening = getFirstFileMatchingPartitionPredicatesInternal( - filesDf, shouldWidenNonDeterministicPredicates = true, shouldWidenAllUdf = true, + candidates, shouldWidenNonDeterministicPredicates = true, shouldWidenAllUdf = true, useDataSkipping) fileWithWidening.flatMap { fileWithWidening => val fileWithoutWidening = getFirstFileMatchingPartitionPredicatesInternal( - filesDf, shouldWidenNonDeterministicPredicates = false, shouldWidenAllUdf = false, - useDataSkipping) + candidates, shouldWidenNonDeterministicPredicates = false, + shouldWidenAllUdf = false, useDataSkipping) if (fileWithoutWidening.isEmpty) { // Conflict due to widening of non-deterministic predicate. recordDeltaEvent(deltaLog, @@ -1128,7 +1125,7 @@ private[delta] class ConflictChecker( } private def getFirstFileMatchingPartitionPredicatesInternal( - filesDf: DataFrame, + candidates: CandidateFiles, shouldWidenNonDeterministicPredicates: Boolean, shouldWidenAllUdf: Boolean, useDataSkipping: Boolean): Option[AddFile] = { @@ -1164,22 +1161,23 @@ private[delta] class ConflictChecker( // With data skipping, each probe is a partition predicate plus the data predicates of the reads // that share it, so a read's data predicates are only combined with that same read's partition // predicate: a file conflicts iff it could match (P_i AND D_i) for some read i, not the looser - // (P_1 OR P_2) AND (D_1 OR D_2). Like ExpressionSet, non-deterministic predicates are never - // merged. + // (P_1 OR P_2) AND (D_1 OR D_2). Reads are grouped by their partition predicate after widening, + // so predicates that widen to the same one share a probe. Like ExpressionSet, non-deterministic + // predicates are never merged. val probes: Seq[(DeltaTableReadPredicate, Seq[Seq[Expression]])] = if (useDataSkipping) { val groups = mutable.LinkedHashMap[Any, (DeltaTableReadPredicate, mutable.Buffer[Seq[Expression]])]() currentTransactionInfo.readPredicates.zipWithIndex.foreach { case (rp, i) => - val key = if (rp.partitionPredicate.deterministic) { - (rp.shouldRewriteFilter, rp.partitionPredicate.canonicalized) + val widened = rewritePredicateFn(rp.partitionPredicate, rp.shouldRewriteFilter) + val key = if (widened.partitionPredicate.deterministic) { + (widened.shouldRewriteFilter, widened.partitionPredicate.canonicalized) } else { i } - groups.getOrElseUpdate(key, (rp, mutable.ArrayBuffer.empty))._2 += rp.dataPredicates + groups.getOrElseUpdate(key, (widened, mutable.ArrayBuffer.empty))._2 += rp.dataPredicates } - groups.values.toSeq.map { case (rp, dataFiltersPerRead) => - rewritePredicateFn(rp.partitionPredicate, rp.shouldRewriteFilter) -> - dataFiltersPerRead.toSeq + groups.values.toSeq.map { case (widened, dataFiltersPerRead) => + widened -> dataFiltersPerRead.toSeq } } else { canonicalPredicates.toSeq.map(_ -> Seq.empty[Seq[Expression]]) @@ -1188,16 +1186,17 @@ private[delta] class ConflictChecker( import org.apache.spark.sql.delta.implicits._ val filesMatchingPartitionPredicates = probes.iterator .flatMap { case (readPredicate, dataFiltersPerRead) => - val filesMatchingPartition = DeltaLog.filterFileList( + def filterByPartition(filesDf: DataFrame): DataFrame = DeltaLog.filterFileList( partitionSchema = currentTransactionInfo.partitionSchemaAtReadTime, files = filesDf, partitionFilters = readPredicate.partitionPredicates, shouldRewritePartitionFilters = readPredicate.shouldRewriteFilter ) val matchingFileOpt = if (useDataSkipping) { - getFirstFileMatchingDataPredicates(filesMatchingPartition, dataFiltersPerRead) + getFirstFileMatchingDataPredicates( + candidates, dataFiltersPerRead, partitionFilter = Some(filterByPartition)) } else { - filesMatchingPartition.as[AddFile].head(1).headOption + filterByPartition(candidates.all).as[AddFile].head(1).headOption } matchingFileOpt.foreach { f => logInfo(log"Partition predicate is matching a file changed by the winning transaction: " + @@ -1212,28 +1211,52 @@ private[delta] class ConflictChecker( } /** - * Returns the first file in `filesDf` whose column stats do not prove it fails every read in - * `dataFiltersPerRead`, i.e. a file that must still be treated as a conflict. The skipping filter - * is fused into the same `head(1)` job as the caller's partition filter, which is applied first. + * The concurrently-added files to check, as the DataFrames the probes need. Built lazily on first + * use and shared by every probe. + */ + private class CandidateFiles(val files: Seq[AddFile], batchSize: Int) { + import org.apache.spark.sql.delta.implicits._ + lazy val all: DataFrame = files.toDF(spark) + // The files form a local relation, whose filters the optimizer evaluates over every row before + // applying a limit, so `head(1)` alone does not stop early. Bounded batches let a probe stop at + // the first batch that holds a conflicting file. + lazy val batches: LazyList[DataFrame] = + files.grouped(batchSize).map(_.toDF(spark)).to(LazyList) + } + + /** + * Returns the first candidate file that passes `partitionFilter` and whose column stats do not + * prove it fails every read in `dataFiltersPerRead`, i.e. a file that must still be treated as a + * conflict. The partition and skipping filters are fused into the same query, run over bounded + * batches of candidates until one holds a match. When nothing can be skipped, no stats are parsed + * and an unpartitioned check (`partitionFilter = None`) runs no query at all. * * Skipping is a pure optimization over conservative conflict detection: if building or * evaluating the skipping filter fails, this records an event and falls back to checking without * skipping, exactly as when the feature is disabled. */ private def getFirstFileMatchingDataPredicates( - filesDf: DataFrame, - dataFiltersPerRead: Seq[Seq[Expression]]): Option[AddFile] = { + candidates: CandidateFiles, + dataFiltersPerRead: Seq[Seq[Expression]], + partitionFilter: Option[DataFrame => DataFrame]): Option[AddFile] = { import org.apache.spark.sql.delta.implicits._ def firstFile(df: DataFrame): Option[AddFile] = df.as[AddFile].head(1).headOption + def firstFileWithoutSkipping: Option[AddFile] = partitionFilter match { + case Some(filter) => firstFile(filter(candidates.all)) + case None => candidates.files.headOption + } + val readSnapshot = currentTransactionInfo.readSnapshot try { - currentTransactionInfo.readSnapshot - .filterFilesDfMatchingAnyReadPredicate(filesDf, dataFiltersPerRead) match { - case Some(skippedDf) => - val matchingFileOpt = firstFile(skippedDf) - if (matchingFileOpt.isEmpty) dataSkippingAppliedWithoutConflict = true - matchingFileOpt - case None => firstFile(filesDf) - } + readSnapshot.conflictSkippingCondition(dataFiltersPerRead) match { + case None => firstFileWithoutSkipping + case Some(condition) => + val matchingFileOpt = candidates.batches.iterator.map { batch => + val filtered = partitionFilter.map(filter => filter(batch)).getOrElse(batch) + firstFile(readSnapshot.applyConflictSkippingCondition(filtered, condition)) + }.collectFirst { case Some(file) => file } + dataSkippingApplied = true + matchingFileOpt + } } catch { case NonFatal(e) => logWarning(log"Conflict-time data skipping failed; falling back to treating all added " + @@ -1244,7 +1267,7 @@ private[delta] class ConflictChecker( "winningCommitVersion" -> winningCommitVersion, "exceptionClass" -> e.getClass.getName, "exceptionMessage" -> e.getMessage)) - firstFile(filesDf) + firstFileWithoutSkipping } } diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/OptimisticTransaction.scala b/spark/src/main/scala/org/apache/spark/sql/delta/OptimisticTransaction.scala index fbe1210005f..4bcb590ee2a 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/OptimisticTransaction.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/OptimisticTransaction.scala @@ -483,6 +483,12 @@ trait OptimisticTransactionImpl extends TransactionHelper */ protected var commitAttemptStartTimeMillis: Long = _ + /** + * Whether conflict-time data skipping was applied while checking this transaction against any + * winning commit, across all commit attempts. Reported once the transaction commits. + */ + private var conflictDataSkippingApplied = false + /** * Tracks actions within the transaction, will commit along with the passed-in actions in the * commit function. @@ -3609,6 +3615,7 @@ trait OptimisticTransactionImpl extends TransactionHelper winningCommitSummary, firstWinningVersion, updatedCurrentTransactionInfo).getOrElse { conflictChecker.checkConflictsAndValidateActions() } + conflictDataSkippingApplied ||= conflictChecker.conflictDataSkippingApplied logInfo(logPrefix + log"No conflicts in version ${MDC(DeltaLogKeys.VERSION, otherCommitVersion)}, " + @@ -3733,6 +3740,11 @@ trait OptimisticTransactionImpl extends TransactionHelper committedTransactionInfoOpt.map(_.finalActionsToCommit).getOrElse(Seq.empty) val catalogTableForCommittedTransaction = committedTransactionInfoOpt.flatMap(_.catalogTable).orElse(catalogTable) + if (conflictDataSkippingApplied) { + recordDeltaEvent(deltaLog, + opType = "delta.conflictDetection.dataSkipping.applied", + data = Map("committedVersion" -> committedVersion)) + } committed = Some(CommittedTransaction( txnId = txnId, deltaLog = deltaLog, diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala b/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala index 855c3278a8b..41faac651bf 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala @@ -618,6 +618,17 @@ trait DeltaSQLConfBase extends DeltaSQLConfUtils { .booleanConf .createWithDefault(false) + val DELTA_CONFLICT_DETECTION_DATA_SKIPPING_BATCH_SIZE = + buildConf("conflictDetection.dataSkipping.batchSize") + .internal() + .doc( + """Number of concurrently-added files evaluated per batch by conflict-time data skipping. + |Each batch's stats are parsed and filtered in full, so smaller batches stop sooner once a + |conflicting file is found.""".stripMargin) + .intConf + .checkValue(_ > 0, "must be positive") + .createWithDefault(1000) + val DELTA_PROTOCOL_DEFAULT_WRITER_VERSION = buildConf("properties.defaults.minWriterVersion") .doc("The default writer protocol version to create new tables with, unless a feature " + diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/stats/ConflictDataSkippingReader.scala b/spark/src/main/scala/org/apache/spark/sql/delta/stats/ConflictDataSkippingReader.scala index c7e84fda0bb..49f25d619ab 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/stats/ConflictDataSkippingReader.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/stats/ConflictDataSkippingReader.scala @@ -130,23 +130,29 @@ trait ConflictDataSkippingReader extends DeltaLogging { self: DataSkippingReader /** * Lazily restricts `filesDf` (a DataFrame of [[AddFile]]s with JSON `stats`) to the files that - * could still match ANY of several INDEPENDENT reads. Each inner `Seq[Expression]` is one logical - * read's data filters (AND-combined via [[buildDataSkippingPredicate]]); the reads are - * OR-combined, matching read semantics -- a file is a candidate if it could match any one of - * them. Returns None when nothing can be skipped: no reads, a table without stats, or a read - * with no usable skipping predicate (empty / ineligible filters match every file). - * - * No job is launched here, so callers can fuse the filter into their own query, e.g. after a - * partition filter so stats are only parsed for files in matching partitions. The returned - * DataFrame keeps the input's columns, with `stats` still the original JSON string. + * could still match ANY of several INDEPENDENT reads, or returns None when nothing can be + * skipped. Combines [[conflictSkippingCondition]] and [[applyConflictSkippingCondition]]. + */ + private[delta] def filterFilesDfMatchingAnyReadPredicate( + filesDf: DataFrame, + dataFiltersPerRead: Seq[Seq[Expression]]): Option[DataFrame] = + conflictSkippingCondition(dataFiltersPerRead).map(applyConflictSkippingCondition(filesDf, _)) + + /** + * Builds the condition a file must satisfy to possibly match ANY of several INDEPENDENT reads. + * Each inner `Seq[Expression]` is one logical read's data filters (AND-combined via + * [[buildDataSkippingPredicate]]); the reads are OR-combined, matching read semantics -- a file + * is a candidate if it could match any one of them. Returns None when nothing can be skipped: no + * reads, a table without stats, or a read with no usable skipping predicate (empty / ineligible + * filters match every file). Needs no DataFrame, so callers can check whether skipping applies + * before building one. * * One-way safe: each read contributes `expr || !verifyStatsForFilter(...)` exactly as * [[getDataSkippedFiles]], so a file is dropped only when its stats *prove* it fails EVERY read * -- a real conflict is never a false negative. Files with missing/insufficient stats are kept. */ - private[delta] def filterFilesDfMatchingAnyReadPredicate( - filesDf: DataFrame, - dataFiltersPerRead: Seq[Seq[Expression]]): Option[DataFrame] = { + private[delta] def conflictSkippingCondition( + dataFiltersPerRead: Seq[Seq[Expression]]): Option[Column] = { if (dataFiltersPerRead.isEmpty || schema.isEmpty) return None val perReadPredicates = dataFiltersPerRead.map { dataFilters => if (dataFilters.isEmpty) None else buildDataSkippingPredicate(dataFilters) @@ -154,17 +160,27 @@ trait ConflictDataSkippingReader extends DeltaLogging { self: DataSkippingReader if (perReadPredicates.exists(_.isEmpty)) return None // Per read, `expr || !verifyStatsForFilter(...)` keeps any file whose referenced stats are // missing/NULL (mirrors getDataSkippedFiles); OR the reads so a match against any one keeps it. - val survivorCondition = perReadPredicates.flatten + Some(perReadPredicates.flatten .map(pred => pred.expr || !verifyStatsForFilter(pred.referencedStats)) - .reduce(_ || _) + .reduce(_ || _)) + } + + /** + * Lazily applies a [[conflictSkippingCondition]] to `filesDf`. No job is launched here, so + * callers can fuse the filter into their own query, e.g. with a partition filter. The returned + * DataFrame keeps the input's columns, with `stats` still the original JSON string. + */ + private[delta] def applyConflictSkippingCondition( + filesDf: DataFrame, + condition: Column): DataFrame = { // The skipping expressions reference the parsed `stats` struct; keep the raw JSON aside so the // result still has the AddFile shape. val rawStatsCol = "__conflict_skipping_raw_stats" - Some(filesDf + filesDf .withColumn(rawStatsCol, col("stats")) .withColumn("stats", parseAndDecodeStats(col("stats"))) - .where(survivorCondition) + .where(condition) .withColumn("stats", col(rawStatsCol)) - .drop(rawStatsCol)) + .drop(rawStatsCol) } } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/ConflictDataSkippingSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/ConflictDataSkippingSuite.scala index 09d34ff76cd..8a0ea046d63 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/ConflictDataSkippingSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/ConflictDataSkippingSuite.scala @@ -180,7 +180,7 @@ class ConflictDataSkippingSuite extends QueryTest name: String, partitionValues: Map[String, String] = Map.empty): AddFile = AddFile(name, partitionValues, size = 1L, modificationTime = 1L, dataChange = true) - private val noConflictEvent = "delta.conflictDetection.dataSkipping.noConflict" + private val appliedEvent = "delta.conflictDetection.dataSkipping.applied" test("empty read predicates on a non-blind-append txn: added file still conflicts") { // Soundness of the Serializable blind-append case. A transaction can be a NON-blind append @@ -363,13 +363,13 @@ class ConflictDataSkippingSuite extends QueryTest assert(run(150, 151, winnerP = 1, dataSkipping = true).isSuccess) assert(run(10, 11, winnerP = 2, dataSkipping = true).isSuccess) } - assert(filterUsageRecords(usageLogs, noConflictEvent).size == 2) - // A file matching one of the reads still conflicts, and records no noConflict event. + assert(filterUsageRecords(usageLogs, appliedEvent).size == 2) + // A file matching one of the reads still conflicts, and records no applied event. val conflictLogs = Log4jUsageLogger.track { assertConflict(run(10, 11, winnerP = 1, dataSkipping = true)) assertConflict(run(150, 151, winnerP = 2, dataSkipping = true)) } - assert(filterUsageRecords(conflictLogs, noConflictEvent).isEmpty) + assert(filterUsageRecords(conflictLogs, appliedEvent).isEmpty) // Feature disabled: partition pruning alone keeps the file -> conflict. assertConflict(run(150, 151, winnerP = 1, dataSkipping = false)) } @@ -400,7 +400,37 @@ class ConflictDataSkippingSuite extends QueryTest } val fallbackEvent = "delta.conflictDetection.dataSkipping.fallback" assert(filterUsageRecords(usageLogs, fallbackEvent).size == 1) - assert(filterUsageRecords(usageLogs, noConflictEvent).isEmpty) + assert(filterUsageRecords(usageLogs, appliedEvent).isEmpty) + } + } + + test("skipping evaluates candidates in batches and finds a conflict in a later batch") { + withTempDir { dir => + createTable(dir) + val log = DeltaLog.forTable(spark, dir.getCanonicalPath) + val id = AttributeReference("id", LongType)() + // Winner adds four files: three whose stats prove they miss `id < 50`, then the [0,100) file, + // which matches. Reusing the table's files gives them real stats. + val (matching, missing) = log.update().allFiles.collect().toSeq + .partition(_.stats.contains("\"minValues\":{\"id\":0}")) + val winnerAdds = missing.take(3) ++ matching + assert(winnerAdds.size == 4) + + def run(adds: Seq[AddFile]): scala.util.Try[Long] = withSQLConf( + DeltaSQLConf.DELTA_CONFLICT_DETECTION_DATA_SKIPPING_ENABLED.key -> "true", + DeltaSQLConf.DELTA_CONFLICT_DETECTION_DATA_SKIPPING_BATCH_SIZE.key -> "1") { + val loser = log.startTransaction() + loser.filterFiles(Seq(LessThan(id, Literal(50L)))) + log.startTransaction().commit(adds, DeltaOperations.Write(SaveMode.Append)) + scala.util.Try(loser.commit( + Seq(manufacturedAdd(s"loser-${adds.size}.parquet")), + DeltaOperations.Write(SaveMode.Append))) + } + // Only the three non-matching files: every batch is skipped, no conflict. + assert(run(winnerAdds.take(3).map(f => f.copy(path = "a-" + f.path))).isSuccess) + // The matching file is the fourth batch: still found. + assert(run(winnerAdds.map(f => f.copy(path = "b-" + f.path))).failed.toOption + .exists(_.isInstanceOf[io.delta.exceptions.ConcurrentAppendException])) } } }