diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 00000000000..3f6d2caa7a1 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,20 @@ +{ + "name": "Delta Lake Dev", + "image": "mcr.microsoft.com/devcontainers/java:17-bookworm", + "features": { + "ghcr.io/devcontainers/features/python:1": { + "version": "3.10" + } + }, + "customizations": { + "vscode": { + "extensions": [ + "scalameta.metals" + ], + "settings": { + "metals.sbtScript": "${containerWorkspaceFolder}/build/sbt" + } + } + }, + "postCreateCommand": "echo 'Dev container ready. Run: build/sbt compile'" +} diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 4b5435c3192..b98d398fa23 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -39,12 +39,12 @@ jobs: key: delta-sbt-cache-cross-spark-${{ hashFiles('project/scripts/setup_unitycatalog_main.sh') }} # publishM2 compiles every aggregated project, including storage, which has - # unitycatalog-client as a compile-scope dependency. test_cross_spark_publish.py also + # unitycatalog-client as a compile-scope dependency. test_published_artifacts.py also # iterates over released Spark versions (sbt -DsparkVersion=), so we need UC's # spark connector published for each variant Delta will resolve. Discover the version list # at runtime from project/spark-versions.json (same source the matrix workflows use) and # invoke the setup script per variant; cache hits (the steady state) make the script a - # no-op. Snapshot versions are skipped by the cross-Spark test, so only released ones are + # no-op. Snapshot versions are skipped by the published-artifact test, so only released ones are # published here. - name: Set up pinned Unity Catalog for each released Spark version run: | @@ -54,8 +54,8 @@ jobs: echo "::endgroup::" done - - name: Run cross-Spark build test - run: python project/tests/test_cross_spark_publish.py + - name: Test published artifacts + run: python project/tests/test_published_artifacts.py - name: Save SBT cache if: github.event_name == 'push' && github.ref == 'refs/heads/master' diff --git a/.github/workflows/flink_test.yaml b/.github/workflows/flink_test.yaml index 7055789c7e9..10f23209730 100644 --- a/.github/workflows/flink_test.yaml +++ b/.github/workflows/flink_test.yaml @@ -6,6 +6,9 @@ on: paths: - 'flink/**' - 'kernel/**' + - 'build.sbt' + - 'project/CrossFlinkVersions.scala' + - '.github/workflows/flink_test.yaml' - '!**/*.md' - '!**/*.txt' pull_request: @@ -13,6 +16,9 @@ on: paths: - 'flink/**' - 'kernel/**' + - 'build.sbt' + - 'project/CrossFlinkVersions.scala' + - '.github/workflows/flink_test.yaml' - '!**/*.md' - '!**/*.txt' @@ -26,9 +32,34 @@ env: SBT_OPTS: "-Dsbt.coursier.home-dir=/home/runner/.cache/coursier -Dsbt.ivy.home=/home/runner/.ivy2" jobs: + generate-matrix: + name: "Generate Flink versions matrix" + runs-on: ubuntu-24.04 + outputs: + flink-versions: ${{ steps.versions.outputs.flink-versions }} + steps: + - name: Checkout code + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + - name: Install Java + uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4.8.0 + with: + distribution: "zulu" + java-version: "17" + - name: Generate matrix + id: versions + run: | + build/sbt exportFlinkVersionsJson + versions="$(jq -c '[.[].fullVersion]' target/flink-versions.json)" + echo "flink-versions=$versions" >> "$GITHUB_OUTPUT" + test: - name: "DF" + name: "DF: Flink ${{ matrix.flink }}" + needs: generate-matrix runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + flink: ${{ fromJson(needs.generate-matrix.outputs.flink-versions) }} steps: - name: Show runner specs run: | @@ -56,7 +87,7 @@ jobs: ~/.sbt ~/.ivy2 ~/.cache/coursier - key: sbt-flink + key: sbt-flink-${{ matrix.flink }} - name: Check cache status run: | if [ "${{ steps.cache-sbt.outputs.cache-hit }}" == "true" ]; then @@ -70,7 +101,7 @@ jobs: uses: ./.github/actions/setup-unitycatalog - name: Run unit tests run: | - build/sbt flinkGroup/test + build/sbt -DflinkVersion=${{ matrix.flink }} flinkGroup/test - name: Save SBT cache if: github.event_name == 'push' && github.ref == 'refs/heads/master' uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 @@ -79,4 +110,4 @@ jobs: ~/.sbt ~/.ivy2 ~/.cache/coursier - key: sbt-flink + key: sbt-flink-${{ matrix.flink }} diff --git a/PROTOCOL.md b/PROTOCOL.md index bdd7f52a411..4bb4e2f347a 100644 --- a/PROTOCOL.md +++ b/PROTOCOL.md @@ -110,6 +110,7 @@ - [Generated Columns](#generated-columns) - [Default Columns](#default-columns) - [Identity Columns](#identity-columns) + - [Materialize Partition Columns](#materialize-partition-columns) - [Writer Version Requirements](#writer-version-requirements) - [Requirements for Readers](#requirements-for-readers) - [Reader Version Requirements](#reader-version-requirements) @@ -621,7 +622,7 @@ Field Name | Data Type | Description | optional/required path| String | A relative path to a file from the root of the table or an absolute path to a file that should be removed from the table. The path is a URI as specified by [RFC 2396 URI Generic Syntax](https://www.ietf.org/rfc/rfc2396.txt), which needs to be decoded to get the data file path. | required deletionTimestamp | Option[Long] | The time the deletion occurred, represented as milliseconds since the epoch | optional dataChange | Boolean | When `false` the records in the removed file must be contained in one or more `add` file actions in the same version | required -extendedFileMetadata | Boolean | When `true` the fields `partitionValues`, `size`, and `tags` are present | optional +extendedFileMetadata | Boolean | When `true` the fields `partitionValues` and `size` are present | optional partitionValues| Map[String, String] | A map from partition column to value for this file. See also [Partition Value Serialization](#Partition-Value-Serialization) | optional size| Long | The size of this data file in bytes | optional stats | [Statistics Struct](#Per-file-Statistics) | Contains statistics (e.g., count, min/max values for columns) about the data in this logical file | optional @@ -1836,7 +1837,7 @@ The following is an example for the `domainMetadata` action definition of a tabl { "domainMetadata": { "domain": "delta.clustering", - "configuration": "{\"clusteringColumns\":[\"col-daadafd7-7c20-4697-98f8-bff70199b1f9\", \"col-5abe0e80-cf57-47ac-9ffc-a861a3d1077e\"]}", + "configuration": "{\"clusteringColumns\":[[\"col-daadafd7-7c20-4697-98f8-bff70199b1f9\"], [\"col-5abe0e80-cf57-47ac-9ffc-a861a3d1077e\"]]}", "removed": false } } @@ -1845,11 +1846,12 @@ The example above converts `configuration` field into JSON format, including esc ```json { "clusteringColumns": [ - "col-daadafd7-7c20-4697-98f8-bff70199b1f9", - "col-5abe0e80-cf57-47ac-9ffc-a861a3d1077e" + ["col-daadafd7-7c20-4697-98f8-bff70199b1f9"], + ["col-5abe0e80-cf57-47ac-9ffc-a861a3d1077e"] ] } ``` +Each entry in `clusteringColumns` is the name path of a clustering column: a single segment for a top-level column, and multiple segments for a nested column (for example, `["user", "address", "city"]`). If [Column Mapping](#column-mapping) is enabled, physical names are used for each segment. # Variant Data Type @@ -2490,6 +2492,34 @@ When `delta.identity.allowExplicitInsert` is false, writers should meet the foll - Overflow when calculating generated Identity values should be detected and such writes should not be allowed. - `delta.identity.highWaterMark` should be updated to the new highest value when the write operation commits. +## Materialize Partition Columns + +When this feature is supported, partition columns are physically written to Parquet files alongside the data columns. To support this feature: + - The table must be on Writer Version 7, and a feature name `materializePartitionColumns` must exist in the table `protocol`'s `writerFeatures`. + +Unlike most writer features, `materializePartitionColumns` has no associated `delta.enable*` table property and defines no additional metadata requirements. It is therefore [active](#active-features) whenever it is [supported](#supported-features): its presence in the `protocol`'s `writerFeatures` alone forces the writer requirements below. Hence, for this feature, the terms *supported*, *enabled*, and *active* (including in the decision matrix below) all refer to the same state. + +When supported: + - When the writer feature `materializePartitionColumns` is set in the protocol, writers must materialize partition columns into every newly added Parquet data file referenced by an `AddFile` action. Each materialized partition column value must equal the corresponding logical partition value recorded in that file's `AddFile` `partitionValues`. This mimics the same partition column materialization requirement from [IcebergCompatV1](#iceberg-compatibility-v1) and [IcebergCompatV2](#iceberg-compatibility-v2). As such, the `materializePartitionColumns` feature can be seen as a subset of the requirements imposed by those features, providing the partition column materialization guarantee independently without requiring full Iceberg compatibility. + - When the writer feature `materializePartitionColumns` is not set in the table protocol, writers are not required to write partition columns to data files. Note that other features might still require materialization of partition values, such as [IcebergCompatV1](#iceberg-compatibility-v1). + +When [Column Mapping](#column-mapping) is enabled, materialized partition columns are written to the Parquet data file using their assigned physical column names and field IDs, the same as data columns. + +This feature does not impose any requirements on readers. All Delta readers must be able to read the table regardless of whether partition columns are materialized in the data files. If partition values are present in both parquet and AddFile metadata, Delta readers should continue to read partition values from AddFile metadata. Also, [file-level statistics](#per-file-statistics) should not be written for the partition column as it would repeat information already present in an AddFile's `partitionValues`. + +Note that this table feature, as well as [IcebergCompatV1](#iceberg-compatibility-v1) (and related table features that require partition column materialization), if enabled, take priority over the `delta.writePartitionColumnsToParquet` table property. In other words, if a table feature is enabled that requires materialization of partition columns, and table metadata contains a `false` value for `delta.writePartitionColumnsToParquet`, partition columns must be materialized. + +| Table feature enablement | Value of `delta.writePartitionColumnsToParquet` table property | Writer requirement | +| ------------------------ | -------------------------------------------------------------- | ------------------ | +| A writer feature requiring partition column materialization (eg. `materializePartitionColumns`) is *enabled* | `false` | Partition columns *must* be materialized in parquet data files | +| A writer feature requiring partition column materialization (eg. `materializePartitionColumns`) is *enabled* | `true` | Partition columns *must* be materialized in parquet data files | +| A writer feature requiring partition column materialization (eg. `materializePartitionColumns`) is *enabled* | unset | Partition columns *must* be materialized in parquet data files | +| No writer feature requiring partition column materialization is enabled | `false` | Partition columns *should not* be materialized in parquet data files | +| No writer feature requiring partition column materialization is enabled | `true` | Partition columns *should* be materialized in parquet data files | +| No writer feature requiring partition column materialization is enabled | unset | No requirement on partition column materialization | + +The value of having both the table feature `materializePartitionColumns` and the table property `delta.writePartitionColumnsToParquet` supported is that not every table is going to need the heightened requirement of only allowing writes from writers that understand `materializePartitionColumns`. In other words, `materializePartitionColumns` imposes a writer compatibility edge that `delta.writePartitionColumnsToParquet` does not. + ## Writer Version Requirements The requirements of the writers according to the protocol versions are summarized in the table below. Each row inherits the requirements from the preceding row. @@ -2525,6 +2555,7 @@ Property | Description | Details `delta.parquet.compression.codec` | Compression codec writers SHOULD use for new Parquet data and checkpoint files. Changing this property does not affect existing files; a table may contain files written with different codecs, which is a normal and expected state. | Widely supported values (matched case-insensitively): `uncompressed`/`none` (no compression), `snappy`, `gzip`, `lz4` (deprecated, Hadoop framing), `lz4_raw` ([LZ4 block format](https://parquet.apache.org/docs/file-format/data-pages/compression/#lz4_raw)), `zstd`.

When absent, writers SHOULD default to `zstd`. If a writer does not support or recognize the specified codec, it SHOULD abort with an appropriate error or fall back to a default codec.

Readers SHOULD support all codecs listed above regardless of the current property value. Parquet files written with other [parquet-supported codecs](https://parquet.apache.org/docs/file-format/data-pages/compression/) may also exist; readers MAY support reading these files. `delta.parquet.format.version` | Parquet data page format writers SHOULD use for new data and checkpoint files. This property is a directive to writers only; readers do not need to consult it, as Parquet pages are self-describing via the `PageType` field in each page header. Changing this property does not affect existing files; a table MAY contain files written with different data page versions, which is a normal and expected state. | Valid values: `1.0.0` (DataPageV1) and `2.x.x` (DataPageV2, where `x.x` is any minor.patch version). Recommended values are `1.0.0` and `2.12.0`.

When absent, writers SHOULD default to `1.0.0`. Writers SHOULD validate this property and abort if the value does not match `1.0.0` or `2.MINOR.PATCH`.

Readers SHOULD support both DataPageV1 and DataPageV2 pages regardless of this property's value. Tables intended for access by engines beyond the Delta Lake connectors SHOULD use `1.0.0`, as DataPageV2 support varies across the broader Parquet ecosystem. `delta.enableVariantShredding` | When `true`, writers could write variant data to parquet files in [shredded](#variant-shredding) format. | Valid values: `true` (shredding allowed) and `false` (shredding not allowed).

When enabled, writers must ensure that the `variantShredding` table feature is present in the table `protocol`'s `readerFeatures` and `writerFeatures`. +`delta.writePartitionColumnsToParquet` | Controls whether writers SHOULD write partition columns in newly written data parquet files, in the absence of any writer features that necessitate writing of partition columns (eg. `IcebergCompatV1`). In other words, if no writer feature is enabled that requires materialization of partition columns, writers should read this property to decide whether to materialize partition columns in data parquet files or not. Writer features requirements take precedence over this property's value. Readers should continue to read partition values off of AddFile actions, regardless of the presence of partition values in data files. File-level statistics should not be present for partition columns in partitioned tables in any case. This setting does not apply to writers of files of any other file format. | Boolean field, with valid values `false` and `true`. # Appendix @@ -2551,6 +2582,7 @@ Feature | Name | Readers or Writers? [Clustered Table](#clustered-table) | `clustering` | Writers only [VACUUM Protocol Check](#vacuum-protocol-check) | `vacuumProtocolCheck` | Readers and Writers [In-Commit Timestamps](#in-commit-timestamps) | `inCommitTimestamp` | Writers only +[Materialize Partition Columns](#materialize-partition-columns) | `materializePartitionColumns` | Writers only ## Deletion Vector Format diff --git a/build.sbt b/build.sbt index 5321bcdd49b..d28ace29443 100644 --- a/build.sbt +++ b/build.sbt @@ -64,8 +64,15 @@ val scalaTestVersionForConnectors = "3.0.8" val parquet4sVersion = "1.9.4" val protoVersion = "3.25.1" val grpcVersion = "1.62.2" -val flinkVersion = "2.0.1" +val flinkVersion = CrossFlinkVersions.getFlinkArtifactVersion() val gcsConnectorVersion = "4.0.4" +val junit5Version = "5.14.0" +val junit5TestDependencies = Seq( + "org.junit.jupiter" % "junit-jupiter-api" % junit5Version % "test", + "org.junit.jupiter" % "junit-jupiter-engine" % junit5Version % "test", + "org.junit.jupiter" % "junit-jupiter-params" % junit5Version % "test", + "com.github.sbt.junit" % "jupiter-interface" % "0.16.0" % "test" +) // Optional kernel version override. See `project/KernelVersion.scala` for the // resolution rule and `-DkernelVersion=` semantics. @@ -513,16 +520,12 @@ lazy val sparkV2 = { exportJars := true, // Export as JAR to avoid classpath conflicts Test / javaOptions ++= Seq("-ea"), - libraryDependencies ++= Seq( + libraryDependencies ++= junit5TestDependencies ++ Seq( + "org.mockito" % "mockito-inline" % "4.11.0" % "test", "org.apache.spark" %% "spark-sql" % sparkArtifactVersion.value % "provided", "org.apache.spark" %% "spark-core" % sparkArtifactVersion.value % "provided", "org.apache.spark" %% "spark-catalyst" % sparkArtifactVersion.value % "provided", - // Test dependencies - "org.junit.jupiter" % "junit-jupiter-api" % "5.11.4" % "test", - "org.junit.jupiter" % "junit-jupiter-engine" % "5.11.4" % "test", - "org.junit.jupiter" % "junit-jupiter-params" % "5.11.4" % "test", - "com.github.sbt.junit" % "jupiter-interface" % "0.17.0" % "test", // Spark test classes for Scala/Java test utilities "org.apache.spark" %% "spark-catalyst" % sparkArtifactVersion.value % "test" classifier "tests", "org.apache.spark" %% "spark-core" % sparkArtifactVersion.value % "test" classifier "tests", @@ -674,7 +677,7 @@ lazy val spark = (project in file("spark-unified")) projectDependencies.value.filterNot(dep => internalModules.contains(dep.name)) }, - libraryDependencies ++= Seq( + libraryDependencies ++= junit5TestDependencies ++ Seq( "org.apache.spark" %% "spark-hive" % sparkArtifactVersion.value % "provided", "org.apache.spark" %% "spark-sql" % sparkArtifactVersion.value % "provided", "org.apache.spark" %% "spark-core" % sparkArtifactVersion.value % "provided", @@ -685,10 +688,6 @@ lazy val spark = (project in file("spark-unified")) "org.scalatestplus" %% "scalacheck-1-15" % "3.2.9.0" % "test", "junit" % "junit" % "4.13.2" % "test", "com.novocode" % "junit-interface" % "0.11" % "test", - "org.junit.jupiter" % "junit-jupiter-api" % "5.11.4" % "test", - "org.junit.jupiter" % "junit-jupiter-engine" % "5.11.4" % "test", - "org.junit.jupiter" % "junit-jupiter-params" % "5.11.4" % "test", - "com.github.sbt.junit" % "jupiter-interface" % "0.17.0" % "test", "org.apache.spark" %% "spark-catalyst" % sparkArtifactVersion.value % "test" classifier "tests", "org.apache.spark" %% "spark-core" % sparkArtifactVersion.value % "test" classifier "tests", "org.apache.spark" %% "spark-sql" % sparkArtifactVersion.value % "test" classifier "tests", @@ -945,13 +944,8 @@ lazy val sparkUnityCatalog = (project in file("spark/unitycatalog")) "com.fasterxml.jackson.datatype" % "jackson-datatype-jdk8" % sparkUnityCatalogJacksonVersion ), - libraryDependencies ++= Seq( + libraryDependencies ++= junit5TestDependencies ++ Seq( "org.assertj" % "assertj-core" % "3.26.3" % "test", - // JUnit 5 test dependencies - "org.junit.jupiter" % "junit-jupiter-api" % "5.11.4" % "test", - "org.junit.jupiter" % "junit-jupiter-engine" % "5.11.4" % "test", - "org.junit.jupiter" % "junit-jupiter-params" % "5.11.4" % "test", - "com.github.sbt.junit" % "jupiter-interface" % "0.17.0" % "test", // Lombok for generating boilerplate code "org.projectlombok" % "lombok" % "1.18.34" % "test", @@ -1053,7 +1047,7 @@ lazy val kernelApi = (project in file("kernel/kernel-api")) // can depend on test utilities via a published artifact instead of depending on raw class directories. Test / publishArtifact := true, Test / packageBin / artifactClassifier := Some("tests"), - libraryDependencies ++= Seq( + libraryDependencies ++= junit5TestDependencies ++ Seq( "org.roaringbitmap" % "RoaringBitmap" % "0.9.25", "org.slf4j" % "slf4j-api" % "1.7.36", @@ -1066,8 +1060,6 @@ lazy val kernelApi = (project in file("kernel/kernel-api")) "com.google.code.findbugs" % "jsr305" % "3.0.2", "org.scalatest" %% "scalatest" % scalaTestVersion % "test", - "junit" % "junit" % "4.13.2" % "test", - "com.novocode" % "junit-interface" % "0.11" % "test", "org.apache.logging.log4j" % "log4j-slf4j-impl" % "2.25.3" % "test", "org.apache.logging.log4j" % "log4j-core" % "2.25.3" % "test", "org.assertj" % "assertj-core" % "3.26.3" % "test", @@ -1400,14 +1392,14 @@ lazy val iceberg = (project in file("iceberg")) // Fix Iceberg's legacy java.lang.NoClassDefFoundError: scala/jdk/CollectionConverters$ error // due to legacy scala. "org.scala-lang.modules" %% "scala-collection-compat" % "2.1.1", - "com.github.ben-manes.caffeine" % "caffeine" % "2.9.3", + "com.github.ben-manes.caffeine" % "caffeine" % "3.2.4", "com.jolbox" % "bonecp" % "0.8.0.RELEASE" % "test", "org.eclipse.jetty" % "jetty-server" % "11.0.26" % "test", "org.eclipse.jetty" % "jetty-servlet" % "11.0.26" % "test", "org.xerial" % "sqlite-jdbc" % "3.45.0.0" % "test", - "org.apache.httpcomponents.core5" % "httpcore5" % "5.2.4" % "test", - "org.apache.httpcomponents.client5" % "httpclient5" % "5.3.1" % "test", - "org.apache.iceberg" %% icebergSparkRuntimeArtifactName % "1.11.0" % "provided", + "org.apache.httpcomponents.core5" % "httpcore5" % "5.4.3" % "test", + "org.apache.httpcomponents.client5" % "httpclient5" % "5.6.4" % "test", + "org.apache.iceberg" %% icebergSparkRuntimeArtifactName % "1.12.0" % "provided", // For FixedGcsAccessTokenProvider (GCS server-side planning credentials) "com.google.cloud.bigdataoss" % "util-hadoop" % gcsConnectorVersion % "provided" ) @@ -1438,7 +1430,7 @@ lazy val iceberg = (project in file("iceberg")) s"iceberg-shaded_${scalaBinaryVersion.value}-${version.value}.jar", s"scala-library-${scala213}.jar", s"scala-collection-compat_${scalaBinaryVersion.value}-2.1.1.jar", - "caffeine-2.9.3.jar", + "caffeine-3.2.4.jar", // Note: We are excluding // - antlr4-runtime-4.9.3.jar // - checker-qual-3.19.0.jar @@ -1454,6 +1446,8 @@ lazy val iceberg = (project in file("iceberg")) } }, assembly / assemblyMergeStrategy := { + case "module-info.class" => MergeStrategy.discard + case PathList("META-INF", "versions", _*) => MergeStrategy.discard // Project iceberg `dependsOn` spark and accidentally brings in it, along with its // compile-time dependencies (like delta-storage). We want these excluded from the // delta-iceberg jar. @@ -1478,7 +1472,7 @@ lazy val iceberg = (project in file("iceberg")) ) // scalastyle:on println -val icebergShadedVersion = "1.11.0" +val icebergShadedVersion = "1.12.0" lazy val icebergShaded = (project in file("icebergShaded")) .dependsOn(spark % "provided") .disablePlugins(JavaFormatterPlugin, ScalafmtPlugin) @@ -1636,9 +1630,14 @@ lazy val flink = (project in file("flink")) .dependsOn(kernelUnityCatalog) .settings( name := "delta-flink", + // Publish one artifact per compatible Flink minor line, for example delta-flink_2.0. + moduleName := s"delta-flink_${CrossFlinkVersions.getFlinkVersionSpec().shortVersion}", commonSettings, releaseSettings, javafmtCheckSettings(), + // The base release publishes non-Flink modules. Flink versions are published separately by + // CrossFlinkVersions.crossFlinkReleaseSteps. + publish / skip := sys.props.getOrElse("skipFlinkPublish", "false").toBoolean, publishArtifact := scalaBinaryVersion.value == "2.13", // only publish once autoScalaLibrary := false, // exclude scala-library from dependencies assembly / assemblyJarName := s"delta-flink-$flinkVersion-${version.value}.jar", @@ -1674,7 +1673,7 @@ lazy val flink = (project in file("flink")) // Publish the pinned UC jars before sbt tries to resolve them. update := update.dependsOn(ensurePinnedUnityCatalog).value, - libraryDependencies ++= Seq( + libraryDependencies ++= junit5TestDependencies ++ Seq( "org.apache.flink" % "flink-core" % flinkVersion % "provided", "org.apache.flink" % "flink-table-common" % flinkVersion % "provided", "org.apache.flink" % "flink-streaming-java" % flinkVersion % "provided", @@ -1691,11 +1690,6 @@ lazy val flink = (project in file("flink")) // io.delta.flink.kernel.dv compiles against the same library on the classpath. "org.roaringbitmap" % "RoaringBitmap" % "0.9.25", - // Test dependencies - "org.junit.jupiter" % "junit-jupiter-api" % "5.11.4" % "test", - "org.junit.jupiter" % "junit-jupiter-engine" % "5.11.4" % "test", - "org.junit.jupiter" % "junit-jupiter-params" % "5.11.4" % "test", - "com.github.sbt.junit" % "jupiter-interface" % "0.17.0" % "test", "org.apache.flink" % "flink-test-utils" % flinkVersion % "test", "org.apache.flink" % "flink-clients" % flinkVersion % "test", "org.apache.flink" % "flink-table-api-java-bridge" % flinkVersion % Test, @@ -1854,6 +1848,13 @@ lazy val releaseSettings = Seq( releasePublishArtifactsAction := PgpKeys.publishSigned.value, releaseCrossBuild := true, pgpPassphrase := sys.env.get("PGP_PASSPHRASE").map(_.toArray), + // Allow local Maven overwrites for release versions. The cross-Spark publish workflow + // publishes modules like delta-contribs (no Spark suffix) in both the backward-compat + // step and the per-version steps, producing identical artifacts. SBT 1.9+ blocks + // overwriting release artifacts by default; this restores the prior behavior for local + // publishing only (remote publish via publishSigned is unaffected). + publishLocalConfiguration := publishLocalConfiguration.value.withOverwrite(true), + publishM2Configuration := publishM2Configuration.value.withOverwrite(true), // TODO: This isn't working yet ... sonatypeProfileName := "io.delta", // sonatype account domain name prefix / group ID @@ -1946,7 +1947,8 @@ releaseProcess := Seq[ReleaseStep]( setReleaseVersion, commitReleaseVersion, tagRelease -) ++ CrossSparkVersions.crossSparkReleaseSteps("publishSigned") ++ Seq[ReleaseStep]( +) ++ CrossSparkVersions.crossSparkReleaseSteps("publishSigned") ++ + CrossFlinkVersions.crossFlinkReleaseSteps("publishSigned") ++ Seq[ReleaseStep]( // Do NOT use `sonatypeBundleRelease` - it will actually release to Maven! We want to do that // manually. diff --git a/build/sbt-config/repositories b/build/sbt-config/repositories index 2853417321d..80fabe5499f 100644 --- a/build/sbt-config/repositories +++ b/build/sbt-config/repositories @@ -2,9 +2,9 @@ local local-preloaded-ivy: file:///${sbt.preloaded-${sbt.global.base-${user.home}/.sbt}/preloaded/}, [organization]/[module]/[revision]/[type]s/[artifact](-[classifier]).[ext] local-preloaded: file:///${sbt.preloaded-${sbt.global.base-${user.home}/.sbt}/preloaded/} - mulesoft: https://repository.mulesoft.org/nexus/content/groups/public/ gcs-maven-central-mirror: https://maven-central.storage-download.googleapis.com/maven2/ maven-central + mulesoft: https://repository.mulesoft.org/nexus/content/groups/public/ typesafe-ivy-releases: https://repo.typesafe.com/typesafe/ivy-releases/, [organization]/[module]/[revision]/[type]s/[artifact](-[classifier]).[ext], bootOnly sbt-ivy-snapshots: https://repo.scala-sbt.org/scalasbt/ivy-snapshots/, [organization]/[module]/[revision]/[type]s/[artifact](-[classifier]).[ext], bootOnly sbt-plugin-releases: https://repo.scala-sbt.org/scalasbt/sbt-plugin-releases/, [organization]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)[revision]/[type]s/[artifact](-[classifier]).[ext] diff --git a/examples/scala/build.sbt b/examples/scala/build.sbt index 53426ec0d2d..a44726adcba 100644 --- a/examples/scala/build.sbt +++ b/examples/scala/build.sbt @@ -51,6 +51,8 @@ def getMajorMinor(version: String): (Int, Int) = { // Used as a fallback for local dev when SPARK_VERSION env var is not set. val lookupSparkVersion: PartialFunction[(Int, Int), String] = { // TODO: how to run integration tests for multiple Spark versions + // `major` and `minor` refer to the Delta version, not the Spark version. + case (major, minor) if major >= 4 && minor >= 4 => "4.2.0" case (major, minor) if major >= 4 && minor >= 1 => "4.1.0" // version 4.0.0 case (major, minor) if major >= 4 => "4.0.0" diff --git a/flink/README.md b/flink/README.md index 5e71dce5bf5..1f841f14a56 100644 --- a/flink/README.md +++ b/flink/README.md @@ -8,7 +8,24 @@ Note: this is a private build right now. Suggestions and feedbacks are welcome. ``` ### Supported Flink Versions -- **Flink v2.0** + +Delta tests the latest patch release from each supported Flink minor line: + +| Flink line | Tested version | +|------------|----------------| +| 2.0 | 2.0.2 | +| 2.1 | 2.1.3 | +| 2.2 | 2.2.1 | +| 2.3 | 2.3.0 | + +The default build uses Flink 2.3.0. To select a supported Flink version during development, set +`flinkVersion` on the command line: + +```bash +build/sbt -DflinkVersion=2.1 flink/test +``` + +The full tested version, such as `2.1.3`, is also accepted. ### Connector Overview - Built on **Flink Connector V2 API** diff --git a/flink/src/main/java/io/delta/flink/sink/Conversions.java b/flink/src/main/java/io/delta/flink/sink/Conversions.java index 687fda3a3fd..8a4e4920fd5 100644 --- a/flink/src/main/java/io/delta/flink/sink/Conversions.java +++ b/flink/src/main/java/io/delta/flink/sink/Conversions.java @@ -137,10 +137,16 @@ public static Literal data(StructType rowType, RowData rowData, int colIdx) { if (rowData.isNullAt(colIdx)) { return Literal.ofNull(dataType); } - if (dataType.equivalent(io.delta.kernel.types.IntegerType.INTEGER)) { - return Literal.ofInt(rowData.getInt(colIdx)); + if (dataType.equivalent(io.delta.kernel.types.BooleanType.BOOLEAN)) { + return Literal.ofBoolean(rowData.getBoolean(colIdx)); + } else if (dataType.equivalent(io.delta.kernel.types.BinaryType.BINARY)) { + return Literal.ofBinary(rowData.getBinary(colIdx)); } else if (dataType.equivalent(io.delta.kernel.types.ByteType.BYTE)) { return Literal.ofByte(rowData.getByte(colIdx)); + } else if (dataType.equivalent(io.delta.kernel.types.ShortType.SHORT)) { + return Literal.ofShort(rowData.getShort(colIdx)); + } else if (dataType.equivalent(io.delta.kernel.types.IntegerType.INTEGER)) { + return Literal.ofInt(rowData.getInt(colIdx)); } else if (dataType.equivalent(io.delta.kernel.types.LongType.LONG)) { return Literal.ofLong(rowData.getLong(colIdx)); } else if (dataType.equivalent(io.delta.kernel.types.StringType.STRING)) { diff --git a/flink/src/main/java/io/delta/flink/sink/DeltaSink.java b/flink/src/main/java/io/delta/flink/sink/DeltaSink.java index f195a8b070a..19fa9e56aca 100644 --- a/flink/src/main/java/io/delta/flink/sink/DeltaSink.java +++ b/flink/src/main/java/io/delta/flink/sink/DeltaSink.java @@ -109,6 +109,7 @@ public SinkWriter createWriter(WriterInitContext context) { .withAttemptNumber(context.getTaskInfo().getAttemptNumber()) .withDeltaTable(deltaTable) .withConf(conf) + .withInputSerializer(context.createInputSerializer()) .withMetricGroup(context.metricGroup()) .build(); } diff --git a/flink/src/main/java/io/delta/flink/sink/DeltaSinkWriter.java b/flink/src/main/java/io/delta/flink/sink/DeltaSinkWriter.java index 392d5a0c3bf..4239a4e14bd 100644 --- a/flink/src/main/java/io/delta/flink/sink/DeltaSinkWriter.java +++ b/flink/src/main/java/io/delta/flink/sink/DeltaSinkWriter.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.util.*; import org.apache.flink.api.common.eventtime.Watermark; +import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.connector.sink2.CommittingSinkWriter; import org.apache.flink.metrics.groups.SinkWriterMetricGroup; import org.apache.flink.streaming.api.connector.sink2.SupportsPreWriteTopology; @@ -65,6 +66,7 @@ public class DeltaSinkWriter implements CommittingSinkWriter inputSerializer; private final SinkWriterMetricGroup metricGroup; private volatile long lastSendTimeMs; @@ -81,6 +83,7 @@ private DeltaSinkWriter( int attemptNumber, DeltaTable deltaTable, DeltaSinkConf conf, + TypeSerializer inputSerializer, SinkWriterMetricGroup metricGroup) { this.jobId = jobId; this.subtaskId = subtaskId; @@ -88,6 +91,7 @@ private DeltaSinkWriter( this.deltaTable = deltaTable; this.conf = conf; + this.inputSerializer = inputSerializer; this.metricGroup = metricGroup; metricGroup.setCurrentSendTimeGauge(() -> lastSendTimeMs); @@ -129,10 +133,13 @@ public void write(RowData element, Context context) throws IOException, Interrup // without appending a row. switch (element.getRowKind()) { case INSERT: - mergeStrategy.insert(extractPrimaryKey(element), partitionValues, element, context); + // Writer tasks buffer rows, so take ownership before Flink can reuse the input object. + mergeStrategy.insert( + extractPrimaryKey(element), partitionValues, inputSerializer.copy(element), context); break; case UPDATE_AFTER: - mergeStrategy.upsert(extractPrimaryKey(element), partitionValues, element, context); + mergeStrategy.upsert( + extractPrimaryKey(element), partitionValues, inputSerializer.copy(element), context); break; case UPDATE_BEFORE: // Dropped — see policy comment above. @@ -238,6 +245,7 @@ public static class Builder { private DeltaTable deltaTable; private DeltaSinkConf conf; + private TypeSerializer inputSerializer; private SinkWriterMetricGroup metricGroup; @@ -268,6 +276,11 @@ public Builder withConf(DeltaSinkConf conf) { return this; } + public Builder withInputSerializer(TypeSerializer inputSerializer) { + this.inputSerializer = inputSerializer; + return this; + } + public Builder withMetricGroup(SinkWriterMetricGroup metricGroup) { this.metricGroup = metricGroup; return this; @@ -279,8 +292,10 @@ public DeltaSinkWriter build() { Objects.requireNonNull(deltaTable, "deltaTable must not be null"); Objects.requireNonNull(metricGroup, "metricGroup must not be null"); Objects.requireNonNull(conf, "conf must not be null"); + Objects.requireNonNull(inputSerializer, "inputSerializer must not be null"); - return new DeltaSinkWriter(jobId, subtaskId, attemptNumber, deltaTable, conf, metricGroup); + return new DeltaSinkWriter( + jobId, subtaskId, attemptNumber, deltaTable, conf, inputSerializer, metricGroup); } } } diff --git a/flink/src/main/java/io/delta/flink/sink/DeltaWriterResultAggregator.java b/flink/src/main/java/io/delta/flink/sink/DeltaWriterResultAggregator.java index e49d5987c7b..aebcf5e0bd3 100644 --- a/flink/src/main/java/io/delta/flink/sink/DeltaWriterResultAggregator.java +++ b/flink/src/main/java/io/delta/flink/sink/DeltaWriterResultAggregator.java @@ -51,7 +51,9 @@ public class DeltaWriterResultAggregator implements OneInputStreamOperator< CommittableMessage, CommittableMessage> { private static final Logger LOG = LoggerFactory.getLogger(DeltaWriterResultAggregator.class); + private static final long NO_PENDING_CHECKPOINT = -1; private final Collection results; + private long pendingCheckpointId = NO_PENDING_CHECKPOINT; public DeltaWriterResultAggregator() { results = new ArrayList<>(); @@ -70,13 +72,25 @@ public void open() throws Exception { @Override public void finish() throws IOException { LOG.debug("Finishing"); - prepareSnapshotPreBarrier(Long.MAX_VALUE); + if (pendingCheckpointId != NO_PENDING_CHECKPOINT) { + // Preserve the upstream ID so the final checkpoint can commit bounded-input rows. + prepareSnapshotPreBarrier(pendingCheckpointId); + } } @Override public void prepareSnapshotPreBarrier(long checkpointId) throws IOException { LOG.debug("Aggregating on checkpoint {}", checkpointId); + if (pendingCheckpointId != NO_PENDING_CHECKPOINT) { + Preconditions.checkState( + pendingCheckpointId == checkpointId, + "Cannot aggregate results from checkpoint " + + pendingCheckpointId + + " into checkpoint " + + checkpointId); + } + DeltaWriterResult merged = results.stream() .collect(DeltaWriterResult::new, DeltaWriterResult::merge, DeltaWriterResult::merge); @@ -100,16 +114,31 @@ public void prepareSnapshotPreBarrier(long checkpointId) throws IOException { checkpointId, committable.getDeltaActions().size()); results.clear(); + pendingCheckpointId = NO_PENDING_CHECKPOINT; } @Override public void processElement(StreamRecord> element) throws Exception { // TODO control buffer size - if (element.isRecord() && element.getValue() instanceof CommittableWithLineage) { - LOG.debug("Received writerResult: {}", element.getValue()); - results.add( - ((CommittableWithLineage) element.getValue()).getCommittable()); + if (element.isRecord()) { + CommittableMessage message = element.getValue(); + if (pendingCheckpointId == NO_PENDING_CHECKPOINT) { + pendingCheckpointId = message.getCheckpointId(); + } + Preconditions.checkState( + pendingCheckpointId == message.getCheckpointId(), + "Received results from checkpoints " + + pendingCheckpointId + + " and " + + message.getCheckpointId() + + " in one aggregation"); + + if (!(message instanceof CommittableWithLineage)) { + return; + } + LOG.debug("Received writerResult: {}", message); + results.add(((CommittableWithLineage) message).getCommittable()); } } } diff --git a/flink/src/main/java/io/delta/flink/sink/mergestrategy/AppendOnly.java b/flink/src/main/java/io/delta/flink/sink/mergestrategy/AppendOnly.java index 4e8e174e648..d122226a26c 100644 --- a/flink/src/main/java/io/delta/flink/sink/mergestrategy/AppendOnly.java +++ b/flink/src/main/java/io/delta/flink/sink/mergestrategy/AppendOnly.java @@ -49,9 +49,13 @@ public class AppendOnly private List completedWrites; + /** First task failure, saved because Caffeine does not propagate removal-listener exceptions. */ + private RuntimeException writerCompletionFailure; + public void init(DeltaSinkWriter writer) { this.writer = writer; this.completedWrites = new ArrayList<>(); + this.writerCompletionFailure = null; this.writerTasksByPartition = Caffeine.newBuilder() .executor(Runnable::run) @@ -99,6 +103,9 @@ public void delete(List primaryKey, Map partitionValue @Override public List merge() { writerTasksByPartition.invalidateAll(); + if (writerCompletionFailure != null) { + throw writerCompletionFailure; + } List results = List.copyOf(completedWrites); completedWrites.clear(); return results; @@ -112,8 +119,10 @@ public void onRemoval( if (value != null) { completedWrites.addAll(value.complete()); } - } catch (IOException e) { - throw new RuntimeException(e); + } catch (IOException | RuntimeException e) { + if (writerCompletionFailure == null) { + writerCompletionFailure = ExceptionUtils.wrap(e); + } } } } diff --git a/flink/src/main/java/io/delta/flink/sink/mergestrategy/Upsert.java b/flink/src/main/java/io/delta/flink/sink/mergestrategy/Upsert.java index 5ce17157618..a63ba4c3bec 100644 --- a/flink/src/main/java/io/delta/flink/sink/mergestrategy/Upsert.java +++ b/flink/src/main/java/io/delta/flink/sink/mergestrategy/Upsert.java @@ -69,6 +69,9 @@ public abstract class Upsert private final List completedWrites; + /** First task failure, saved because Caffeine does not propagate removal-listener exceptions. */ + private RuntimeException writerCompletionFailure; + /** * @param rowLocator selects which existing files to scan for PKs that need pre-image removal when * {@link #merge} runs at checkpoint boundaries. @@ -88,6 +91,7 @@ protected Upsert(RowLocator rowLocator) { public void init(DeltaSinkWriter writer) { this.writer = writer; table = (AbstractKernelTable) writer.getTable(); + writerCompletionFailure = null; } @Override @@ -142,6 +146,9 @@ public List merge() throws IOException { // Dump all Writer Tasks writerTasksByPartition.invalidateAll(); + if (writerCompletionFailure != null) { + throw writerCompletionFailure; + } if (!upsertedPrimaryKeys.isEmpty() || !deletedPrimaryKeys.isEmpty()) { try { @@ -179,8 +186,10 @@ public void onRemoval( if (value != null) { completedWrites.addAll(value.complete()); } - } catch (IOException e) { - throw new RuntimeException(e); + } catch (IOException | RuntimeException e) { + if (writerCompletionFailure == null) { + writerCompletionFailure = ExceptionUtils.wrap(e); + } } } diff --git a/flink/src/main/java/io/delta/flink/sink/sql/DeltaDynamicTableSink.java b/flink/src/main/java/io/delta/flink/sink/sql/DeltaDynamicTableSink.java index 87b7be7a6be..cf0301dc579 100644 --- a/flink/src/main/java/io/delta/flink/sink/sql/DeltaDynamicTableSink.java +++ b/flink/src/main/java/io/delta/flink/sink/sql/DeltaDynamicTableSink.java @@ -129,6 +129,8 @@ public SinkRuntimeProvider getSinkRuntimeProvider(Context context) { "unitycatalog.table_name", tableId.asSummaryString())) .withEndpoint(options.get(FlinkUnityCatalogFactory.ENDPOINT.key())) .withToken(options.get(FlinkUnityCatalogFactory.TOKEN.key())) + .withPartitionColNames( + Arrays.asList(options.getOrDefault(PARTITIONS.key(), "").split(","))) .withWriteMode(writeMode) .withPrimaryKey(primaryKeyOrdinals) .build(); diff --git a/flink/src/main/java/io/delta/flink/sink/sql/FlinkUnityCatalog.java b/flink/src/main/java/io/delta/flink/sink/sql/FlinkUnityCatalog.java index 8d8d6d28f20..268cbdf088a 100644 --- a/flink/src/main/java/io/delta/flink/sink/sql/FlinkUnityCatalog.java +++ b/flink/src/main/java/io/delta/flink/sink/sql/FlinkUnityCatalog.java @@ -284,15 +284,13 @@ public void dropFunction(ObjectPath functionPath, boolean ignoreIfNotExists) @Override public CatalogTableStatistics getTableStatistics(ObjectPath tablePath) throws TableNotExistException, CatalogException { - notSupported("GET TABLE STATISTICS"); - return null; + return CatalogTableStatistics.UNKNOWN; } @Override public CatalogColumnStatistics getTableColumnStatistics(ObjectPath tablePath) throws TableNotExistException, CatalogException { - notSupported("GET TABLE COLUMN STATISTICS"); - return null; + return CatalogColumnStatistics.UNKNOWN; } @Override diff --git a/flink/src/main/java/io/delta/flink/sink/sql/FlinkUnityCatalogTable.java b/flink/src/main/java/io/delta/flink/sink/sql/FlinkUnityCatalogTable.java index c2559c657be..aee48f27fb2 100644 --- a/flink/src/main/java/io/delta/flink/sink/sql/FlinkUnityCatalogTable.java +++ b/flink/src/main/java/io/delta/flink/sink/sql/FlinkUnityCatalogTable.java @@ -23,11 +23,13 @@ import io.unitycatalog.client.model.TableInfo; import java.net.URI; import java.util.ArrayList; +import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; import org.apache.flink.table.api.Schema; import org.apache.flink.table.catalog.CatalogBaseTable; import org.apache.flink.table.catalog.CatalogTable; @@ -48,6 +50,7 @@ public class FlinkUnityCatalogTable implements CatalogTable { private final Map properties; private final Schema schema; private final String dataFormat; + private final List partitionKeys; private final URI endpoint; private final String token; @@ -61,12 +64,26 @@ public FlinkUnityCatalogTable( Map properties, URI endpoint, String token) { + this(name, schema, dataFormat, comment, description, properties, List.of(), endpoint, token); + } + + private FlinkUnityCatalogTable( + String name, + Schema schema, + String dataFormat, + String comment, + String description, + Map properties, + List partitionKeys, + URI endpoint, + String token) { this.name = name; this.dataFormat = dataFormat; this.comment = comment; this.description = description; this.properties = properties; this.schema = schema; + this.partitionKeys = List.copyOf(partitionKeys); this.endpoint = endpoint; this.token = token; @@ -82,10 +99,19 @@ public FlinkUnityCatalogTable(TableInfo tableInfo, URI endpoint, String token) { tableInfo.getComment(), "", tableInfo.getProperties(), + buildPartitionKeys(tableInfo.getColumns()), endpoint, token); } + static List buildPartitionKeys(List columnInfos) { + return columnInfos.stream() + .filter(column -> column.getPartitionIndex() != null) + .sorted(Comparator.comparing(ColumnInfo::getPartitionIndex)) + .map(ColumnInfo::getName) + .collect(Collectors.toList()); + } + public static Schema buildSchema(List columnInfos) { Schema.Builder builder = Schema.newBuilder(); columnInfos.forEach( @@ -168,7 +194,9 @@ public static DataType fromJson(String json) { case "boolean": return new AtomicDataType(new BooleanType(nullable)); case "binary": - return new AtomicDataType(new BinaryType(nullable, BinaryType.MAX_LENGTH)); + return new AtomicDataType(new VarBinaryType(nullable, VarBinaryType.MAX_LENGTH)); + case "byte": + return new AtomicDataType(new TinyIntType(nullable)); case "date": return new AtomicDataType(new DateType(nullable)); case "double": @@ -179,6 +207,10 @@ public static DataType fromJson(String json) { return new AtomicDataType(new IntType(nullable)); case "long": return new AtomicDataType(new BigIntType(nullable)); + case "short": + return new AtomicDataType(new SmallIntType(nullable)); + case "void": + return new AtomicDataType(new NullType()); case "string": return new AtomicDataType(new VarCharType(nullable, VarCharType.MAX_LENGTH)); case "timestamp": @@ -200,18 +232,18 @@ public Schema getUnresolvedSchema() { @Override public boolean isPartitioned() { - return false; + return !partitionKeys.isEmpty(); } @Override public List getPartitionKeys() { - return List.of(); + return partitionKeys; } @Override public CatalogTable copy(Map options) { return new FlinkUnityCatalogTable( - name, schema, dataFormat, comment, description, options, endpoint, token); + name, schema, dataFormat, comment, description, options, partitionKeys, endpoint, token); } // TODO: token is visible in Flink EXPLAIN and serialized job graph; consider a secure approach @@ -224,7 +256,9 @@ public Map getOptions() { FlinkUnityCatalogFactory.ENDPOINT.key(), endpoint.toString(), FlinkUnityCatalogFactory.TOKEN.key(), - token); + token, + DeltaDynamicTableSinkFactory.PARTITIONS.key(), + String.join(",", partitionKeys)); } return Map.of(); } @@ -237,7 +271,7 @@ public String getComment() { @Override public CatalogBaseTable copy() { return new FlinkUnityCatalogTable( - name, schema, dataFormat, comment, description, properties, endpoint, token); + name, schema, dataFormat, comment, description, properties, partitionKeys, endpoint, token); } @Override diff --git a/flink/src/main/java/io/delta/flink/table/AbstractKernelTable.java b/flink/src/main/java/io/delta/flink/table/AbstractKernelTable.java index d8c14970d07..10dcb1601e4 100644 --- a/flink/src/main/java/io/delta/flink/table/AbstractKernelTable.java +++ b/flink/src/main/java/io/delta/flink/table/AbstractKernelTable.java @@ -53,6 +53,7 @@ import java.time.Duration; import java.util.*; import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; @@ -139,7 +140,14 @@ protected static List normalize(List rawPartitions) { protected final DeltaCatalog catalog; protected String tableId; - protected String tableUUID; + + /** + * @deprecated The UC Delta API no longer requires a table UUID for credential lookup. This field + * is retained for remaining Kernel and metrics call sites and should be removed after they + * are migrated. + */ + @Deprecated protected String tableUUID; + protected URI tablePath; protected final TableConf conf; /* @@ -270,6 +278,68 @@ public void refresh() { refresh(null); } + /** Internal primitive for committing an append-only schema update. */ + final void updateSchema(StructType targetSchema) { + Objects.requireNonNull(targetSchema, "targetSchema cannot be null"); + if (refreshThreadPool == null) { + throw new IllegalStateException("DeltaTable must be opened before updating its schema"); + } + AtomicReference initialSchema = new AtomicReference<>(); + + withTiming( + "updateSchema", + () -> + withRetry( + () -> { + updateSchemaFromLatestSnapshot(targetSchema, initialSchema); + return null; + })); + } + + private void updateSchemaFromLatestSnapshot( + StructType targetSchema, AtomicReference initialSchema) { + Snapshot latestSnapshot = + snapshot().orElseThrow(() -> new IllegalStateException("Snapshot should exist")); + StructType latestSchema = latestSnapshot.getSchema(); + StructType schemaAtFirstAttempt = initialSchema.get(); + if (schemaAtFirstAttempt == null) { + initialSchema.set(latestSchema); + } else if (!SchemaEvolutionUtils.logicallyEqual(schemaAtFirstAttempt, latestSchema) + && !SchemaEvolutionUtils.logicallyEqual(latestSchema, targetSchema)) { + throw new IllegalArgumentException( + String.format( + "Target schema is stale: table schema changed concurrently from %s to %s", + schemaAtFirstAttempt, latestSchema)); + } + Optional updatedSchema = + SchemaEvolutionUtils.buildAdditiveSchema(latestSchema, targetSchema); + + if (updatedSchema.isEmpty()) { + refresh(latestSnapshot); + return; + } + + Engine localEngine = getEngine(); + Transaction txn = + latestSnapshot + .buildUpdateTableTransaction(ENGINE_INFO, Operation.WRITE) + .withUpdatedSchema(updatedSchema.get()) + .build(localEngine); + TransactionCommitResult result = + withTiming( + "updateSchema.txn", () -> txn.commit(localEngine, CloseableIterable.emptyIterable())); + Snapshot committedSnapshot = + result + .getPostCommitSnapshot() + .orElseGet( + () -> + snapshot() + .orElseThrow( + () -> new IllegalStateException("Snapshot should exist after commit"))); + refresh(committedSnapshot); + onPostCommit(committedSnapshot, result.getTransactionReport()); + } + @Override public Optional commit( CloseableIterable actions, String appId, long txnId, Map properties) { @@ -469,8 +539,10 @@ protected void loadDeltaTable() { partitionColumns, conf.catalogConf(), tableDesc -> { + this.conf.update(tableDesc.requiredProperties); this.tablePath = normalize(tableDesc.tablePath); this.tableUUID = tableDesc.uuid; + credentialManager.initializeCredentials(tableDesc.getStorageProperties()); createDeltaTable(); }); } diff --git a/flink/src/main/java/io/delta/flink/table/CatalogManagedTable.java b/flink/src/main/java/io/delta/flink/table/CatalogManagedTable.java index b23cc3c3968..72388a421d2 100644 --- a/flink/src/main/java/io/delta/flink/table/CatalogManagedTable.java +++ b/flink/src/main/java/io/delta/flink/table/CatalogManagedTable.java @@ -16,9 +16,6 @@ package io.delta.flink.table; -import static io.delta.kernel.internal.tablefeatures.TableFeatures.*; -import static io.delta.kernel.unitycatalog.UCCatalogManagedClient.UC_TABLE_ID_KEY; - import io.delta.kernel.Snapshot; import io.delta.kernel.data.Row; import io.delta.kernel.transaction.CreateTableTransactionBuilder; @@ -32,8 +29,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.stream.Collectors; -import java.util.stream.Stream; import org.apache.flink.util.Preconditions; import org.apache.hadoop.conf.Configuration; import org.slf4j.Logger; @@ -56,17 +51,6 @@ public class CatalogManagedTable extends AbstractKernelTable { private static final Logger LOG = LoggerFactory.getLogger(CatalogManagedTable.class); - public static final Map CATALOG_MANAGED_REQUIRED_FEATURES_CONF = - Stream.of( - CATALOG_MANAGED_RW_FEATURE, - DELETION_VECTORS_RW_FEATURE, - DOMAIN_METADATA_W_FEATURE, - IN_COMMIT_TIMESTAMP_W_FEATURE, - ROW_TRACKING_W_FEATURE, - CHECKPOINT_V2_RW_FEATURE, - VACUUM_PROTOCOL_CHECK_RW_FEATURE) - .map(feature -> String.format("delta.feature.%s", feature.featureName())) - .collect(Collectors.toMap(key -> key, key -> "supported")); private final URI endpoint; private final UnityCatalog.AuthMode authMode; @@ -167,22 +151,12 @@ public Optional commit( return super.commit(actions, appId, txnId, Map.of()); } - @Override - protected void createDeltaTable() { - conf.update(Map.of(UC_TABLE_ID_KEY, tableUUID)); - super.createDeltaTable(); - } - @Override protected CreateTableTransactionBuilder buildCreateTableTransaction() { + // Pass the table identifier so the UC committer finalizes (promotes) the staging table when + // the version 0 commit is written, rather than relying on a separate catalog registration. return catalogManagedClient.buildCreateTableTransaction( - tableUUID, tablePath.toString(), schema, ENGINE_INFO); - } - - @Override - protected Map extraConf() { - // Make sure the table supports table features needed by CatalogManaged. - return CATALOG_MANAGED_REQUIRED_FEATURES_CONF; + tableUUID, tablePath.toString(), schema, ENGINE_INFO, getUcTableIdentifier()); } @Override diff --git a/flink/src/main/java/io/delta/flink/table/DeltaCatalog.java b/flink/src/main/java/io/delta/flink/table/DeltaCatalog.java index c97fa6a46a6..0d5504fa526 100644 --- a/flink/src/main/java/io/delta/flink/table/DeltaCatalog.java +++ b/flink/src/main/java/io/delta/flink/table/DeltaCatalog.java @@ -122,6 +122,9 @@ class TableDescriptor { /** Storage properties returned while resolving the table, when available. */ Map storageProperties = Map.of(); + /** Properties the catalog requires when initializing a newly allocated table. */ + Map requiredProperties = Map.of(); + public TableDescriptor() {} public TableDescriptor(String tableId, String uuid, URI tablePath) { @@ -136,6 +139,16 @@ public TableDescriptor( this.storageProperties = Collections.unmodifiableMap(new HashMap<>(storageProperties)); } + public TableDescriptor( + String tableId, + String uuid, + URI tablePath, + Map storageProperties, + Map requiredProperties) { + this(tableId, uuid, tablePath, storageProperties); + this.requiredProperties = Collections.unmodifiableMap(new HashMap<>(requiredProperties)); + } + public String getTableId() { return tableId; } @@ -151,5 +164,9 @@ public URI getTablePath() { public Map getStorageProperties() { return storageProperties; } + + public Map getRequiredProperties() { + return requiredProperties; + } } } diff --git a/flink/src/main/java/io/delta/flink/table/SchemaEvolutionUtils.java b/flink/src/main/java/io/delta/flink/table/SchemaEvolutionUtils.java new file mode 100644 index 00000000000..c76604244d3 --- /dev/null +++ b/flink/src/main/java/io/delta/flink/table/SchemaEvolutionUtils.java @@ -0,0 +1,102 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.flink.table; + +import io.delta.kernel.types.ArrayType; +import io.delta.kernel.types.DataType; +import io.delta.kernel.types.MapType; +import io.delta.kernel.types.StructField; +import io.delta.kernel.types.StructType; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +/** Internal helpers for validating append-only schema updates. */ +final class SchemaEvolutionUtils { + + private SchemaEvolutionUtils() {} + + static Optional buildAdditiveSchema( + StructType currentSchema, StructType targetSchema) { + if (targetSchema.length() < currentSchema.length()) { + throw new IllegalArgumentException( + String.format( + "Target schema is stale: current schema has %d fields but target schema has %d", + currentSchema.length(), targetSchema.length())); + } + + for (int ordinal = 0; ordinal < currentSchema.length(); ordinal++) { + StructField currentField = currentSchema.at(ordinal); + StructField targetField = targetSchema.at(ordinal); + if (!logicallyEqual(currentField, targetField)) { + throw new IllegalArgumentException( + String.format( + "Unsupported schema change at ordinal %d. Current field: %s; target field: %s. " + + "Only nullable top-level columns appended to the schema are supported", + ordinal, currentField, targetField)); + } + } + + if (targetSchema.length() == currentSchema.length()) { + return Optional.empty(); + } + + List fields = new ArrayList<>(currentSchema.fields()); + for (StructField newField : + targetSchema.fields().subList(currentSchema.length(), targetSchema.length())) { + if (!newField.isNullable()) { + throw new IllegalArgumentException( + String.format("New column '%s' must be nullable", newField.getName())); + } + fields.add(newField); + } + return Optional.of(new StructType(fields)); + } + + static boolean logicallyEqual(DataType currentType, DataType targetType) { + if (currentType instanceof StructType && targetType instanceof StructType) { + StructType currentStruct = (StructType) currentType; + StructType targetStruct = (StructType) targetType; + if (currentStruct.length() != targetStruct.length()) { + return false; + } + for (int ordinal = 0; ordinal < currentStruct.length(); ordinal++) { + if (!logicallyEqual(currentStruct.at(ordinal), targetStruct.at(ordinal))) { + return false; + } + } + return true; + } + if (currentType instanceof ArrayType && targetType instanceof ArrayType) { + return logicallyEqual( + ((ArrayType) currentType).getElementField(), ((ArrayType) targetType).getElementField()); + } + if (currentType instanceof MapType && targetType instanceof MapType) { + MapType currentMap = (MapType) currentType; + MapType targetMap = (MapType) targetType; + return logicallyEqual(currentMap.getKeyField(), targetMap.getKeyField()) + && logicallyEqual(currentMap.getValueField(), targetMap.getValueField()); + } + return currentType.equals(targetType); + } + + private static boolean logicallyEqual(StructField currentField, StructField targetField) { + return currentField.getName().equals(targetField.getName()) + && currentField.isNullable() == targetField.isNullable() + && logicallyEqual(currentField.getDataType(), targetField.getDataType()); + } +} diff --git a/flink/src/main/java/io/delta/flink/table/UnityCatalog.java b/flink/src/main/java/io/delta/flink/table/UnityCatalog.java index ba99e5b9398..bd206a4e296 100644 --- a/flink/src/main/java/io/delta/flink/table/UnityCatalog.java +++ b/flink/src/main/java/io/delta/flink/table/UnityCatalog.java @@ -19,10 +19,10 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import dev.failsafe.function.CheckedSupplier; -import io.delta.kernel.internal.types.DataTypeJsonSerDe; import io.delta.kernel.types.*; import io.delta.kernel.unitycatalog.UCTableIdentifier; import io.delta.storage.commit.TableIdentifier; +import io.delta.storage.commit.actions.AbstractProtocol; import io.delta.storage.commit.uccommitcoordinator.UCConfigUtils; import io.delta.storage.commit.uccommitcoordinator.UCDeltaClient; import io.delta.storage.commit.uccommitcoordinator.UCDeltaModels; @@ -40,98 +40,27 @@ import java.util.*; import java.util.function.Consumer; import java.util.stream.Collectors; -import java.util.stream.IntStream; import org.apache.flink.util.Preconditions; import org.apache.hadoop.conf.Configuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * A {@code UnityCatalog} is a {@link DeltaCatalog} implementation that interacts with an external - * catalog service via UnityCatalog OSS APIs. + * A {@code UnityCatalog} is a {@link DeltaCatalog} implementation backed by Unity Catalog. * - *

This catalog resolves table metadata and access credentials by communicating with a remote - * catalog endpoint. It is typically used in environments where table discovery, authorization, and - * credential issuance are managed centrally by a service rather than through filesystem-based - * catalogs. + *

Table resolution, credential vending, and creation go through the UC Delta-Tables API (via + * {@link UCDeltaTokenBasedRestClient#loadTable} / {@code createStagingTable}). The credentials it + * vends register a Unity Catalog credential provider that refreshes short-lived credentials + * automatically. Catalog browsing (schema/table listing) uses the UC OpenAPI client, which has no + * Delta-Tables equivalent. * - *

The catalog uses: - * - *

    - *
  • a REST endpoint to load table metadata (e.g., storage location, table identifiers), and - *
  • temporary credentials generated by the catalog service to access underlying storage. - *
- * - *

Authentication to the catalog service is performed using a bearer token supplied at - * construction time. The token is attached to all outgoing REST requests. + *

Authentication is performed using a bearer token (or OAuth client credentials) supplied at + * construction time. */ public class UnityCatalog implements DeltaCatalog { private static final Logger LOG = LoggerFactory.getLogger(UnityCatalog.class); - private static ColumnTypeName columnTypeName(DataType dataType) { - if (dataType instanceof IntegerType) { - return ColumnTypeName.INT; - } else if (dataType instanceof LongType) { - return ColumnTypeName.LONG; - } else if (dataType instanceof ShortType) { - return ColumnTypeName.SHORT; - } else if (dataType instanceof ByteType) { - return ColumnTypeName.BYTE; - } else if (dataType instanceof BooleanType) { - return ColumnTypeName.BOOLEAN; - } else if (dataType instanceof FloatType) { - return ColumnTypeName.FLOAT; - } else if (dataType instanceof DoubleType) { - return ColumnTypeName.DOUBLE; - } else if (dataType instanceof StringType) { - return ColumnTypeName.STRING; - } else if (dataType instanceof BinaryType) { - return ColumnTypeName.BINARY; - } else if (dataType instanceof DecimalType) { - return ColumnTypeName.DECIMAL; - } else if (dataType instanceof DateType) { - return ColumnTypeName.DATE; - } else if (dataType instanceof TimestampType) { - return ColumnTypeName.TIMESTAMP; - } else if (dataType instanceof TimestampNTZType) { - return ColumnTypeName.TIMESTAMP_NTZ; - } else if (dataType instanceof ArrayType) { - return ColumnTypeName.ARRAY; - } else if (dataType instanceof MapType) { - return ColumnTypeName.MAP; - } else if (dataType instanceof StructType) { - return ColumnTypeName.STRUCT; - } else { - throw new UnsupportedOperationException("Unsupported data type: " + dataType); - } - } - - private static String typeText(DataType dataType) { - if (dataType instanceof DecimalType) { - DecimalType decimalType = (DecimalType) dataType; - return String.format("DECIMAL(%d,%d)", decimalType.getPrecision(), decimalType.getScale()); - } else if (dataType instanceof BasePrimitiveType) { - return dataType.toString().toUpperCase(Locale.ROOT); - } else if (dataType instanceof ArrayType) { - ArrayType arrayType = (ArrayType) dataType; - return String.format("ARRAY<%s>", typeText(arrayType.getElementType())); - } else if (dataType instanceof MapType) { - MapType mapType = (MapType) dataType; - return String.format( - "MAP<%s,%s>", typeText(mapType.getKeyType()), typeText(mapType.getValueType())); - } else if (dataType instanceof StructType) { - StructType structType = (StructType) dataType; - return String.format( - "STRUCT<%s>", - structType.fields().stream() - .map(field -> String.format("%s:%s", field.getName(), typeText(field.getDataType()))) - .collect(Collectors.joining(","))); - } else { - throw new UnsupportedOperationException("Unsupported data type: " + dataType); - } - } - enum AuthMode { token, oauth; @@ -181,6 +110,56 @@ static Map buildUcConfig( return ucConfig; } + private static Map requiredCreateProperties( + String tableId, + Map requestedProperties, + UCDeltaModels.StagingTableInfo stagingTableInfo) { + Map requiredProperties = new HashMap<>(); + AbstractProtocol requiredProtocol = stagingTableInfo.getRequiredProtocol(); + if (requiredProtocol != null) { + Set requiredFeatures = new HashSet<>(); + if (requiredProtocol.getReaderFeatures() != null) { + requiredFeatures.addAll(requiredProtocol.getReaderFeatures()); + } + if (requiredProtocol.getWriterFeatures() != null) { + requiredFeatures.addAll(requiredProtocol.getWriterFeatures()); + } + requiredFeatures.forEach( + feature -> + putRequiredProperty( + tableId, + requestedProperties, + requiredProperties, + "delta.feature." + feature, + "supported")); + } + stagingTableInfo + .getRequiredProperties() + .forEach( + (key, value) -> { + if (value != null) { + putRequiredProperty(tableId, requestedProperties, requiredProperties, key, value); + } + }); + return requiredProperties; + } + + private static void putRequiredProperty( + String tableId, + Map requestedProperties, + Map requiredProperties, + String key, + String requiredValue) { + String requestedValue = requestedProperties.get(key); + if (requestedValue != null && !requestedValue.equals(requiredValue)) { + throw new IllegalArgumentException( + String.format( + "Cannot create table %s: catalog requires %s=%s, but the requested value is %s", + tableId, key, requiredValue, requestedValue)); + } + requiredProperties.put(key, requiredValue); + } + private final String name; private final URI endpoint; private final AuthMode authMode; @@ -397,14 +376,14 @@ public TableDescriptor getTable(String tableName) { } /** - * Creates a new table in the backed UnityCatalog + * Reserves a staging table in Unity Catalog and hands its allocated storage location to {@code + * callback} so the caller can initialize the Delta table there. * - *

The {@code tableId} identifies the table within UC. This operation registers the table - * metadata with the remote catalog, including schema, partition specification, and table - * properties. + *

The staging table is promoted to a real managed table by the Kernel UC committer when the + * initial (version 0) commit is written, so this method does not register the table itself. * - *

This call will create a staging table, init the snapshot, then upgrade the staging table to - * real table. + * @param tableId the {@code catalog.schema.table} identifier of the table to create + * @param callback invoked with the reserved table id and storage location */ @Override public void createTable( @@ -419,61 +398,18 @@ public void createTable( Objects.requireNonNull(properties); withErrorHandling( () -> { - TablesApi tablesApi = new TablesApi(apiClient); - // Obtain names - UCTableIdentifier tableIdentifier = toUcTableIdentifier(tableId); - String schemaName = tableIdentifier.getSchemaName(); - String tableName = tableIdentifier.getTableName(); - // Column Info - List columnInfos = - IntStream.range(0, schema.fields().size()) - .mapToObj( - index -> { - StructField field = schema.fields().get(index); - ColumnInfo colInfo = - new ColumnInfo() - .name(field.getName()) - .typeName(columnTypeName(field.getDataType())) - .typeText(typeText(field.getDataType())) - .typeJson(DataTypeJsonSerDe.serializeDataType(field.getDataType())) - .position(index) - .nullable(field.isNullable()); - if (field.getDataType() instanceof DecimalType) { - DecimalType decimalType = (DecimalType) field.getDataType(); - colInfo - .typePrecision(decimalType.getPrecision()) - .typeScale(decimalType.getScale()); - } - return colInfo; - }) - .collect(Collectors.toList()); - - // Create a staging table and init the storage - StagingTableInfo stagingTableInfo = - tablesApi.createStagingTable( - new CreateStagingTable() - .catalogName(getName()) - .schemaName(schemaName) - .name(tableName)); - - // Init Delta snapshot - String storageLocation = stagingTableInfo.getStagingLocation(); - Objects.requireNonNull(storageLocation); - - TableDescriptor desc = - new TableDescriptor(tableId, stagingTableInfo.getId(), URI.create(storageLocation)); - callback.accept(desc); - - tablesApi.createTable( - new CreateTable() - .catalogName(getName()) - .schemaName(schemaName) - .name(tableName) - .storageLocation(stagingTableInfo.getStagingLocation()) - .tableType(TableType.MANAGED) - .dataSourceFormat(DataSourceFormat.DELTA) - .columns(columnInfos) - .properties(properties)); + UCDeltaModels.StagingTableInfo stagingTableInfo = + deltaClient.createStagingTable(toStorageTableIdentifier(tableId)); + String storageLocation = Objects.requireNonNull(stagingTableInfo.getLocation()); + Map requiredProperties = + requiredCreateProperties(tableId, properties, stagingTableInfo); + callback.accept( + new TableDescriptor( + tableId, + stagingTableInfo.getTableId().toString(), + URI.create(storageLocation), + stagingTableInfo.getStorageProperties(), + requiredProperties)); return null; }); } @@ -502,9 +438,29 @@ public Map getCredentials(String tableName) { () -> deltaClient.loadTable(toStorageTableIdentifier(tableName)).getStorageProperties()); } - /* =================================== - * Extra UnityCatalog Operations - * ==================================*/ + /** + * Reports post-commit metrics for the given table to Unity Catalog via the Delta-Tables API. + * + * @param tableName the {@code catalog.schema.table} identifier + * @param tableId the table's UC UUID + * @param report the per-commit metrics payload + */ + public void reportMetrics(String tableName, String tableId, UCDeltaModels.CommitReport report) { + withErrorHandling( + () -> { + deltaClient.reportMetrics(tableId, toStorageTableIdentifier(tableName), report); + return null; + }); + } + + /* ============================================================================== + * Catalog browsing (SHOW DATABASES / SHOW TABLES) for the Flink SQL catalog. + * + * These stay on the UC OpenAPI client (SchemasApi / TablesApi): the UC + * Delta-Tables client used everywhere else only serves per-table data-plane + * operations (load, stage, create, commit, metrics) and exposes no + * schema/table listing. Migrate these once that listing is available. + * ============================================================================== */ public List listSchemas() { return withErrorHandling( diff --git a/flink/src/main/java/io/delta/flink/table/postcommit/ReportCommitMetricsListener.java b/flink/src/main/java/io/delta/flink/table/postcommit/ReportCommitMetricsListener.java index 43f15a16b18..c547a254887 100644 --- a/flink/src/main/java/io/delta/flink/table/postcommit/ReportCommitMetricsListener.java +++ b/flink/src/main/java/io/delta/flink/table/postcommit/ReportCommitMetricsListener.java @@ -17,21 +17,20 @@ package io.delta.flink.table.postcommit; import io.delta.flink.table.*; -import io.delta.flink.table.postcommit.po.CommitReport; -import io.delta.flink.table.postcommit.po.DeltaMetricsApi; -import io.delta.flink.table.postcommit.po.ReportDeltaMetricsRequest; import io.delta.kernel.Snapshot; import io.delta.kernel.metrics.FileSizeHistogramResult; import io.delta.kernel.metrics.TransactionMetricsResult; import io.delta.kernel.metrics.TransactionReport; +import io.delta.storage.commit.uccommitcoordinator.UCDeltaModels; import java.util.Arrays; import java.util.List; +import java.util.Optional; import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * Post-commit listener that reports commit metrics to Unity Catalog via {@link DeltaMetricsApi}. + * Post-commit listener that reports commit metrics to Unity Catalog via the Delta-Tables API. * *

This listener is best-effort: failures are logged as warnings but never propagated to the * caller, so a metrics delivery failure cannot break the commit path. @@ -43,8 +42,6 @@ public class ReportCommitMetricsListener implements TableEventListener { private static final Logger LOG = LoggerFactory.getLogger(ReportCommitMetricsListener.class); - private transient DeltaMetricsApi metricsApi; - @Override public void onPostCommit( AbstractKernelTable source, Snapshot snapshot, TransactionReport report) { @@ -59,10 +56,11 @@ public void onPostCommit( } try { - DeltaMetricsApi api = getOrCreateApi((UnityCatalog) catalog); - ReportDeltaMetricsRequest request = buildRequest(tableUUID, report); - - source.executeWithTiming("postcommit.reportmetrics", () -> api.reportMetrics(request)); + UCDeltaModels.CommitReport commitReport = buildReport(report); + UnityCatalog uc = (UnityCatalog) catalog; + source.executeWithTiming( + "postcommit.reportmetrics", + () -> uc.reportMetrics(source.getId(), tableUUID, commitReport)); LOG.info( "Reported commit metrics for table {} version {}", source.getId(), snapshot.getVersion()); @@ -72,40 +70,30 @@ public void onPostCommit( } } - private DeltaMetricsApi getOrCreateApi(UnityCatalog catalog) { - if (metricsApi == null) { - metricsApi = new DeltaMetricsApi(catalog.getApiClient()); - } - return metricsApi; - } - - private ReportDeltaMetricsRequest buildRequest(String tableUUID, TransactionReport txnReport) { + private UCDeltaModels.CommitReport buildReport(TransactionReport txnReport) { TransactionMetricsResult metrics = txnReport.getTransactionMetrics(); - - CommitReport report = - new CommitReport() - .numFilesAdded(metrics.getNumAddFiles()) - .numFilesRemoved(metrics.getNumRemoveFiles()) - .numBytesAdded(metrics.getTotalAddFilesSizeInBytes()) - .numBytesRemoved(metrics.getTotalRemoveFilesSizeInBytes()); - - metrics - .getTableFileSizeHistogram() - .ifPresent( - histogram -> - report.fileSizeHistogram( - toFileSizeHistogram(histogram, txnReport.getCommittedVersion().orElse(null)))); - - return ReportDeltaMetricsRequest.of(tableUUID, report); + Optional histogram = + metrics + .getTableFileSizeHistogram() + .map(h -> toFileSizeHistogram(h, txnReport.getCommittedVersion().orElse(-1L))); + return new UCDeltaModels.CommitReport( + metrics.getNumAddFiles(), + metrics.getNumRemoveFiles(), + metrics.getTotalAddFilesSizeInBytes(), + metrics.getTotalRemoveFilesSizeInBytes(), + Optional.empty() /* numRowsInserted */, + Optional.empty() /* numRowsRemoved */, + Optional.empty() /* numRowsUpdated */, + histogram); } - private static CommitReport.FileSizeHistogram toFileSizeHistogram( - FileSizeHistogramResult result, Long commitVersion) { - return new CommitReport.FileSizeHistogram() - .sortedBinBoundaries(toLongList(result.getSortedBinBoundaries())) - .fileCounts(toLongList(result.getFileCounts())) - .totalBytes(toLongList(result.getTotalBytes())) - .commitVersion(commitVersion); + private static UCDeltaModels.FileSizeHistogram toFileSizeHistogram( + FileSizeHistogramResult result, long commitVersion) { + return new UCDeltaModels.FileSizeHistogram( + toLongList(result.getSortedBinBoundaries()), + toLongList(result.getFileCounts()), + toLongList(result.getTotalBytes()), + commitVersion); } private static List toLongList(long[] array) { diff --git a/flink/src/main/java/io/delta/flink/table/postcommit/po/CommitReport.java b/flink/src/main/java/io/delta/flink/table/postcommit/po/CommitReport.java deleted file mode 100644 index 41d7f531180..00000000000 --- a/flink/src/main/java/io/delta/flink/table/postcommit/po/CommitReport.java +++ /dev/null @@ -1,274 +0,0 @@ -/* - * Copyright (2026) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.delta.flink.table.postcommit.po; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonTypeInfo; -import com.fasterxml.jackson.annotation.JsonTypeName; -import java.util.List; -import java.util.Objects; - -/** - * Commit metrics payload matching the server-side {@code CommitReport} schema. - * - *

The {@code @JsonTypeInfo(WRAPPER_OBJECT)} + {@code @JsonTypeName("commit_report")} annotations - * cause Jackson to serialize this as {@code {"commit_report": { ... }}}, matching the server's - * expected envelope format. - * - *

All fields are optional ({@code NON_NULL} inclusion) so that only populated fields appear in - * the JSON. - */ -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.WRAPPER_OBJECT) -@JsonTypeName("commit_report") -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -public class CommitReport { - - @JsonProperty("num_files_added") - private Long numFilesAdded; - - @JsonProperty("num_files_removed") - private Long numFilesRemoved; - - @JsonProperty("num_bytes_added") - private Long numBytesAdded; - - @JsonProperty("num_bytes_removed") - private Long numBytesRemoved; - - @JsonProperty("num_clustered_bytes_added") - private Long numClusteredBytesAdded; - - @JsonProperty("num_clustered_bytes_removed") - private Long numClusteredBytesRemoved; - - @JsonProperty("file_size_histogram") - private FileSizeHistogram fileSizeHistogram; - - @JsonProperty("num_rows_inserted") - private Long numRowsInserted; - - @JsonProperty("num_rows_removed") - private Long numRowsRemoved; - - @JsonProperty("num_rows_updated") - private Long numRowsUpdated; - - public CommitReport() {} - - // -- accessors & fluent setters -- - - public Long getNumFilesAdded() { - return numFilesAdded; - } - - public CommitReport numFilesAdded(Long numFilesAdded) { - this.numFilesAdded = numFilesAdded; - return this; - } - - public Long getNumFilesRemoved() { - return numFilesRemoved; - } - - public CommitReport numFilesRemoved(Long numFilesRemoved) { - this.numFilesRemoved = numFilesRemoved; - return this; - } - - public Long getNumBytesAdded() { - return numBytesAdded; - } - - public CommitReport numBytesAdded(Long numBytesAdded) { - this.numBytesAdded = numBytesAdded; - return this; - } - - public Long getNumBytesRemoved() { - return numBytesRemoved; - } - - public CommitReport numBytesRemoved(Long numBytesRemoved) { - this.numBytesRemoved = numBytesRemoved; - return this; - } - - public Long getNumClusteredBytesAdded() { - return numClusteredBytesAdded; - } - - public CommitReport numClusteredBytesAdded(Long numClusteredBytesAdded) { - this.numClusteredBytesAdded = numClusteredBytesAdded; - return this; - } - - public Long getNumClusteredBytesRemoved() { - return numClusteredBytesRemoved; - } - - public CommitReport numClusteredBytesRemoved(Long numClusteredBytesRemoved) { - this.numClusteredBytesRemoved = numClusteredBytesRemoved; - return this; - } - - public FileSizeHistogram getFileSizeHistogram() { - return fileSizeHistogram; - } - - public CommitReport fileSizeHistogram(FileSizeHistogram fileSizeHistogram) { - this.fileSizeHistogram = fileSizeHistogram; - return this; - } - - public Long getNumRowsInserted() { - return numRowsInserted; - } - - public CommitReport numRowsInserted(Long numRowsInserted) { - this.numRowsInserted = numRowsInserted; - return this; - } - - public Long getNumRowsRemoved() { - return numRowsRemoved; - } - - public CommitReport numRowsRemoved(Long numRowsRemoved) { - this.numRowsRemoved = numRowsRemoved; - return this; - } - - public Long getNumRowsUpdated() { - return numRowsUpdated; - } - - public CommitReport numRowsUpdated(Long numRowsUpdated) { - this.numRowsUpdated = numRowsUpdated; - return this; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof CommitReport)) return false; - CommitReport that = (CommitReport) o; - return Objects.equals(numFilesAdded, that.numFilesAdded) - && Objects.equals(numFilesRemoved, that.numFilesRemoved) - && Objects.equals(numBytesAdded, that.numBytesAdded) - && Objects.equals(numBytesRemoved, that.numBytesRemoved) - && Objects.equals(numClusteredBytesAdded, that.numClusteredBytesAdded) - && Objects.equals(numClusteredBytesRemoved, that.numClusteredBytesRemoved) - && Objects.equals(fileSizeHistogram, that.fileSizeHistogram) - && Objects.equals(numRowsInserted, that.numRowsInserted) - && Objects.equals(numRowsRemoved, that.numRowsRemoved) - && Objects.equals(numRowsUpdated, that.numRowsUpdated); - } - - @Override - public int hashCode() { - return Objects.hash( - numFilesAdded, - numFilesRemoved, - numBytesAdded, - numBytesRemoved, - numClusteredBytesAdded, - numClusteredBytesRemoved, - fileSizeHistogram, - numRowsInserted, - numRowsRemoved, - numRowsUpdated); - } - - // --------------------------------------------------------------------------- - // FileSizeHistogram - // --------------------------------------------------------------------------- - - /** Histogram of file sizes in a commit, matching the server-side schema. */ - @JsonIgnoreProperties(ignoreUnknown = true) - @JsonInclude(JsonInclude.Include.NON_NULL) - public static class FileSizeHistogram { - - @JsonProperty("sorted_bin_boundaries") - private List sortedBinBoundaries; - - @JsonProperty("file_counts") - private List fileCounts; - - @JsonProperty("total_bytes") - private List totalBytes; - - @JsonProperty("commit_version") - private Long commitVersion; - - public FileSizeHistogram() {} - - public List getSortedBinBoundaries() { - return sortedBinBoundaries; - } - - public FileSizeHistogram sortedBinBoundaries(List sortedBinBoundaries) { - this.sortedBinBoundaries = sortedBinBoundaries; - return this; - } - - public List getFileCounts() { - return fileCounts; - } - - public FileSizeHistogram fileCounts(List fileCounts) { - this.fileCounts = fileCounts; - return this; - } - - public List getTotalBytes() { - return totalBytes; - } - - public FileSizeHistogram totalBytes(List totalBytes) { - this.totalBytes = totalBytes; - return this; - } - - public Long getCommitVersion() { - return commitVersion; - } - - public FileSizeHistogram commitVersion(Long commitVersion) { - this.commitVersion = commitVersion; - return this; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof FileSizeHistogram)) return false; - FileSizeHistogram that = (FileSizeHistogram) o; - return Objects.equals(sortedBinBoundaries, that.sortedBinBoundaries) - && Objects.equals(fileCounts, that.fileCounts) - && Objects.equals(totalBytes, that.totalBytes) - && Objects.equals(commitVersion, that.commitVersion); - } - - @Override - public int hashCode() { - return Objects.hash(sortedBinBoundaries, fileCounts, totalBytes, commitVersion); - } - } -} diff --git a/flink/src/main/java/io/delta/flink/table/postcommit/po/DeltaMetricsApi.java b/flink/src/main/java/io/delta/flink/table/postcommit/po/DeltaMetricsApi.java deleted file mode 100644 index 2cc7552c1e9..00000000000 --- a/flink/src/main/java/io/delta/flink/table/postcommit/po/DeltaMetricsApi.java +++ /dev/null @@ -1,214 +0,0 @@ -/* - * Copyright (2026) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.delta.flink.table.postcommit.po; - -import com.fasterxml.jackson.databind.ObjectMapper; -import io.unitycatalog.client.ApiClient; -import io.unitycatalog.client.ApiException; -import io.unitycatalog.client.ApiResponse; -import java.io.IOException; -import java.io.InputStream; -import java.net.URI; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; -import java.time.Duration; -import java.util.Map; -import java.util.function.Consumer; - -/** - * Hand-written API class for {@code POST /delta/preview/metrics}, following the same structure as - * the OpenAPI-generated {@code *Api} classes (e.g. {@code TablesApi}, {@code SchemasApi}). - * - *

This class is intentionally modeled after the generated code so that it benefits from the same - * {@link ApiClient} infrastructure: authentication, retry via {@code RetryingHttpClient}, - * User-Agent headers, read timeouts, and response interceptors. - * - *

Usage (identical to any generated API class): - * - * 

{@code
- * ApiClient apiClient = ApiClientBuilder.create()
- *     .uri("http://localhost:8080")
- *     .tokenProvider(TokenProvider.create(config))
- *     .build();
- *
- * DeltaMetricsApi metricsApi = new DeltaMetricsApi(apiClient);
- * metricsApi.reportMetrics(new ReportDeltaMetricsRequest(tableId, report));
- * }
- */ -public class DeltaMetricsApi { - - private final HttpClient memberVarHttpClient; - private final ObjectMapper memberVarObjectMapper; - private final String memberVarBaseUri; - private final Consumer memberVarInterceptor; - private final Duration memberVarReadTimeout; - private final Consumer> memberVarResponseInterceptor; - - public DeltaMetricsApi(ApiClient apiClient) { - memberVarHttpClient = apiClient.getHttpClient(); - memberVarObjectMapper = apiClient.getObjectMapper(); - memberVarBaseUri = apiClient.getBaseUri(); - memberVarInterceptor = apiClient.getRequestInterceptor(); - memberVarReadTimeout = apiClient.getReadTimeout(); - memberVarResponseInterceptor = apiClient.getResponseInterceptor(); - } - - // --------------------------------------------------------------------------- - // reportMetrics — POST /delta/preview/metrics - // --------------------------------------------------------------------------- - - /** - * Report commit metrics for a UC-managed Delta table. - * - * @param reportDeltaMetricsRequest the metrics request payload (required) - * @throws ApiException if the API call fails - */ - public void reportMetrics(ReportDeltaMetricsRequest reportDeltaMetricsRequest) - throws ApiException { - reportMetrics(reportDeltaMetricsRequest, null); - } - - /** - * Report commit metrics for a UC-managed Delta table. - * - * @param reportDeltaMetricsRequest the metrics request payload (required) - * @param headers optional additional headers to include in the request - * @throws ApiException if the API call fails - */ - public void reportMetrics( - ReportDeltaMetricsRequest reportDeltaMetricsRequest, Map headers) - throws ApiException { - reportMetricsWithHttpInfo(reportDeltaMetricsRequest, headers); - } - - /** - * Report commit metrics (with full HTTP response info). - * - * @param reportDeltaMetricsRequest the metrics request payload (required) - * @return ApiResponse wrapping the HTTP status, headers, and response - * @throws ApiException if the API call fails - */ - public ApiResponse reportMetricsWithHttpInfo( - ReportDeltaMetricsRequest reportDeltaMetricsRequest) throws ApiException { - return reportMetricsWithHttpInfo(reportDeltaMetricsRequest, null); - } - - /** - * Report commit metrics (with full HTTP response info). - * - * @param reportDeltaMetricsRequest the metrics request payload (required) - * @param headers optional additional headers to include in the request - * @return ApiResponse wrapping the HTTP status, headers, and response - * @throws ApiException if the API call fails - */ - public ApiResponse reportMetricsWithHttpInfo( - ReportDeltaMetricsRequest reportDeltaMetricsRequest, Map headers) - throws ApiException { - HttpRequest.Builder localVarRequestBuilder = - reportMetricsRequestBuilder(reportDeltaMetricsRequest, headers); - try { - HttpResponse localVarResponse = - memberVarHttpClient.send( - localVarRequestBuilder.build(), HttpResponse.BodyHandlers.ofInputStream()); - if (memberVarResponseInterceptor != null) { - memberVarResponseInterceptor.accept(localVarResponse); - } - InputStream localVarResponseBody = null; - try { - if (localVarResponse.statusCode() / 100 != 2) { - throw getApiException("reportMetrics", localVarResponse); - } - localVarResponseBody = localVarResponse.body(); - if (localVarResponseBody != null) { - localVarResponseBody.readAllBytes(); - } - return new ApiResponse<>( - localVarResponse.statusCode(), - localVarResponse.headers().map(), - new ReportDeltaMetricsResponse()); - } finally { - if (localVarResponseBody != null) { - localVarResponseBody.close(); - } - } - } catch (IOException e) { - throw new ApiException(e); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new ApiException(e); - } - } - - private HttpRequest.Builder reportMetricsRequestBuilder( - ReportDeltaMetricsRequest reportDeltaMetricsRequest, Map headers) - throws ApiException { - if (reportDeltaMetricsRequest == null) { - throw new ApiException( - 400, - "Missing the required parameter 'reportDeltaMetricsRequest'" - + " when calling reportMetrics"); - } - - HttpRequest.Builder localVarRequestBuilder = HttpRequest.newBuilder(); - - String localVarPath = "/delta/preview/metrics"; - localVarRequestBuilder.uri(URI.create(memberVarBaseUri + localVarPath)); - - localVarRequestBuilder.header("Content-Type", "application/json"); - localVarRequestBuilder.header("Accept", "application/json"); - - try { - byte[] localVarPostBody = memberVarObjectMapper.writeValueAsBytes(reportDeltaMetricsRequest); - localVarRequestBuilder.method( - "POST", HttpRequest.BodyPublishers.ofByteArray(localVarPostBody)); - } catch (IOException e) { - throw new ApiException(e); - } - - if (memberVarReadTimeout != null) { - localVarRequestBuilder.timeout(memberVarReadTimeout); - } - if (headers != null) { - for (Map.Entry entry : headers.entrySet()) { - localVarRequestBuilder.header(entry.getKey(), entry.getValue()); - } - } - if (memberVarInterceptor != null) { - memberVarInterceptor.accept(localVarRequestBuilder); - } - return localVarRequestBuilder; - } - - // --------------------------------------------------------------------------- - // Error handling (same as generated code) - // --------------------------------------------------------------------------- - - protected ApiException getApiException(String operationId, HttpResponse response) - throws IOException { - String body = response.body() == null ? null : new String(response.body().readAllBytes()); - String message = formatExceptionMessage(operationId, response.statusCode(), body); - return new ApiException(response.statusCode(), message, response.headers(), body); - } - - private String formatExceptionMessage(String operationId, int statusCode, String body) { - if (body == null || body.isEmpty()) { - body = "[no body]"; - } - return operationId + " call failed with: " + statusCode + " - " + body; - } -} diff --git a/flink/src/main/java/io/delta/flink/table/postcommit/po/ReportDeltaMetricsRequest.java b/flink/src/main/java/io/delta/flink/table/postcommit/po/ReportDeltaMetricsRequest.java deleted file mode 100644 index 517c263cb6a..00000000000 --- a/flink/src/main/java/io/delta/flink/table/postcommit/po/ReportDeltaMetricsRequest.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright (2026) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.delta.flink.table.postcommit.po; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; -import java.util.Objects; - -/** - * Top-level request body for {@code POST /api/2.1/unity-catalog/delta/preview/metrics}. - * - *

Serializes to: - * - * 

{@code
- * {
- *   "table_id": "",
- *   "report": {
- *     "commit_report": { ... }
- *   }
- * }
- * }
- * - *

The {@code report} field uses Jackson {@code WRAPPER_OBJECT} polymorphism via {@link - * CommitReport}, so the concrete type name (e.g. {@code "commit_report"}) becomes the JSON wrapper - * key automatically. - */ -@JsonIgnoreProperties(ignoreUnknown = true) -public class ReportDeltaMetricsRequest { - - @JsonProperty("table_id") - private String tableId; - - @JsonProperty("report") - private CommitReport report; - - public ReportDeltaMetricsRequest() {} - - public ReportDeltaMetricsRequest(String tableId, CommitReport report) { - this.tableId = tableId; - this.report = report; - } - - public String getTableId() { - return tableId; - } - - public ReportDeltaMetricsRequest tableId(String tableId) { - this.tableId = tableId; - return this; - } - - public CommitReport getReport() { - return report; - } - - public ReportDeltaMetricsRequest report(CommitReport report) { - this.report = report; - return this; - } - - /** - * Convenience factory for building a request from a table UUID and a {@link CommitReport}. - * - * @param tableId the UC table UUID string - * @param commitReport the commit metrics payload - */ - public static ReportDeltaMetricsRequest of(String tableId, CommitReport commitReport) { - return new ReportDeltaMetricsRequest(tableId, commitReport); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof ReportDeltaMetricsRequest)) return false; - ReportDeltaMetricsRequest that = (ReportDeltaMetricsRequest) o; - return Objects.equals(tableId, that.tableId) && Objects.equals(report, that.report); - } - - @Override - public int hashCode() { - return Objects.hash(tableId, report); - } - - @Override - public String toString() { - return "ReportDeltaMetricsRequest{" + "tableId='" + tableId + '\'' + ", report=" + report + '}'; - } -} diff --git a/flink/src/main/java/io/delta/flink/table/postcommit/po/ReportDeltaMetricsResponse.java b/flink/src/main/java/io/delta/flink/table/postcommit/po/ReportDeltaMetricsResponse.java deleted file mode 100644 index d02f3ee769e..00000000000 --- a/flink/src/main/java/io/delta/flink/table/postcommit/po/ReportDeltaMetricsResponse.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright (2026) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.delta.flink.table.postcommit.po; - -/** - * Response from {@code POST /api/2.1/unity-catalog/delta/preview/metrics}. - * - *

Currently empty — the server returns {@code {}} on success. Fields will be added here if the - * server response evolves. - */ -public class ReportDeltaMetricsResponse {} diff --git a/flink/src/test/java/io/delta/flink/MockHttp.java b/flink/src/test/java/io/delta/flink/MockHttp.java index c6304124fa4..4156612d84b 100644 --- a/flink/src/test/java/io/delta/flink/MockHttp.java +++ b/flink/src/test/java/io/delta/flink/MockHttp.java @@ -20,6 +20,7 @@ import com.github.tomakehurst.wiremock.WireMockServer; import com.github.tomakehurst.wiremock.core.WireMockConfiguration; +import com.github.tomakehurst.wiremock.matching.RequestPatternBuilder; import com.github.tomakehurst.wiremock.stubbing.Scenario; import dev.failsafe.function.CheckedConsumer; import java.net.URI; @@ -32,6 +33,14 @@ public class MockHttp { "/api/2.1/unity-catalog/delta/v1/catalogs/[^/]+/schemas/[^/]+/tables/[^/]+$"; private static final String DELTA_CREDENTIALS_PATH = "/api/2.1/unity-catalog/delta/v1/catalogs/[^/]+/schemas/[^/]+/tables/[^/]+/credentials$"; + private static final String DELTA_CREATE_TABLE_PATH = + "/api/2.1/unity-catalog/delta/v1/catalogs/[^/]+/schemas/[^/]+/tables$"; + private static final String DELTA_STAGING_TABLE_PATH = + "/api/2.1/unity-catalog/delta/v1/catalogs/[^/]+/schemas/[^/]+/staging-tables$"; + private static final String DELTA_STAGING_CREDENTIALS_PATH = + "/api/2.1/unity-catalog/delta/v1/staging-tables/[^/]+/credentials$"; + private static final String DELTA_METRICS_PATH = + "/api/2.1/unity-catalog/delta/v1/catalogs/[^/]+/schemas/[^/]+/tables/[^/]+/metrics$"; private static String loadTableJson(String tableId, String tablePath) { return String.format( @@ -53,13 +62,9 @@ public static void withMock(MockHttp mock, CheckedConsumer body) { public static MockHttp forNewUCTable(String tableId, String tablePath) { Map stubs = new HashMap<>(); - stubs.put( - "/api/2.1/unity-catalog/staging-tables", - String.format("{\"id\": \"%s\", \"staging_location\":\"%s\"}", tableId, tablePath)); - stubs.put("/api/2.1/unity-catalog/tables", "{}"); // For write - stubs.put("/api/2.1/unity-catalog/temporary-table-credentials", "{}"); stubs.put(DELTA_CREDENTIALS_PATH, "{}"); - stubs.put("/api/2.1/unity-catalog/delta/preview/metrics", ""); + stubs.put(DELTA_STAGING_CREDENTIALS_PATH, "{}"); + stubs.put(DELTA_METRICS_PATH, "{}"); MockHttp mock = new MockHttp(stubs, Map.of()); String scenario = "table-lifecycle"; @@ -69,12 +74,22 @@ public static MockHttp forNewUCTable(String tableId, String tablePath) { .whenScenarioStateIs(Scenario.STARTED) .willReturn(notFound())); mock.wireMockServer.stubFor( - post(urlPathMatching("/api/2.1/unity-catalog/staging-tables")) + post(urlPathMatching(DELTA_STAGING_TABLE_PATH)) .inScenario(scenario) .willReturn( okJson( String.format( - "{\"id\": \"%s\", \"staging_location\":\"%s\"}", tableId, tablePath))) + "{\"table-id\": \"%s\", \"table-type\": \"MANAGED\", \"location\": \"%s\", " + + "\"required-protocol\": {\"min-reader-version\": 3, " + + "\"min-writer-version\": 7, \"reader-features\": [\"deletionVectors\"], " + + "\"writer-features\": [\"deletionVectors\"]}, " + + "\"required-properties\": {\"delta.enableDeletionVectors\": \"true\", " + + "\"io.unitycatalog.tableId\": \"%s\"}}", + tableId, tablePath, tableId)))); + mock.wireMockServer.stubFor( + post(urlPathMatching(DELTA_CREATE_TABLE_PATH)) + .inScenario(scenario) + .willReturn(okJson(loadTableJson(tableId, tablePath))) .willSetStateTo("created")); mock.wireMockServer.stubFor( get(urlPathMatching(DELTA_TABLES_PATH)) @@ -100,7 +115,7 @@ public static MockHttp forExistingUCTable( stubs.put( "/api/2.1/unity-catalog/delta/preview/commits", "{\"commits\": [], \"latest_table_version\": 1230}"); - stubs.put("/api/2.1/unity-catalog/delta/preview/metrics", ""); + stubs.put(DELTA_METRICS_PATH, "{}"); return new MockHttp(stubs, Map.of()); } @@ -161,4 +176,16 @@ public int port() { public void verifyCredentialRequests(int count) { wireMockServer.verify(count, getRequestedFor(urlPathMatching(DELTA_CREDENTIALS_PATH))); } + + public void verifyStagingCredentialRequests(int count) { + wireMockServer.verify(count, getRequestedFor(urlPathMatching(DELTA_STAGING_CREDENTIALS_PATH))); + } + + public void verifyPostRequest(String path) { + this.wireMockServer.verify(1, postRequestedFor(urlPathEqualTo(path))); + } + + public void verify(RequestPatternBuilder requestPattern) { + wireMockServer.verify(requestPattern); + } } diff --git a/flink/src/test/java/io/delta/flink/inttest/CatalogManagedTableIntTest.java b/flink/src/test/java/io/delta/flink/inttest/CatalogManagedTableIntTest.java index 2bd1720ea37..20d7fd64d1b 100644 --- a/flink/src/test/java/io/delta/flink/inttest/CatalogManagedTableIntTest.java +++ b/flink/src/test/java/io/delta/flink/inttest/CatalogManagedTableIntTest.java @@ -21,6 +21,7 @@ import io.delta.flink.table.CatalogManagedTable; import io.delta.flink.table.DataColumnVectorView; +import io.delta.flink.table.SchemaUpdateTestUtils; import io.delta.flink.table.UnityCatalog; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.FilteredColumnarBatch; @@ -35,6 +36,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.stream.IntStream; import org.apache.spark.sql.Dataset; @@ -163,4 +165,30 @@ void testWriteNewTables() throws Exception { spark.sql(String.format("SELECT COUNT(1) FROM %s", TEST_NEW_TABLE_NAME)); assertCount(100, count); } + + @IntTest + void testUpdateSchema() throws Exception { + spark.sql( + String.format( + "ALTER TABLE %s SET TBLPROPERTIES ('delta.columnMapping.mode'='name')", + TEST_TABLE_NAME)); + StructType targetSchema = SCHEMA_WITH_ALL_TYPES.add("flink_added_column", StringType.STRING); + UnityCatalog catalog = new UnityCatalog("main", catalogEndpoint, catalogToken); + try (CatalogManagedTable table = + new CatalogManagedTable( + catalog, TEST_TABLE_NAME, Collections.emptyMap(), catalogEndpoint, catalogToken)) { + table.open(); + SchemaUpdateTestUtils.updateSchema(table, targetSchema); + + assertEquals(targetSchema.fieldNames(), table.getSchema().fieldNames()); + assertEquals( + "flink_added_column", + Objects.requireNonNull(catalog.getTableDetail(TEST_TABLE_NAME).getColumns()) + .get(targetSchema.length() - 1) + .getName()); + } + + spark.sql(String.format("REFRESH TABLE %s", TEST_TABLE_NAME)); + assertEquals(targetSchema.fieldNames(), List.of(spark.table(TEST_TABLE_NAME).columns())); + } } diff --git a/flink/src/test/java/io/delta/flink/sink/ConversionsFlinkToDeltaTest.java b/flink/src/test/java/io/delta/flink/sink/ConversionsFlinkToDeltaTest.java index 5de4687b84a..7ffafc60d68 100644 --- a/flink/src/test/java/io/delta/flink/sink/ConversionsFlinkToDeltaTest.java +++ b/flink/src/test/java/io/delta/flink/sink/ConversionsFlinkToDeltaTest.java @@ -193,6 +193,23 @@ void testNullability() { assertTrue(expected.equivalent(actual)); } + @Test + void testPrimitiveDataConversion() { + RowType flinkSchema = + row(f("sm", new SmallIntType()), f("bool", new BooleanType()), f("bin", new BinaryType())); + + StructType deltaSchema = Conversions.FlinkToDelta.schema(flinkSchema); + + GenericRowData rowData = GenericRowData.of((short) 1000, true, new byte[] {1, 2, 3}); + + assertEquals( + Literal.ofShort((short) 1000), Conversions.FlinkToDelta.data(deltaSchema, rowData, 0)); + assertEquals(Literal.ofBoolean(true), Conversions.FlinkToDelta.data(deltaSchema, rowData, 1)); + assertArrayEquals( + (byte[]) Literal.ofBinary(new byte[] {1, 2, 3}).getValue(), + (byte[]) Conversions.FlinkToDelta.data(deltaSchema, rowData, 2).getValue()); + } + @TestAllFlinkTypes void testPrimitiveDataNotNull(LogicalType primitiveType) { RowType flinkType = RowType.of(new LogicalType[] {primitiveType}, new String[] {"id"}); diff --git a/flink/src/test/java/io/delta/flink/sink/DeltaSinkTest.java b/flink/src/test/java/io/delta/flink/sink/DeltaSinkTest.java index 2db4cb0851f..e5d9b534963 100644 --- a/flink/src/test/java/io/delta/flink/sink/DeltaSinkTest.java +++ b/flink/src/test/java/io/delta/flink/sink/DeltaSinkTest.java @@ -49,6 +49,7 @@ import org.apache.flink.table.api.DataTypes; import org.apache.flink.table.data.*; import org.apache.flink.table.runtime.typeutils.InternalTypeInfo; +import org.apache.flink.table.runtime.typeutils.RowDataSerializer; import org.apache.flink.table.types.logical.*; import org.apache.flink.table.types.logical.BinaryType; import org.apache.flink.table.types.logical.BooleanType; @@ -71,6 +72,17 @@ public static void runSink( SerializableFunction supplier, SerializableFunction parser) throws Exception { + runSink(sink, flinkSchema, rounds, supplier, parser, 2); + } + + private static void runSink( + DeltaSink sink, + RowType flinkSchema, + int rounds, + SerializableFunction supplier, + SerializableFunction parser, + int completedCheckpointsBeforeFinish) + throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(5); env.enableCheckpointing(100); @@ -85,7 +97,7 @@ public static void runSink( DataStream input = env.fromSource( - new DelayFinishTestSource<>(dataList, 2), + new DelayFinishTestSource<>(dataList, completedCheckpointsBeforeFinish), WatermarkStrategy.noWatermarks(), "source") .returns(String.class) @@ -101,6 +113,43 @@ public RowData map(String value) throws Exception { env.execute("DeltaSink integration test"); } + @Test + void testBoundedSourceCommitsFinalRows() { + withTempDir( + dir -> { + String tablePath = dir.getPath(); + RowType flinkSchema = RowType.of(new LogicalType[] {new IntType()}, new String[] {"id"}); + DeltaSink deltaSink = + DeltaSink.builder() + .withTablePath(tablePath) + .withFlinkSchema(flinkSchema) + .withPartitionColNames(Collections.emptyList()) + .build(); + + int rounds = 10; + try { + runSink( + deltaSink, + flinkSchema, + rounds, + String::valueOf, + value -> GenericRowData.of(Integer.parseInt(value)), + 0); + } catch (Exception e) { + throw new RuntimeException(e); + } + + verifyTableContent( + tablePath, + (version, actions, props) -> { + List actionList = new ArrayList<>(); + actions.iterator().forEachRemaining(actionList::add); + assertEquals( + rounds, actionList.stream().mapToLong(a -> a.getNumRecords().get()).sum()); + }); + }); + } + @Test void testMiniE2eTestToEmptyTable() { withTempDir( @@ -391,7 +440,9 @@ void testCreateWriterAndCommitter() { .withPartitionColNames(List.of("part")) .build(); - assertNotNull(deltaSink.createWriter(new TestWriterInitContext(1, 1, 1))); + assertNotNull( + deltaSink.createWriter( + new TestWriterInitContext(1, 1, 1, new RowDataSerializer(flinkSchema)))); assertNotNull(deltaSink.createCommitter(new TestCommitterInitContext(1, 1, 1))); }); } diff --git a/flink/src/test/java/io/delta/flink/sink/DeltaSinkWriterTest.java b/flink/src/test/java/io/delta/flink/sink/DeltaSinkWriterTest.java index 03704801437..a571004b990 100644 --- a/flink/src/test/java/io/delta/flink/sink/DeltaSinkWriterTest.java +++ b/flink/src/test/java/io/delta/flink/sink/DeltaSinkWriterTest.java @@ -31,6 +31,8 @@ import io.delta.kernel.defaults.engine.DefaultEngine; import io.delta.kernel.engine.Engine; import io.delta.kernel.internal.InternalScanFileUtils; +import io.delta.kernel.internal.actions.AddFile; +import io.delta.kernel.internal.actions.SingleAction; import io.delta.kernel.internal.data.ScanStateRow; import io.delta.kernel.types.*; import io.delta.kernel.utils.CloseableIterable; @@ -46,6 +48,12 @@ import org.apache.flink.metrics.groups.UnregisteredMetricsGroup; import org.apache.flink.table.data.GenericRowData; import org.apache.flink.table.data.StringData; +import org.apache.flink.table.data.binary.BinaryRowData; +import org.apache.flink.table.data.writer.BinaryRowWriter; +import org.apache.flink.table.runtime.typeutils.RowDataSerializer; +import org.apache.flink.table.types.logical.IntType; +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.flink.table.types.logical.VarCharType; import org.apache.flink.types.RowKind; import org.apache.hadoop.conf.Configuration; import org.junit.jupiter.api.Disabled; @@ -56,6 +64,65 @@ /** JUnit test suite for {@link DeltaSinkWriter}. */ class DeltaSinkWriterTest extends TestHelper { + @Test + void testWriteCopiesReusedInsertRowData() { + assertWriteCopiesReusedRowData(false); + } + + @Test + void testWriteCopiesReusedUpdateAfterRowData() { + assertWriteCopiesReusedRowData(true); + } + + private void assertWriteCopiesReusedRowData(boolean upsert) { + withTempDir( + dir -> { + StructType schema = + new StructType().add("id", IntegerType.INTEGER).add("name", StringType.STRING); + HadoopTable table = + new HadoopTable( + URI.create(dir.getAbsolutePath()), + Collections.emptyMap(), + schema, + Collections.emptyList()); + table.open(); + DeltaSinkWriter sinkWriter = + newSinkWriter( + table, + upsert + ? upsertConf(schema, new int[] {0}) + : new DeltaSinkConf(schema, Collections.emptyMap())); + + BinaryRowData reusedRow = new BinaryRowData(2); + RowKind rowKind = upsert ? RowKind.UPDATE_AFTER : RowKind.INSERT; + BinaryRowWriter rowWriter = new BinaryRowWriter(reusedRow); + rowWriter.writeInt(0, 100); + rowWriter.writeString(1, StringData.fromString("Alice")); + rowWriter.complete(); + reusedRow.setRowKind(rowKind); + sinkWriter.write(reusedRow, new TestSinkWriterContext(0, 0)); + + rowWriter.reset(); + rowWriter.writeInt(0, 101); + rowWriter.writeString(1, StringData.fromString("Bob")); + rowWriter.complete(); + reusedRow.setRowKind(rowKind); + sinkWriter.write(reusedRow, new TestSinkWriterContext(0, 0)); + + DeltaWriterResult result = sinkWriter.prepareCommit().iterator().next(); + AddFile addFile = + new AddFile(result.getDeltaActions().get(0).getStruct(SingleAction.ADD_FILE_ORDINAL)); + List rows = readParquet(dir.toPath().resolve(addFile.getPath()), schema); + + assertEquals(2, rows.size()); + assertEquals( + Set.of("100:Alice", "101:Bob"), + rows.stream() + .map(row -> row.getInt(0) + ":" + row.getString(1)) + .collect(Collectors.toSet())); + }); + } + @Test void testWriteToEmptyTableWithNoPartition() { withTempDir( @@ -76,6 +143,7 @@ void testWriteToEmptyTableWithNoPartition() { .withAttemptNumber(1) .withDeltaTable(table) .withConf(new DeltaSinkConf(schema, Collections.emptyMap())) + .withInputSerializer(testSerializer(schema)) .withMetricGroup(UnregisteredMetricsGroup.createSinkWriterMetricGroup()) .build(); @@ -152,6 +220,16 @@ void testPrepareCommitDoesNotReemitPreviousResultsUpsertMode() throws Exception }); } + @Test + void testPrepareCommitPropagatesAppendWriterFailure() { + assertPrepareCommitPropagatesWriterFailure(/* upsert= */ false); + } + + @Test + void testPrepareCommitPropagatesUpsertWriterFailure() { + assertPrepareCommitPropagatesWriterFailure(/* upsert= */ true); + } + @Test void testWriteToEmptyTableUsingMultiplePartitions() { withTempDir( @@ -172,6 +250,7 @@ void testWriteToEmptyTableUsingMultiplePartitions() { .withAttemptNumber(1) .withDeltaTable(table) .withConf(new DeltaSinkConf(schema, Collections.emptyMap())) + .withInputSerializer(testSerializer(schema)) .withMetricGroup(UnregisteredMetricsGroup.createSinkWriterMetricGroup()) .build(); @@ -231,6 +310,7 @@ void testWriteToExistingTableUsingMultiplePartitions() { .withSubtaskId(0) .withAttemptNumber(1) .withConf(new DeltaSinkConf(schema, Collections.emptyMap())) + .withInputSerializer(testSerializer(schema)) .withMetricGroup(UnregisteredMetricsGroup.createSinkWriterMetricGroup()) .build(); @@ -288,6 +368,7 @@ void testWriterMetricsAfterWriteAndPrepareCommit() { .withAttemptNumber(1) .withDeltaTable(table) .withConf(new DeltaSinkConf(schema, Collections.emptyMap())) + .withInputSerializer(testSerializer(schema)) .withMetricGroup(writerMetrics) .build(); @@ -331,6 +412,7 @@ void testWriterMetricsNumRecordsSendOnBinaryRowData() { .withAttemptNumber(1) .withDeltaTable(table) .withConf(new DeltaSinkConf(schema, Collections.emptyMap())) + .withInputSerializer(testSerializer(schema)) .withMetricGroup(writerMetrics) .build(); @@ -370,6 +452,7 @@ void testMemoryIsStableOnLargeAmountOfPartitions() { .withAttemptNumber(1) .withDeltaTable(table) .withConf(new DeltaSinkConf(schema, Collections.emptyMap())) + .withInputSerializer(testSerializer(schema)) .withMetricGroup(UnregisteredMetricsGroup.createSinkWriterMetricGroup()) .build(); @@ -744,16 +827,68 @@ private static DeltaSinkConf upsertConf(StructType schema, int[] pkOrdinals) { /** Build a {@link DeltaSinkWriter} with the standard test wiring. */ private static DeltaSinkWriter newSinkWriter(HadoopTable table, DeltaSinkConf conf) { + return newSinkWriter(table, conf, testSerializer(conf.getSinkSchema())); + } + + private static DeltaSinkWriter newSinkWriter( + HadoopTable table, DeltaSinkConf conf, RowDataSerializer inputSerializer) { return new DeltaSinkWriter.Builder() .withJobId("test-job") .withSubtaskId(0) .withAttemptNumber(1) .withDeltaTable(table) .withConf(conf) + .withInputSerializer(inputSerializer) .withMetricGroup(UnregisteredMetricsGroup.createSinkWriterMetricGroup()) .build(); } + private static RowDataSerializer testSerializer(StructType schema) { + LogicalType[] types = + schema.fields().stream() + .map( + field -> { + if (field.getDataType() instanceof IntegerType) { + return new IntType(); + } else if (field.getDataType() instanceof StringType) { + return new VarCharType(VarCharType.MAX_LENGTH); + } + throw new IllegalArgumentException( + "Unsupported test data type: " + field.getDataType()); + }) + .toArray(LogicalType[]::new); + return new RowDataSerializer(types); + } + + private void assertPrepareCommitPropagatesWriterFailure(boolean upsert) { + withTempDir( + dir -> { + StructType schema = + new StructType().add("id", IntegerType.INTEGER).add("name", StringType.STRING); + HadoopTable table = + new HadoopTable( + URI.create(dir.getAbsolutePath()), + Collections.emptyMap(), + schema, + Collections.emptyList()); + table.open(); + + DeltaSinkConf conf = + upsert + ? upsertConf(schema, new int[] {0}) + : new DeltaSinkConf(schema, Collections.emptyMap()); + // Keep this malformed row intact so the test exercises failure propagation from the + // asynchronous file writer instead of the input serializer's arity check. + RowDataSerializer malformedInputSerializer = + testSerializer(new StructType().add("id", IntegerType.INTEGER)); + DeltaSinkWriter sinkWriter = newSinkWriter(table, conf, malformedInputSerializer); + + // The missing string field is detected while the cached writer task is completed. + sinkWriter.write(GenericRowData.of(1), new TestSinkWriterContext(0, 0)); + assertThrows(ArrayIndexOutOfBoundsException.class, sinkWriter::prepareCommit); + }); + } + /** * Write the given rows into {@code table} via a {@link DeltaWriterTask} and commit them. After * this returns, {@code table}'s snapshot reflects the new AddFile actions. diff --git a/flink/src/test/java/io/delta/flink/sink/DeltaWriterResultAggregatorTest.java b/flink/src/test/java/io/delta/flink/sink/DeltaWriterResultAggregatorTest.java new file mode 100644 index 00000000000..f8ddaaf688b --- /dev/null +++ b/flink/src/test/java/io/delta/flink/sink/DeltaWriterResultAggregatorTest.java @@ -0,0 +1,107 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.flink.sink; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.util.List; +import org.apache.flink.streaming.api.connector.sink2.CommittableMessage; +import org.apache.flink.streaming.api.connector.sink2.CommittableSummary; +import org.apache.flink.streaming.api.connector.sink2.CommittableWithLineage; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness; +import org.junit.jupiter.api.Test; + +class DeltaWriterResultAggregatorTest { + + @Test + void finishPreservesIncomingCheckpointId() throws Exception { + DeltaWriterResultAggregator aggregator = new DeltaWriterResultAggregator(); + OneInputStreamOperatorTestHarness< + CommittableMessage, CommittableMessage> + harness = new OneInputStreamOperatorTestHarness<>(aggregator); + try { + harness.open(); + long checkpointId = 42; + harness.processElement( + new StreamRecord<>(new CommittableSummary<>(0, 1, checkpointId, 1, 1))); + harness.processElement( + new StreamRecord<>( + new CommittableWithLineage<>(new DeltaWriterResult(), checkpointId, 0))); + + aggregator.finish(); + + List> output = harness.extractOutputValues(); + assertEquals(2, output.size()); + assertInstanceOf(CommittableSummary.class, output.get(0)); + assertEquals(checkpointId, output.get(0).getCheckpointId()); + CommittableWithLineage committable = + assertInstanceOf(CommittableWithLineage.class, output.get(1)); + assertEquals(checkpointId, committable.getCheckpointId()); + DeltaCommittable deltaCommittable = + assertInstanceOf(DeltaCommittable.class, committable.getCommittable()); + assertEquals(checkpointId, deltaCommittable.getCheckpointId()); + } finally { + harness.close(); + } + } + + @Test + void rejectsMixedCheckpointIds() throws Exception { + DeltaWriterResultAggregator aggregator = new DeltaWriterResultAggregator(); + OneInputStreamOperatorTestHarness< + CommittableMessage, CommittableMessage> + harness = new OneInputStreamOperatorTestHarness<>(aggregator); + try { + harness.open(); + harness.processElement(new StreamRecord<>(new CommittableSummary<>(0, 1, 42, 1, 1))); + + IllegalStateException exception = + assertThrows( + IllegalStateException.class, + () -> + harness.processElement( + new StreamRecord<>( + new CommittableWithLineage<>(new DeltaWriterResult(), 43, 0)))); + assertEquals( + "Received results from checkpoints 42 and 43 in one aggregation", exception.getMessage()); + } finally { + harness.close(); + } + } + + @Test + void rejectsMismatchedCheckpointBarrier() throws Exception { + DeltaWriterResultAggregator aggregator = new DeltaWriterResultAggregator(); + OneInputStreamOperatorTestHarness< + CommittableMessage, CommittableMessage> + harness = new OneInputStreamOperatorTestHarness<>(aggregator); + try { + harness.open(); + harness.processElement(new StreamRecord<>(new CommittableSummary<>(0, 1, 42, 1, 1))); + + IllegalStateException exception = + assertThrows(IllegalStateException.class, () -> aggregator.prepareSnapshotPreBarrier(43)); + assertEquals( + "Cannot aggregate results from checkpoint 42 into checkpoint 43", exception.getMessage()); + } finally { + harness.close(); + } + } +} diff --git a/flink/src/test/java/io/delta/flink/sink/TestWriterInitContext.java b/flink/src/test/java/io/delta/flink/sink/TestWriterInitContext.java index f9026b9ba60..6599c90115b 100644 --- a/flink/src/test/java/io/delta/flink/sink/TestWriterInitContext.java +++ b/flink/src/test/java/io/delta/flink/sink/TestWriterInitContext.java @@ -48,11 +48,14 @@ public class TestWriterInitContext implements WriterInitContext { private final SinkWriterMetricGroup metricGroup; private final JobInfo jobInfo; private final TaskInfo taskInfo; + private final TypeSerializer inputSerializer; - public TestWriterInitContext(int subtaskId, int parallelism, int attempt) { + public TestWriterInitContext( + int subtaskId, int parallelism, int attempt, TypeSerializer inputSerializer) { this.subtaskId = subtaskId; this.parallelism = parallelism; this.attempt = attempt; + this.inputSerializer = inputSerializer; this.mailboxExecutor = new MailboxExecutorImpl( @@ -162,8 +165,9 @@ public boolean isObjectReuseEnabled() { } @Override + @SuppressWarnings("unchecked") public TypeSerializer createInputSerializer() { - return null; + return (TypeSerializer) inputSerializer; } @Override diff --git a/flink/src/test/java/io/delta/flink/sink/sql/FlinkSqlTest.java b/flink/src/test/java/io/delta/flink/sink/sql/FlinkSqlTest.java index 3d770e1b315..7b13c54c1da 100644 --- a/flink/src/test/java/io/delta/flink/sink/sql/FlinkSqlTest.java +++ b/flink/src/test/java/io/delta/flink/sink/sql/FlinkSqlTest.java @@ -33,6 +33,52 @@ /** Test suite for Flink SQL integration with Delta. */ class FlinkSqlTest extends TestHelper { + @Test + void testGroupedAggregationPreservesEachRow() { + withTempDir( + dir -> { + TableEnvironment tEnv = + TableEnvironment.create(EnvironmentSettings.newInstance().inBatchMode().build()); + tEnv.executeSql( + "CREATE TEMPORARY TABLE sink (\n" + + " group_id INT,\n" + + " item_count BIGINT\n" + + ") WITH (\n" + + " 'connector' = 'delta',\n" + + " 'table_path' = '" + + dir.getPath() + + "'\n" + + ")"); + + tEnv.executeSql( + "INSERT INTO sink " + + "SELECT group_id, COUNT(*) " + + "FROM (VALUES (1), (2)) AS source(group_id) " + + "GROUP BY group_id") + .await(); + + StructType schema = + new StructType() + .add("group_id", IntegerType.INTEGER) + .add("item_count", LongType.LONG); + verifyTableContent( + dir.getPath(), + (version, addfiles, properties) -> { + List actionList = new ArrayList<>(); + addfiles.iterator().forEachRemaining(actionList::add); + Set records = + actionList.stream() + .flatMap( + addFile -> + readParquet(dir.toPath().resolve(addFile.getPath()), schema) + .stream()) + .map(row -> row.getInt(0) + ":" + row.getLong(1)) + .collect(Collectors.toSet()); + assertEquals(Set.of("1:1", "2:1"), records); + }); + }); + } + @Test void testLoadIntoHadoopTable() { withTempDir( diff --git a/flink/src/test/java/io/delta/flink/sink/sql/FlinkUnityCatalogTableTest.java b/flink/src/test/java/io/delta/flink/sink/sql/FlinkUnityCatalogTableTest.java index 08624d4dc38..13cbd867325 100644 --- a/flink/src/test/java/io/delta/flink/sink/sql/FlinkUnityCatalogTableTest.java +++ b/flink/src/test/java/io/delta/flink/sink/sql/FlinkUnityCatalogTableTest.java @@ -19,13 +19,25 @@ import static org.junit.jupiter.api.Assertions.*; import io.unitycatalog.client.model.ColumnInfo; +import io.unitycatalog.client.model.DataSourceFormat; +import io.unitycatalog.client.model.TableInfo; +import java.net.URI; import java.util.List; +import java.util.Map; +import org.apache.flink.table.api.EnvironmentSettings; import org.apache.flink.table.api.Schema; import org.apache.flink.table.api.Schema.UnresolvedPhysicalColumn; +import org.apache.flink.table.api.TableEnvironment; +import org.apache.flink.table.catalog.CatalogTable; +import org.apache.flink.table.catalog.GenericInMemoryCatalog; +import org.apache.flink.table.catalog.ObjectPath; import org.apache.flink.table.types.AtomicDataType; import org.apache.flink.table.types.KeyValueDataType; -import org.apache.flink.table.types.logical.BinaryType; import org.apache.flink.table.types.logical.MapType; +import org.apache.flink.table.types.logical.NullType; +import org.apache.flink.table.types.logical.SmallIntType; +import org.apache.flink.table.types.logical.TinyIntType; +import org.apache.flink.table.types.logical.VarBinaryType; import org.junit.jupiter.api.Test; /** Test suite for {@link FlinkUnityCatalogTable}. */ @@ -99,7 +111,7 @@ void testBuildSchema() { assertTrue( ((AtomicDataType) ((UnresolvedPhysicalColumn) schema.getColumns().get(2)).getDataType()) .getLogicalType() - instanceof BinaryType); + instanceof VarBinaryType); assertTrue( ((UnresolvedPhysicalColumn) schema.getColumns().get(8)).getDataType() instanceof KeyValueDataType); @@ -108,4 +120,79 @@ void testBuildSchema() { .getLogicalType() instanceof MapType); } + + @Test + void testNarrowIntegerTypes() { + assertEquals( + new TinyIntType(true), + FlinkUnityCatalogTable.fromJson("{\"type\":\"byte\",\"nullable\":true}").getLogicalType()); + assertEquals( + new SmallIntType(false), + FlinkUnityCatalogTable.fromJson("{\"type\":\"short\",\"nullable\":false}") + .getLogicalType()); + } + + @Test + void testResolveTableWithVoidColumn() throws Exception { + TableInfo tableInfo = + new TableInfo() + .name("events") + .columns( + List.of( + column("id"), + new ColumnInfo() + .name("pending") + .typeJson( + "{\"name\":\"pending\",\"type\":\"void\",\"nullable\":true," + + "\"metadata\":{}}"))) + .dataSourceFormat(DataSourceFormat.DELTA) + .properties(Map.of()); + FlinkUnityCatalogTable table = + new FlinkUnityCatalogTable(tableInfo, URI.create("https://example.com"), "token"); + GenericInMemoryCatalog catalog = new GenericInMemoryCatalog("test_catalog"); + catalog.createTable(new ObjectPath("default", "events"), table, false); + TableEnvironment tableEnvironment = + TableEnvironment.create(EnvironmentSettings.newInstance().inBatchMode().build()); + tableEnvironment.registerCatalog("test_catalog", catalog); + + assertEquals( + new NullType(), + tableEnvironment + .from("`test_catalog`.`default`.`events`") + .getResolvedSchema() + .getColumnDataTypes() + .get(1) + .getLogicalType()); + } + + @Test + void testPartitionMetadata() { + List columns = + List.of(column("id"), column("day").partitionIndex(1), column("region").partitionIndex(0)); + TableInfo tableInfo = + new TableInfo() + .name("events") + .columns(columns) + .dataSourceFormat(DataSourceFormat.DELTA) + .properties(Map.of()); + + FlinkUnityCatalogTable table = + new FlinkUnityCatalogTable(tableInfo, URI.create("https://example.com"), "token"); + + assertTrue(table.isPartitioned()); + assertEquals(List.of("region", "day"), table.getPartitionKeys()); + assertEquals("region,day", table.getOptions().get("partitions")); + + CatalogTable copiedTable = (CatalogTable) table.copy(); + assertEquals(table.getPartitionKeys(), copiedTable.getPartitionKeys()); + assertEquals(table.getPartitionKeys(), table.copy(Map.of()).getPartitionKeys()); + } + + private static ColumnInfo column(String name) { + return new ColumnInfo() + .name(name) + .typeJson( + String.format( + "{\"name\":\"%s\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}", name)); + } } diff --git a/flink/src/test/java/io/delta/flink/sink/sql/FlinkUnityCatalogTest.java b/flink/src/test/java/io/delta/flink/sink/sql/FlinkUnityCatalogTest.java index cf681cf7437..1bf17b97b67 100644 --- a/flink/src/test/java/io/delta/flink/sink/sql/FlinkUnityCatalogTest.java +++ b/flink/src/test/java/io/delta/flink/sink/sql/FlinkUnityCatalogTest.java @@ -16,12 +16,15 @@ package io.delta.flink.sink.sql; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import io.delta.flink.MockHttp; import java.util.UUID; import org.apache.flink.table.catalog.ObjectPath; +import org.apache.flink.table.catalog.stats.CatalogColumnStatistics; +import org.apache.flink.table.catalog.stats.CatalogTableStatistics; import org.junit.jupiter.api.Test; /** Test suite for {@link FlinkUnityCatalog}. */ @@ -53,6 +56,18 @@ void tableExistsReturnsTrueWhenTableFormatIsUnsupported() { mock -> assertTrue(catalog(mock).tableExists(TABLE_PATH))); } + @Test + void statisticsAreUnknownWhenTheCatalogDoesNotProvideThem() { + MockHttp.withMock( + MockHttp.forExistingUCTable(UUID.randomUUID().toString(), "s3://bucket/default/tbl"), + mock -> { + FlinkUnityCatalog catalog = catalog(mock); + assertEquals(CatalogTableStatistics.UNKNOWN, catalog.getTableStatistics(TABLE_PATH)); + assertEquals( + CatalogColumnStatistics.UNKNOWN, catalog.getTableColumnStatistics(TABLE_PATH)); + }); + } + private static FlinkUnityCatalog catalog(MockHttp mock) { FlinkUnityCatalog catalog = new FlinkUnityCatalog("main", "default", mock.uri().toString(), "token"); diff --git a/flink/src/test/java/io/delta/flink/table/AbstractKernelTableTest.java b/flink/src/test/java/io/delta/flink/table/AbstractKernelTableTest.java index 4d3cec32940..1656c48ea01 100644 --- a/flink/src/test/java/io/delta/flink/table/AbstractKernelTableTest.java +++ b/flink/src/test/java/io/delta/flink/table/AbstractKernelTableTest.java @@ -16,6 +16,9 @@ package io.delta.flink.table; +import static io.delta.kernel.internal.util.ColumnMapping.COLUMN_MAPPING_ID_KEY; +import static io.delta.kernel.internal.util.ColumnMapping.COLUMN_MAPPING_MODE_KEY; +import static io.delta.kernel.internal.util.ColumnMapping.COLUMN_MAPPING_PHYSICAL_NAME_KEY; import static io.delta.kernel.internal.util.Utils.singletonCloseableIterator; import static org.junit.jupiter.api.Assertions.*; @@ -34,8 +37,13 @@ import io.delta.kernel.internal.actions.AddFile; import io.delta.kernel.internal.actions.SingleAction; import io.delta.kernel.internal.util.Utils; +import io.delta.kernel.types.ArrayType; +import io.delta.kernel.types.FieldMetadata; import io.delta.kernel.types.IntegerType; +import io.delta.kernel.types.LongType; +import io.delta.kernel.types.MapType; import io.delta.kernel.types.StringType; +import io.delta.kernel.types.StructField; import io.delta.kernel.types.StructType; import io.delta.kernel.utils.CloseableIterable; import io.delta.kernel.utils.CloseableIterator; @@ -44,7 +52,10 @@ import java.net.URI; import java.nio.file.AccessDeniedException; import java.util.*; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.flink.util.InstantiationUtil; import org.apache.hadoop.conf.Configuration; import org.junit.jupiter.api.Test; @@ -52,6 +63,47 @@ /** JUnit 6 test suite for AbstractKernelTable. */ class AbstractKernelTableTest extends TestHelper { + private static final class BarrierLocalFileSystemTable extends LocalFileSystemTable { + private final CyclicBarrier snapshotBarrier; + private final AtomicBoolean blockNextSnapshot = new AtomicBoolean(); + + private BarrierLocalFileSystemTable( + URI tablePath, Map conf, StructType schema, CyclicBarrier snapshotBarrier) { + super(tablePath, conf, schema, List.of()); + this.snapshotBarrier = snapshotBarrier; + setCacheManager(new SnapshotCacheManager.NoCacheManager()); + } + + void blockNextSnapshot() { + blockNextSnapshot.set(true); + } + + @Override + protected Snapshot loadLatestSnapshot() { + Snapshot snapshot = super.loadLatestSnapshot(); + if (blockNextSnapshot.compareAndSet(true, false)) { + try { + snapshotBarrier.await(30, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } catch (BrokenBarrierException | TimeoutException e) { + throw new RuntimeException(e); + } + } + return snapshot; + } + } + + private static Throwable awaitFailure(Future future) throws Exception { + try { + future.get(30, TimeUnit.SECONDS); + return null; + } catch (ExecutionException e) { + return e.getCause(); + } + } + /** * Helper method to create a test table with default configuration. * @@ -163,6 +215,243 @@ void testTableStoredConfIntoDeltaLogs() { }); } + @Test + void testUpdateSchemaPreservesAndAssignsColumnMappingMetadata() { + StructType initialSchema = new StructType().add("id", IntegerType.INTEGER); + FieldMetadata userMetadata = + FieldMetadata.builder().putString("user.metadata", "preserved").build(); + StructType nestedType = + new StructType().add(new StructField("city", StringType.STRING, true, userMetadata)); + StructType targetSchema = + new StructType() + // Deliberately omit existing mapping metadata. The table snapshot remains + // authoritative. + .add("id", IntegerType.INTEGER) + .add(new StructField("name", StringType.STRING, true, userMetadata)) + .add(new StructField("address", nestedType, true, userMetadata)); + + withTestTable( + initialSchema, + Collections.emptyList(), + Map.of(COLUMN_MAPPING_MODE_KEY, "name"), + table -> { + StructField oldId = table.getSchema().at(0); + long oldIdMapping = oldId.getMetadata().getLong(COLUMN_MAPPING_ID_KEY); + String oldPhysicalName = oldId.getMetadata().getString(COLUMN_MAPPING_PHYSICAL_NAME_KEY); + long oldVersion = table.snapshot().orElseThrow().getVersion(); + + table.updateSchema(targetSchema); + + StructType updatedSchema = table.getSchema(); + assertEquals(List.of("id", "name", "address"), updatedSchema.fieldNames()); + assertEquals(oldVersion + 1, table.snapshot().orElseThrow().getVersion()); + assertEquals( + oldIdMapping, updatedSchema.at(0).getMetadata().getLong(COLUMN_MAPPING_ID_KEY)); + assertEquals( + oldPhysicalName, + updatedSchema.at(0).getMetadata().getString(COLUMN_MAPPING_PHYSICAL_NAME_KEY)); + + StructField name = updatedSchema.at(1); + assertNotNull(name.getMetadata().getLong(COLUMN_MAPPING_ID_KEY)); + assertNotNull(name.getMetadata().getString(COLUMN_MAPPING_PHYSICAL_NAME_KEY)); + assertEquals("preserved", name.getMetadata().getString("user.metadata")); + + StructField nestedCity = ((StructType) updatedSchema.at(2).getDataType()).at(0); + assertNotNull(nestedCity.getMetadata().getLong(COLUMN_MAPPING_ID_KEY)); + assertNotNull(nestedCity.getMetadata().getString(COLUMN_MAPPING_PHYSICAL_NAME_KEY)); + assertEquals("preserved", nestedCity.getMetadata().getString("user.metadata")); + + table.updateSchema(targetSchema); + assertEquals(oldVersion + 1, table.snapshot().orElseThrow().getVersion()); + }); + } + + @Test + void testUpdateSchemaPreservesCallerSuppliedColumnMappingMetadata() { + StructType initialSchema = new StructType().add("id", IntegerType.INTEGER); + FieldMetadata suppliedMappingMetadata = + FieldMetadata.builder() + .putLong(COLUMN_MAPPING_ID_KEY, 999L) + .putString(COLUMN_MAPPING_PHYSICAL_NAME_KEY, "caller-physical-name") + .build(); + StructType targetSchema = + initialSchema.add( + new StructField("name", StringType.STRING, true, suppliedMappingMetadata)); + + withTestTable( + initialSchema, + Collections.emptyList(), + Map.of(COLUMN_MAPPING_MODE_KEY, "name"), + table -> { + table.updateSchema(targetSchema); + + StructField addedField = table.getSchema().at(1); + assertEquals(999L, addedField.getMetadata().getLong(COLUMN_MAPPING_ID_KEY)); + assertEquals( + "caller-physical-name", + addedField.getMetadata().getString(COLUMN_MAPPING_PHYSICAL_NAME_KEY)); + }); + } + + @Test + void testUpdateSchemaRejectsUnsupportedChanges() { + StructType nestedType = new StructType().add("value", IntegerType.INTEGER); + StructType initialSchema = + new StructType().add("id", IntegerType.INTEGER).add("payload", nestedType); + + withTestTable( + initialSchema, + Collections.emptyList(), + Map.of(COLUMN_MAPPING_MODE_KEY, "name"), + table -> { + long initialVersion = table.snapshot().orElseThrow().getVersion(); + List invalidTargets = + List.of( + new StructType().add("id", IntegerType.INTEGER), + new StructType().add("payload", nestedType).add("id", IntegerType.INTEGER), + new StructType().add("id", LongType.LONG).add("payload", nestedType), + new StructType() + .add(new StructField("id", IntegerType.INTEGER, false)) + .add("payload", nestedType), + new StructType() + .add("id", IntegerType.INTEGER) + .add("payload", new StructType().add("renamed", IntegerType.INTEGER)), + initialSchema.add(new StructField("required", StringType.STRING, false))); + + invalidTargets.forEach( + target -> + assertThrows(IllegalArgumentException.class, () -> table.updateSchema(target))); + assertEquals(initialVersion, table.snapshot().orElseThrow().getVersion()); + }); + } + + @Test + void testUpdateSchemaValidatesArrayAndMapTypesRecursively() { + StructType nestedType = + new StructType().add("first", StringType.STRING).add("last", StringType.STRING); + StructType reorderedNestedType = + new StructType().add("last", StringType.STRING).add("first", StringType.STRING); + ArrayType arrayType = new ArrayType(nestedType, true); + MapType mapType = new MapType(StringType.STRING, nestedType, true); + StructType initialSchema = new StructType().add("items", arrayType).add("lookup", mapType); + + withTestTable( + initialSchema, + Collections.emptyList(), + Map.of(COLUMN_MAPPING_MODE_KEY, "name"), + table -> { + long initialVersion = table.snapshot().orElseThrow().getVersion(); + + // The logical schema remains equal even though the snapshot has column mapping metadata. + table.updateSchema(initialSchema); + assertEquals(initialVersion, table.snapshot().orElseThrow().getVersion()); + + List invalidTargets = + List.of( + new StructType() + .add("items", new ArrayType(nestedType, false)) + .add("lookup", mapType), + new StructType() + .add("items", new ArrayType(reorderedNestedType, true)) + .add("lookup", mapType), + new StructType() + .add("items", arrayType) + .add("lookup", new MapType(StringType.STRING, nestedType, false)), + new StructType() + .add("items", arrayType) + .add("lookup", new MapType(StringType.STRING, reorderedNestedType, true))); + + invalidTargets.forEach( + target -> + assertThrows(IllegalArgumentException.class, () -> table.updateSchema(target))); + assertEquals(initialVersion, table.snapshot().orElseThrow().getVersion()); + }); + } + + @Test + void testConcurrentSchemaUpdatesDoNotMergeDifferentTargets() { + withTempDir( + dir -> { + StructType initialSchema = new StructType().add("id", IntegerType.INTEGER); + Map properties = + Map.of( + COLUMN_MAPPING_MODE_KEY, + "name", + "fs.file.impl", + "org.apache.hadoop.fs.RawLocalFileSystem", + "fs.file.impl.disable.cache", + "true"); + CyclicBarrier snapshotBarrier = new CyclicBarrier(2); + ExecutorService executor = Executors.newFixedThreadPool(2); + + // Concurrent commits use separate HDFSLogStore instances, exercising cross-instance + // locking. + try (BarrierLocalFileSystemTable first = + new BarrierLocalFileSystemTable( + dir.toURI(), properties, initialSchema, snapshotBarrier); + BarrierLocalFileSystemTable second = + new BarrierLocalFileSystemTable( + dir.toURI(), properties, initialSchema, snapshotBarrier)) { + first.open(); + second.open(); + + StructType sharedTarget = initialSchema.add("name", StringType.STRING); + first.blockNextSnapshot(); + second.blockNextSnapshot(); + Future firstShared = executor.submit(() -> first.updateSchema(sharedTarget)); + Future secondShared = executor.submit(() -> second.updateSchema(sharedTarget)); + assertNull(awaitFailure(firstShared)); + assertNull(awaitFailure(secondShared)); + assertEquals(1L, first.snapshot().orElseThrow().getVersion()); + + StructType emailTarget = sharedTarget.add("email", StringType.STRING); + StructType phoneTarget = sharedTarget.add("phone", StringType.STRING); + first.blockNextSnapshot(); + second.blockNextSnapshot(); + Future emailUpdate = executor.submit(() -> first.updateSchema(emailTarget)); + Future phoneUpdate = executor.submit(() -> second.updateSchema(phoneTarget)); + Throwable emailFailure = awaitFailure(emailUpdate); + Throwable phoneFailure = awaitFailure(phoneUpdate); + + assertEquals( + 1L, Stream.of(emailFailure, phoneFailure).filter(Objects::nonNull).count()); + assertInstanceOf( + IllegalArgumentException.class, emailFailure == null ? phoneFailure : emailFailure); + Snapshot latest = first.snapshot().orElseThrow(); + assertEquals(2L, latest.getVersion()); + assertTrue( + List.of(emailTarget.fieldNames(), phoneTarget.fieldNames()) + .contains(latest.getSchema().fieldNames())); + } finally { + executor.shutdownNow(); + } + }); + } + + @Test + void testUpdateSchemaRequiresColumnMappingAndOpenTable() { + StructType initialSchema = new StructType().add("id", IntegerType.INTEGER); + StructType targetSchema = initialSchema.add("name", StringType.STRING); + + withTempDir( + dir -> { + LocalFileSystemTable unopened = + new LocalFileSystemTable(dir.toURI(), Map.of(), initialSchema, List.of()); + assertThrows(IllegalStateException.class, () -> unopened.updateSchema(targetSchema)); + }); + + withTestTable( + initialSchema, + Collections.emptyList(), + table -> { + long initialVersion = table.snapshot().orElseThrow().getVersion(); + RuntimeException error = + assertThrows(RuntimeException.class, () -> table.updateSchema(targetSchema)); + assertTrue(error.getMessage().toLowerCase(Locale.ROOT).contains("column mapping")); + assertEquals(initialVersion, table.snapshot().orElseThrow().getVersion()); + }); + } + /** * Reads a single merged Hadoop configuration value from the engine produced by {@link * AbstractKernelTable#createEngine()}. {@code createEngine} builds the configuration internally diff --git a/flink/src/test/java/io/delta/flink/table/CatalogManagedTableTest.java b/flink/src/test/java/io/delta/flink/table/CatalogManagedTableTest.java index b41ac483f7d..fb9a0a80a69 100644 --- a/flink/src/test/java/io/delta/flink/table/CatalogManagedTableTest.java +++ b/flink/src/test/java/io/delta/flink/table/CatalogManagedTableTest.java @@ -16,7 +16,15 @@ package io.delta.flink.table; +import static com.github.tomakehurst.wiremock.client.WireMock.matchingJsonPath; +import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlPathMatching; +import static io.delta.kernel.internal.util.ColumnMapping.COLUMN_MAPPING_ID_KEY; +import static io.delta.kernel.internal.util.ColumnMapping.COLUMN_MAPPING_MODE_KEY; +import static io.delta.kernel.internal.util.ColumnMapping.COLUMN_MAPPING_PHYSICAL_NAME_KEY; +import static io.delta.kernel.unitycatalog.UCCatalogManagedClient.UC_TABLE_ID_KEY; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import io.delta.flink.MockHttp; @@ -24,9 +32,13 @@ import io.delta.kernel.internal.SnapshotImpl; import io.delta.kernel.internal.tablefeatures.TableFeatures; import io.delta.kernel.types.IntegerType; +import io.delta.kernel.types.StringType; +import io.delta.kernel.types.StructField; import io.delta.kernel.types.StructType; +import io.delta.kernel.utils.CloseableIterable; import java.net.URI; import java.util.*; +import java.util.concurrent.TimeUnit; import org.junit.jupiter.api.Test; /** JUnit test suite for CCv2Table. */ @@ -56,21 +68,78 @@ void testCreateCatalogManagedTable() { AbstractKernelTable.normalize(URI.create(dir.getAbsolutePath())), table.getTablePath()); - CatalogManagedTable.CATALOG_MANAGED_REQUIRED_FEATURES_CONF.forEach( - (key, value) -> assertEquals(value, table.conf.catalogConf().get(key))); - assertEquals(uuid, table.conf.catalogConf().get("io.unitycatalog.tableId")); + assertEquals(uuid, table.conf.catalogConf().get(UC_TABLE_ID_KEY)); + assertEquals("true", table.conf.catalogConf().get("delta.enableDeletionVectors")); SnapshotImpl snapshot = (SnapshotImpl) table.snapshot().get(); - assertEquals(uuid, snapshot.getTableProperties().get("io.unitycatalog.tableId")); + assertEquals(uuid, snapshot.getTableProperties().get(UC_TABLE_ID_KEY)); + assertEquals( + "true", snapshot.getTableProperties().get("delta.enableDeletionVectors")); assertTrue( - CatalogManagedTable.CATALOG_MANAGED_REQUIRED_FEATURES_CONF.keySet().stream() - .map(s -> s.replace("delta.feature.", "")) - .allMatch( - s -> - snapshot - .getProtocol() - .supportsFeature(TableFeatures.getTableFeature(s)))); + snapshot + .getProtocol() + .supportsFeature(TableFeatures.DELETION_VECTORS_RW_FEATURE)); + } + }); + }); + } + + @Test + void testCreateCatalogManagedTableWithAmbientCredentials() { + withTempDir( + dir -> { + String uuid = UUID.randomUUID().toString(); + StructType schema = new StructType().add("id", IntegerType.INTEGER); + MockHttp.withMock( + MockHttp.forNewUCTable(uuid, dir.getAbsolutePath()), + mockHttp -> { + try (CatalogManagedTable table = + new CatalogManagedTable( + new UnityCatalog( + "main", mockHttp.uri(), "", /* credentialVendingEnabled = */ false), + "main.abc.def", + Map.of(TableConf.CREDENTIALS_SOURCE.key(), "ambient"), + schema, + List.of(), + mockHttp.uri(), + "")) { + table.open(); + mockHttp.verifyStagingCredentialRequests(0); + } + }); + }); + } + + @Test + void testCommitCatalogManagedTable() { + withTempDir( + dir -> { + String uuid = UUID.randomUUID().toString(); + StructType schema = new StructType().add("id", IntegerType.INTEGER); + MockHttp.withMock( + MockHttp.forNewUCTable(uuid, dir.getAbsolutePath()), + mockHttp -> { + try (CatalogManagedTable table = + new CatalogManagedTable( + new UnityCatalog("main", mockHttp.uri(), ""), + "main.abc.def", + Collections.emptyMap(), + schema, + List.of(), + mockHttp.uri(), + "")) { + table.open(); + + table.commit( + CloseableIterable.emptyIterable(), "test-app", 1L, Collections.emptyMap()); + + mockHttp.verifyPostRequest( + "/api/2.1/unity-catalog/delta/v1/catalogs/main/schemas/abc/tables/def"); + + // Let asynchronous post-commit work finish before the mock and temp table close. + table.generalThreadPool.shutdown(); + assertTrue(table.generalThreadPool.awaitTermination(10, TimeUnit.SECONDS)); } }); }); @@ -92,4 +161,45 @@ void testSerializability() throws Exception { checkSerializability(table); } } + + @Test + void testUpdateSchemaUsesDeltaTablesSetColumns() { + withTempDir( + dir -> { + String uuid = UUID.randomUUID().toString(); + StructType initialSchema = new StructType().add("id", IntegerType.INTEGER); + StructType targetSchema = initialSchema.add("name", StringType.STRING); + MockHttp.withMock( + MockHttp.forNewUCTable(uuid, dir.getAbsolutePath()), + dummyHttp -> { + try (CatalogManagedTable table = + new CatalogManagedTable( + new UnityCatalog("main", dummyHttp.uri(), ""), + "main.abc.def", + Map.of(COLUMN_MAPPING_MODE_KEY, "name"), + initialSchema, + List.of(), + dummyHttp.uri(), + "")) { + table.open(); + table.updateSchema(targetSchema); + + assertEquals(List.of("id", "name"), table.getSchema().fieldNames()); + StructField addedField = table.getSchema().at(1); + assertNotNull(addedField.getMetadata().getLong(COLUMN_MAPPING_ID_KEY)); + assertNotNull( + addedField.getMetadata().getString(COLUMN_MAPPING_PHYSICAL_NAME_KEY)); + + dummyHttp.verify( + postRequestedFor( + urlPathMatching( + "/api/2.1/unity-catalog/delta/v1/catalogs/main/schemas/abc/tables/def")) + .withRequestBody( + matchingJsonPath("$.updates[?(@.action == 'set-columns')]")) + .withRequestBody( + matchingJsonPath("$.updates[?(@.action == 'add-commit')]"))); + } + }); + }); + } } diff --git a/flink/src/test/java/io/delta/flink/table/SchemaUpdateTestUtils.java b/flink/src/test/java/io/delta/flink/table/SchemaUpdateTestUtils.java new file mode 100644 index 00000000000..fa5bc07d3b4 --- /dev/null +++ b/flink/src/test/java/io/delta/flink/table/SchemaUpdateTestUtils.java @@ -0,0 +1,29 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.flink.table; + +import io.delta.kernel.types.StructType; + +/** Test-only access to package-private table primitives. */ +public final class SchemaUpdateTestUtils { + + private SchemaUpdateTestUtils() {} + + public static void updateSchema(AbstractKernelTable table, StructType targetSchema) { + table.updateSchema(targetSchema); + } +} diff --git a/flink/src/test/java/io/delta/flink/table/UnityCatalogTest.java b/flink/src/test/java/io/delta/flink/table/UnityCatalogTest.java index da9548704b5..5a6618abb20 100644 --- a/flink/src/test/java/io/delta/flink/table/UnityCatalogTest.java +++ b/flink/src/test/java/io/delta/flink/table/UnityCatalogTest.java @@ -24,6 +24,7 @@ import io.delta.kernel.types.*; import io.delta.kernel.unitycatalog.UCTableIdentifier; import java.net.URI; +import java.util.List; import java.util.Map; import java.util.UUID; import org.junit.jupiter.api.Test; @@ -103,4 +104,32 @@ void testGetCredentials() { mockHttp.verifyCredentialRequests(1); }); } + + @Test + void testCreateTableDescriptorIncludesStagingProperties() { + withTempDir( + dir -> { + String tableName = "main.default.tbl"; + String uuid = UUID.randomUUID().toString(); + MockHttp.withMock( + MockHttp.forNewUCTable(uuid, dir.getAbsolutePath()), + mockHttp -> { + UnityCatalog uc = new UnityCatalog("main", mockHttp.uri(), ""); + uc.open(); + + uc.createTable( + tableName, + new StructType().add("id", IntegerType.INTEGER), + List.of(), + Map.of(), + descriptor -> { + assertEquals(uuid, descriptor.getUuid()); + assertEquals(Map.of(), descriptor.getStorageProperties()); + assertEquals( + "true", + descriptor.getRequiredProperties().get("delta.enableDeletionVectors")); + }); + }); + }); + } } diff --git a/hudi/src/main/scala/org/apache/spark/sql/delta/hudi/HudiConverter.scala b/hudi/src/main/scala/org/apache/spark/sql/delta/hudi/HudiConverter.scala index 10caf282d07..469ea1d7481 100644 --- a/hudi/src/main/scala/org/apache/spark/sql/delta/hudi/HudiConverter.scala +++ b/hudi/src/main/scala/org/apache/spark/sql/delta/hudi/HudiConverter.scala @@ -273,7 +273,7 @@ class HudiConverter ) val actionIterators = - DeltaFileProviderUtils.parallelReadAndParseDeltaFilesAsIterator(spark, log, commits) + DeltaFileProviderUtils.parallelReadAndParseDeltaFilesAsIterator(spark, commits) actionIterators.foreach { actionsIter => try { actionsIter.grouped(actionBatchSize).foreach { actions => diff --git a/iceberg/src/main/scala/org/apache/spark/sql/delta/icebergShaded/IcebergConverter.scala b/iceberg/src/main/scala/org/apache/spark/sql/delta/icebergShaded/IcebergConverter.scala index a2986299cb5..b00addab505 100644 --- a/iceberg/src/main/scala/org/apache/spark/sql/delta/icebergShaded/IcebergConverter.scala +++ b/iceberg/src/main/scala/org/apache/spark/sql/delta/icebergShaded/IcebergConverter.scala @@ -395,7 +395,7 @@ class IcebergConverter // If `rowTrackingBackfillRequired` is enabled, the Row Tracking Backfill command // will first be triggered, adding one or more ROW TRACKING BACKFILL commits // to the table. After backfilling completes, here we must regenerate the entire - // Iceberg metadata to keep the snapshot version in sync. + // Iceberg metadata to keep snapshot sequence numbers in sync with Delta versions. case _ if rowTrackingJustEnabled => val commitInfos = createSnapshotsForReplaceTable( snapshotToConvert, prevConvertedSnapshotOpt, icebergTxn, catalogTable, @@ -430,7 +430,7 @@ class IcebergConverter ) val actionIterators = - DeltaFileProviderUtils.parallelReadAndParseDeltaFilesAsIterator(spark, log, commits) + DeltaFileProviderUtils.parallelReadAndParseDeltaFilesAsIterator(spark, commits) var deltaVersion = prevSnapshot.version val commitInfos = actionIterators.map { actionsIter => try { @@ -843,7 +843,24 @@ class IcebergConverter } addFiles = addBuffer.toSeq removeFiles = removeBuffer.toSeq - val dataChange = DataChange(dataChangeBits) + val fileActionsDataChange = DataChange(dataChangeBits) + val dataChange = CommitInfo.commitChangedData(commitInfo) match { + // Nothing to convert either way, so the recorded value cannot say otherwise. + case Some(_) if dataChangeBits == 0 => DataChange.Empty + case Some(true) => DataChange.All + case Some(false) => DataChange.None + // No recorded value: fall back to what the file actions say. + case None => fileActionsDataChange + } + + if (dataChange != fileActionsDataChange) { + throw new IllegalStateException( + s"Iceberg conversion dataChange mismatch at version ${targetSnapshot.version}: " + + s"commitInfo says $dataChange but file actions say $fileActionsDataChange " + + s"(operation=${commitInfo.map(_.operation).getOrElse("")}, " + + s"hasAdd=${addFiles.nonEmpty}, hasRemove=${removeFiles.nonEmpty}, " + + s"hasDv=$hasDv)") + } (addFiles.nonEmpty, removeFiles.nonEmpty, dataChange) match { case (true, false, DataChange.All) if !hasDv => @@ -992,7 +1009,12 @@ class DummySnapshotWithAllFilesSupport( override def allFiles: Dataset[AddFile] = { SparkSession.getActiveSession.map { spark => import org.apache.spark.sql.delta.implicits._ - val replay = new InMemoryLogReplay(None, None) + val replay = new InMemoryLogReplay( + minFileRetentionTimestamp = None, + minSetTransactionRetentionTimestamp = None, + tableRoot = deltaLog.dataPath, + useDeletionVectorObjectIdentity = FileAction.useDeletionVectorObjectIdentity( + metadata, protocol, spark)) val baseVersion = version - 1 if (baseVersion < 0) { // No prior commit exists replay.append(0, txnInfo.finalActionsToCommit.iterator) diff --git a/iceberg/src/main/scala/org/apache/spark/sql/delta/icebergShaded/IcebergTransactionUtils.scala b/iceberg/src/main/scala/org/apache/spark/sql/delta/icebergShaded/IcebergTransactionUtils.scala index f958d0b94ab..03641d2769b 100644 --- a/iceberg/src/main/scala/org/apache/spark/sql/delta/icebergShaded/IcebergTransactionUtils.scala +++ b/iceberg/src/main/scala/org/apache/spark/sql/delta/icebergShaded/IcebergTransactionUtils.scala @@ -19,7 +19,7 @@ package org.apache.spark.sql.delta.icebergShaded import java.nio.ByteBuffer import java.time.Instant import java.time.format.DateTimeParseException -import java.util.{Base64, List => JList, UUID} +import java.util.{Base64, List => JList} import scala.collection.JavaConverters._ import scala.reflect.runtime.universe @@ -37,7 +37,6 @@ import shadedForDelta.org.apache.iceberg.Metrics import shadedForDelta.org.apache.iceberg.StructLike import shadedForDelta.org.apache.iceberg.catalog.{Namespace, TableIdentifier => IcebergTableIdentifier} import shadedForDelta.org.apache.iceberg.hive.HiveCatalog -import shadedForDelta.org.apache.iceberg.unityCatalog.{IcebergSnapshotIdGenerator, UnityCatalogTableOperations} import shadedForDelta.org.apache.iceberg.util.DateTimeUtil import org.apache.spark.sql.catalyst.{InternalRow, TableIdentifier => SparkTableIdentifier} @@ -218,8 +217,8 @@ object IcebergTransactionUtils getMicrosSinceEpoch(utcInstant) } case _ => - throw DeltaErrors.universalFormatConversionFailedException( - version, "iceberg", "Unexpected partition data type " + elemType) + throw DeltaErrors.universalFormatConversionFailedUnexpectedPartitionDataTypeException( + version, "iceberg", elemType) } } @@ -352,28 +351,6 @@ object IcebergTransactionUtils } } - /** - * Pre-assigns a deterministic snapshot ID for a given Iceberg transaction. - * - * This method ensures that when committing through Unity Catalog, the next - * snapshot ID is deterministically assigned based on the Delta version - * and the table identifier. - */ - def preAssignSnapshotIdForTxn(txn: IcebergTransaction, version: Long, tableId: String): Unit = { - txn match { - case baseTxn: BaseTransaction => baseTxn.underlyingOps() match { - case unityCatalogOps: UnityCatalogTableOperations => - unityCatalogOps.setPreAssignedNextSnapshotId( - IcebergSnapshotIdGenerator.encode( - version, - UUID.fromString(tableId))) - case _ => throw new IllegalStateException( - "Expected tableOps to be UnityCatalogTableOperations") - } - case _ => throw new IllegalStateException("Expected txn to be BaseTransaction") - } - } - /** * Use reflection to set lastSequenceNumber in TableMetadata * @param txn diff --git a/iceberg/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/IcebergRESTCatalogPlanningClient.scala b/iceberg/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/IcebergRESTCatalogPlanningClient.scala index 30c94349601..62371b6af5c 100644 --- a/iceberg/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/IcebergRESTCatalogPlanningClient.scala +++ b/iceberg/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/IcebergRESTCatalogPlanningClient.scala @@ -360,7 +360,7 @@ class IcebergRESTCatalogPlanningClient( // Request planning for current snapshot. snapshotId = 0 means "use current snapshot" // in the Iceberg REST API spec. Time-travel queries are not yet supported. - val builder = new PlanTableScanRequest.Builder() + val builder = PlanTableScanRequest.builder() .withSnapshotId(CURRENT_SNAPSHOT_ID) // Set caseSensitive=false (defaults to true in spec) to match Spark's case-insensitive // column handling. Server should validate and block requests with caseSensitive=true. @@ -378,18 +378,11 @@ class IcebergRESTCatalogPlanningClient( builder.withSelect(columnNames.asJava) } - val request = builder.build() - - // Iceberg 1.11 adds withMinRowsRequested() support. For now, manually inject the field. - val requestJson = sparkLimitOption match { - case Some(limit) => - implicit val formats: Formats = DefaultFormats - val jsonAst = parse(PlanTableScanRequestParser.toJson(request)) - val modifiedJson = jsonAst merge JObject("min-rows-requested" -> JLong(limit.toLong)) - compact(render(modifiedJson)) - case None => - PlanTableScanRequestParser.toJson(request) + sparkLimitOption.foreach { limit => + builder.withMinRowsRequested(limit.toLong) } + + val requestJson = PlanTableScanRequestParser.toJson(builder.build()) val httpPost = new HttpPost(planTableScanUri) httpPost.setEntity(new StringEntity(requestJson, ContentType.APPLICATION_JSON)) val httpResponse = httpClient.execute(httpPost) diff --git a/iceberg/src/test/java/shadedForDelta/org/apache/iceberg/rest/IcebergRESTCatalogAdapterWithPlanSupport.java b/iceberg/src/test/java/shadedForDelta/org/apache/iceberg/rest/IcebergRESTCatalogAdapterWithPlanSupport.java index 3fefbefd3f2..c54e4c4b603 100644 --- a/iceberg/src/test/java/shadedForDelta/org/apache/iceberg/rest/IcebergRESTCatalogAdapterWithPlanSupport.java +++ b/iceberg/src/test/java/shadedForDelta/org/apache/iceberg/rest/IcebergRESTCatalogAdapterWithPlanSupport.java @@ -325,7 +325,7 @@ private PlanTableScanResponse handlePlanTableScan( TableIdentifier tableIdent = extractTableIdentifier(request.path()); LOG.debug("Table identifier: {}", tableIdent); - // Parse request (min-rows-requested is natively supported in Iceberg 1.11.0) + // Parse request (min-rows-requested is natively supported since Iceberg 1.11.0) PlanTableScanRequest planRequest = parsePlanRequest(request); Long minRowsRequested = planRequest.minRowsRequested(); LOG.debug("Plan request parsed: snapshotId={}, minRowsRequested={}", @@ -405,10 +405,9 @@ private PlanTableScanResponse handlePlanTableScan( LOG.debug("Table has {} partition specs", specsById.size()); // Build response (Pattern 1: COMPLETED with direct tasks) - return PlanTableScanResponse.builder() + return PlanTableScanResponse.builder(specsById) .withPlanStatus(PlanStatus.COMPLETED) .withFileScanTasks(tasksToReturn) - .withSpecsById(specsById) .build(); } } diff --git a/iceberg/src/test/java/shadedForDelta/org/apache/iceberg/rest/IcebergRESTServletWithPlanSupport.java b/iceberg/src/test/java/shadedForDelta/org/apache/iceberg/rest/IcebergRESTServletWithPlanSupport.java index c018f380e44..bdf81a12bf5 100644 --- a/iceberg/src/test/java/shadedForDelta/org/apache/iceberg/rest/IcebergRESTServletWithPlanSupport.java +++ b/iceberg/src/test/java/shadedForDelta/org/apache/iceberg/rest/IcebergRESTServletWithPlanSupport.java @@ -263,7 +263,7 @@ private String injectStorageCredentials( // Parse original JSON Map responseMap = mapper.readValue(originalJson, Map.class); - // Build storage-credentials structure (Iceberg 1.11.0 requires "prefix" field) + // Build storage-credentials structure (Iceberg REST requires "prefix" field) Map credWrapper = new HashMap<>(); credWrapper.put("prefix", "*"); credWrapper.put("config", new HashMap<>(credentials)); diff --git a/iceberg/src/test/scala/org/apache/spark/sql/delta/ConvertIcebergToDeltaSuite.scala b/iceberg/src/test/scala/org/apache/spark/sql/delta/ConvertIcebergToDeltaSuite.scala index 270c5407d6a..82d379bf2d4 100644 --- a/iceberg/src/test/scala/org/apache/spark/sql/delta/ConvertIcebergToDeltaSuite.scala +++ b/iceberg/src/test/scala/org/apache/spark/sql/delta/ConvertIcebergToDeltaSuite.scala @@ -73,17 +73,17 @@ trait ConvertIcebergToDeltaUtils extends SharedSparkSession { .filterNot(identity).map(_ => "NO STATISTICS").getOrElse("") - override def beforeAll(): Unit = { + override protected def beforeAll(): Unit = { warehousePath = Utils.createTempDir() super.beforeAll() } - override def afterAll(): Unit = { + override protected def afterAll(): Unit = { super.afterAll() if (warehousePath != null) Utils.deleteRecursively(warehousePath) } - override def afterEach(): Unit = { + override protected def afterEach(): Unit = { sql(s"DROP TABLE IF EXISTS $table") super.afterEach() } diff --git a/iceberg/src/test/scala/org/apache/spark/sql/delta/uniform/UniFormIcebergVerifier.scala b/iceberg/src/test/scala/org/apache/spark/sql/delta/uniform/UniFormIcebergVerifier.scala index f6c836cb96e..77d4802fd37 100644 --- a/iceberg/src/test/scala/org/apache/spark/sql/delta/uniform/UniFormIcebergVerifier.scala +++ b/iceberg/src/test/scala/org/apache/spark/sql/delta/uniform/UniFormIcebergVerifier.scala @@ -265,7 +265,8 @@ class UniFormIcebergVerifier( .flatMap { manifest => manifest.content() match { case ManifestContent.DATA => - ManifestFiles.read(manifest, icebergTable.io()).asScala.toSeq + ManifestFiles.read(manifest, icebergTable.io(), icebergTable.specs()) + .asScala.toSeq case _ => Seq.empty } } @@ -293,7 +294,8 @@ class UniFormIcebergVerifier( .map( _.deleteManifests(icebergTable.io()) .asScala - .flatMap(ManifestFiles.readDeleteManifest(_, icebergTable.io(), null).asScala.toSeq) + .flatMap(ManifestFiles.readDeleteManifest( + _, icebergTable.io(), icebergTable.specs()).asScala.toSeq) .map(df => (df.referencedDataFile, df.path)) .toMap) .getOrElse(Map.empty) diff --git a/icebergShaded/src/main/java/org/apache/iceberg/Delegates.java b/icebergShaded/src/main/java/org/apache/iceberg/Delegates.java index a97a4163d94..62c303c80d1 100644 --- a/icebergShaded/src/main/java/org/apache/iceberg/Delegates.java +++ b/icebergShaded/src/main/java/org/apache/iceberg/Delegates.java @@ -24,7 +24,7 @@ import java.util.Set; /** - * This class is directly copied from iceberg repo 1.11.0 with the following change: + * This class is directly copied from iceberg repo 1.12.0 with the following change: * Changes: L40: suppressFirstRowId returns the file directly instead of nulling out its first_row_id. * UniForm assigns each Iceberg data file's first_row_id from the Delta baseRowId; the * upstream behavior of suppressing the explicitly-set first_row_id for newly-added @@ -231,6 +231,11 @@ public Map nanValueCounts() { return wrapped.nanValueCounts(); } + @Override + public Map avgValueSizes() { + return wrapped.avgValueSizes(); + } + @Override public Map lowerBounds() { return wrapped.lowerBounds(); diff --git a/icebergShaded/src/main/java/org/apache/iceberg/MetadataUpdate.java b/icebergShaded/src/main/java/org/apache/iceberg/MetadataUpdate.java index a35994353d4..9abcd125314 100644 --- a/icebergShaded/src/main/java/org/apache/iceberg/MetadataUpdate.java +++ b/icebergShaded/src/main/java/org/apache/iceberg/MetadataUpdate.java @@ -30,7 +30,7 @@ /** Represents a change to table or view metadata. */ /** - * This class is directly copied from iceberg repo 1.11.0 with following changes + * This class is directly copied from iceberg repo 1.12.0 with following changes * Changes: L91 Added back the deprecated API (removed in 1.11.0) * public AddSchema(Schema schema, int lastColumnId) */ diff --git a/icebergShaded/src/main/java/org/apache/iceberg/PartitionSpec.java b/icebergShaded/src/main/java/org/apache/iceberg/PartitionSpec.java index 9e3619cc8dc..43fe4139e16 100644 --- a/icebergShaded/src/main/java/org/apache/iceberg/PartitionSpec.java +++ b/icebergShaded/src/main/java/org/apache/iceberg/PartitionSpec.java @@ -50,7 +50,16 @@ *

Partition data is produced by transforming columns in a table. Each column transform is * represented by a named {@link PartitionField}. * - * This class is directly copied from iceberg repo 1.11.0; The only change is this sets checkConflicts + *

Partition specs are created using a builder obtained from {@link #builderFor(Schema)}: + * + * 

{@code
+ * PartitionSpec spec = PartitionSpec.builderFor(schema)
+ *     .hour("ts")
+ *     .bucket("id", 10)
+ *     .build();
+ * }
+ * + * This class is directly copied from iceberg repo 1.12.0; The only change is this sets checkConflicts * to false by default for partition spec converted from Delta to honor the field id assigned by Delta */ public class PartitionSpec implements Serializable { @@ -133,15 +142,8 @@ public StructType partitionType() { List structFields = Lists.newArrayListWithExpectedSize(fields.length); for (PartitionField field : fields) { - Type sourceType = schema.findType(field.sourceId()); - Type resultType = field.transform().getResultType(sourceType); - - // When the source field has been dropped we cannot determine the type - if (sourceType == null) { - resultType = Types.UnknownType.get(); - } - - structFields.add(Types.NestedField.optional(field.fieldId(), field.name(), resultType)); + structFields.add( + Types.NestedField.optional(field.fieldId(), field.name(), resultType(field))); } this.lazyPartitionType = Types.StructType.of(structFields); @@ -186,14 +188,7 @@ public Class[] javaClasses() { if (field.transform() instanceof UnknownTransform) { classes[i] = Object.class; } else { - Type sourceType = schema.findType(field.sourceId()); - if (null == sourceType) { - // When the source field has been dropped we cannot determine the type - sourceType = Types.UnknownType.get(); - } - - Type result = field.transform().getResultType(sourceType); - classes[i] = result.typeId().javaClass(); + classes[i] = resultType(field).typeId().javaClass(); } } @@ -205,6 +200,17 @@ public Class[] javaClasses() { return lazyJavaClasses; } + private Type resultType(PartitionField field) { + Type sourceType = schema.findType(field.sourceId()); + if (sourceType == null) { + // When the source field has been dropped, the source type has been lost + // Transforms with a fixed result type still work, so use unknown for source + sourceType = Types.UnknownType.get(); + } + + return field.transform().getResultType(sourceType); + } + @SuppressWarnings("unchecked") private T get(StructLike data, int pos, Class javaClass) { return data.get(pos, (Class) javaClass); diff --git a/icebergShaded/src/main/java/org/apache/iceberg/TableMetadata.java b/icebergShaded/src/main/java/org/apache/iceberg/TableMetadata.java index 636bc3e9da5..b4d2307a7c7 100644 --- a/icebergShaded/src/main/java/org/apache/iceberg/TableMetadata.java +++ b/icebergShaded/src/main/java/org/apache/iceberg/TableMetadata.java @@ -51,7 +51,7 @@ import org.apache.iceberg.util.SerializableSupplier; /** - * This class is directly copied from iceberg repo 1.11.0 with following changes + * This class is directly copied from iceberg repo 1.12.0 with following changes * Change: L602 add back the deprecated API * public TableMetadata updateSchema(Schema newSchema, int newLastColumnId) * L848 add the sql conf check to bypass snap sequenceNumber check @@ -69,6 +69,8 @@ public class TableMetadata implements Serializable { static final int DEFAULT_TABLE_FORMAT_VERSION = 2; static final int SUPPORTED_TABLE_FORMAT_VERSION = 4; static final int MIN_FORMAT_VERSION_ROW_LINEAGE = 3; + static final int MIN_FORMAT_VERSION_PARQUET_MANIFESTS = 4; + static final int MIN_FORMAT_VERSION_OPTIONAL_LOCATION = 4; static final int INITIAL_SPEC_ID = 0; static final int INITIAL_SORT_ORDER_ID = 1; static final int INITIAL_SCHEMA_ID = 0; @@ -149,7 +151,7 @@ static TableMetadata newTableMetadata( // Validate the metrics configuration. Note: we only do this on new tables to we don't // break existing tables. - MetricsConfig.fromProperties(properties).validateReferencedColumns(schema); + MetricsConfig.validate(properties, schema); PropertyUtil.validateCommitProperties(properties); @@ -323,6 +325,16 @@ public String toString() { SUPPORTED_TABLE_FORMAT_VERSION); Preconditions.checkArgument( formatVersion == 1 || uuid != null, "UUID is required in format v%s", formatVersion); + boolean locationOptional = formatVersion >= MIN_FORMAT_VERSION_OPTIONAL_LOCATION; + Preconditions.checkArgument( + locationOptional || location != null, + "Table location is required in format v%s", + formatVersion); + Preconditions.checkArgument( + !locationOptional || location == null || LocationUtil.hasScheme(location), + "Invalid table location in format v%s, must be absolute: %s", + formatVersion, + location); Preconditions.checkArgument( formatVersion > 1 || lastSequenceNumber == 0, "Sequence number must be 0 in v1: %s", @@ -1354,6 +1366,10 @@ public Builder setRef(String name, SnapshotRef ref) { Snapshot snapshot = snapshotsById.get(snapshotId); ValidationException.check( snapshot != null, "Cannot set %s to unknown snapshot: %s", name, snapshotId); + ValidationException.check( + !SnapshotRef.MAIN_BRANCH.equals(name) || ref.isBranch(), + "Cannot set %s to a tag, it must be a branch", + SnapshotRef.MAIN_BRANCH); if (SnapshotRef.MAIN_BRANCH.equals(name)) { this.currentSnapshotId = ref.snapshotId(); @@ -1543,10 +1559,10 @@ public Builder addEncryptionKey(EncryptedKey key) { } public Builder removeEncryptionKey(String keyId) { - boolean removed = encryptionKeys.removeIf(key -> key.keyId().equals(keyId)); - keysById.remove(keyId); + EncryptedKey removedKey = keysById.remove(keyId); - if (removed) { + if (removedKey != null) { + encryptionKeys.remove(removedKey); changes.add(new MetadataUpdate.RemoveEncryptionKey(keyId)); } @@ -1631,7 +1647,7 @@ public TableMetadata build() { .flatMap(List::stream) .collect(Collectors.toList()), nextRowId, - encryptionKeys, + ImmutableList.copyOf(encryptionKeys), discardChanges ? ImmutableList.of() : ImmutableList.copyOf(changes)); } diff --git a/icebergShaded/src/main/java/org/apache/iceberg/hive/HiveCatalog.java b/icebergShaded/src/main/java/org/apache/iceberg/hive/HiveCatalog.java index b3f733d06d3..bbc17a79500 100644 --- a/icebergShaded/src/main/java/org/apache/iceberg/hive/HiveCatalog.java +++ b/icebergShaded/src/main/java/org/apache/iceberg/hive/HiveCatalog.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -36,6 +37,7 @@ import org.apache.hadoop.hive.metastore.api.PrincipalType; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.UnknownDBException; +import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.iceberg.BaseMetastoreTableOperations; import org.apache.iceberg.CatalogProperties; import org.apache.iceberg.CatalogUtil; @@ -52,6 +54,7 @@ import org.apache.iceberg.encryption.KeyManagementClient; import org.apache.iceberg.exceptions.*; import org.apache.iceberg.hadoop.HadoopFileIO; +import org.apache.iceberg.io.CloseableGroup; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting; import org.apache.iceberg.relocated.com.google.common.base.MoreObjects; @@ -73,7 +76,7 @@ import org.slf4j.LoggerFactory; /** - * This class is directly copied from iceberg 1.11.0; The only change made is + * This class is directly copied from iceberg 1.12.0; The only change made is * 1. * accept metadataUpdates in constructor and pass to HiveTableOperations * to support using schema/partitionSpec with field ids assigned by Delta lake @@ -102,6 +105,7 @@ public class HiveCatalog extends BaseMetastoreViewCatalog private boolean listAllTables = false; private boolean uniqueTableLocation; private Map catalogProperties; + private CloseableGroup closeableGroup; // HACK-HACK This is newly added private List metadataUpdates = new ArrayList(); @@ -154,6 +158,12 @@ public void initialize(String inputName, Map properties) { CatalogProperties.UNIQUE_TABLE_LOCATION_DEFAULT); this.clients = new CachedClientPool(conf, properties); + + this.closeableGroup = new CloseableGroup(); + closeableGroup.addCloseable(fileIO); + closeableGroup.addCloseable(keyManagementClient); + closeableGroup.addCloseable(metricsReporter()); + closeableGroup.setSuppressCloseFailure(true); } @Override @@ -173,18 +183,19 @@ public List listTables(Namespace namespace) { String database = namespace.level(0); try { - List tableNames = clients.run(client -> client.getAllTables(database)); List tableIdentifiers; if (listAllTables) { + List tableNames = clients.run(client -> client.getAllTables(database)); tableIdentifiers = tableNames.stream() .map(t -> TableIdentifier.of(namespace, t)) .collect(Collectors.toList()); } else { + // Retrieving only the matching table names from HMS via server-side filter tableIdentifiers = - listIcebergTables( - tableNames, namespace, BaseMetastoreTableOperations.ICEBERG_TABLE_TYPE_VALUE); + listIcebergTablesByFilter( + namespace, BaseMetastoreTableOperations.ICEBERG_TABLE_TYPE_VALUE); } LOG.debug( @@ -360,6 +371,37 @@ private List listIcebergTables( .collect(Collectors.toList()); } + /** + * Lists the {@link TableIdentifier} of all Iceberg tables under the given database. + * + * @param namespace the namespace corresponding to the database + * @param tableTypeValue the value of {@code parameters.table_type} to filter on. + */ + private List listIcebergTablesByFilter( + Namespace namespace, String tableTypeValue) throws TException, InterruptedException { + String database = namespace.level(0); + // HMS normalizes the `table_type` parameter value to upper case when persisting it + // (e.g. "iceberg" is stored as "ICEBERG"), so the filter value must be upper-cased to match. + // Note: `like` (rather than `=`) is used intentionally. Some HMS backends (e.g. Derby and + // Oracle, where PARAM_VALUE is a CLOB) do not support `=` comparison on property values and + // fail the filter query (see HIVE-21614); newer HMS versions internally rewrite `=` to `like` + // for this reason. As the value carries no `%`/`_` wildcards, `like` is equivalent to `=` here. + String filter = + hive_metastoreConstants.HIVE_FILTER_FIELD_PARAMS + + BaseMetastoreTableOperations.TABLE_TYPE_PROP + + " like \"" + + tableTypeValue.toUpperCase(Locale.ROOT) + + "\""; + + List icebergTableNames = + clients.run( + client -> client.listTableNamesByFilter(database, filter, (short) -1 /* no limit */)); + + return icebergTableNames.stream() + .map(tableName -> TableIdentifier.of(namespace, tableName)) + .collect(Collectors.toList()); + } + @SuppressWarnings("checkstyle:CyclomaticComplexity") private void renameTableOrView( TableIdentifier from, @@ -862,10 +904,8 @@ protected Map properties() { @Override public void close() throws IOException { - super.close(); - - if (keyManagementClient != null) { - keyManagementClient.close(); + if (closeableGroup != null) { + closeableGroup.close(); } } diff --git a/icebergShaded/src/main/java/org/apache/iceberg/hive/HiveTableOperations.java b/icebergShaded/src/main/java/org/apache/iceberg/hive/HiveTableOperations.java index f6771a1c4c2..0ae7d577801 100644 --- a/icebergShaded/src/main/java/org/apache/iceberg/hive/HiveTableOperations.java +++ b/icebergShaded/src/main/java/org/apache/iceberg/hive/HiveTableOperations.java @@ -59,7 +59,7 @@ /** * TODO we should be able to extract some more commonalities to BaseMetastoreTableOperations to * avoid code duplication between this class and Metacat Tables. - * This class is directly copied from iceberg 1.11.0; The only change made are + * This class is directly copied from iceberg 1.12.0; The only change made are * 1) accept metadataUpdates in constructor apply those before writing metadata * to support using schema/partitionSpec with field ids assigned by Delta lake; * 2) handle NoSuchIcebergTableException in doRefresh to regard a table entry diff --git a/icebergShaded/src/main/java/org/apache/iceberg/unityCatalog/IcebergSnapshotIdGenerator.java b/icebergShaded/src/main/java/org/apache/iceberg/unityCatalog/IcebergSnapshotIdGenerator.java deleted file mode 100644 index 3ff00a13cbb..00000000000 --- a/icebergShaded/src/main/java/org/apache/iceberg/unityCatalog/IcebergSnapshotIdGenerator.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (2021) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.iceberg.unityCatalog; - -import java.math.BigInteger; -import java.util.UUID; - -import org.apache.iceberg.relocated.com.google.common.base.Preconditions; - -// Utils class for generating deterministic Iceberg snapshot ID based on Delta version and table ID -public class IcebergSnapshotIdGenerator { - private static final BigInteger PRIME = new BigInteger("11400714819323198485"); - private static final BigInteger TWO_POW_64 = BigInteger.ONE.shiftLeft(64); - private static final BigInteger LONG_MAX_VALUE = BigInteger.valueOf(Long.MAX_VALUE); - - public static long encode(long version, UUID uuid) { - Preconditions.checkArgument(version >= 0 && version < 1L << 44); - // Get 20-bit UUID fragment - long uuidFragment = uuid.getLeastSignificantBits() & 0xFFFFF; - - // Store 43-bit version + 20-bit UUID fragment - BigInteger shiftedVersion = BigInteger.valueOf(version).shiftLeft(20); - BigInteger combined = shiftedVersion.or(BigInteger.valueOf(uuidFragment)); - - // Scrambling - BigInteger scrambled = combined.multiply(PRIME).and(TWO_POW_64.subtract(BigInteger.ONE)); - - // Final result - return scrambled.and(LONG_MAX_VALUE).longValue(); - } - - public static long decode(long encoded) { - Preconditions.checkArgument(encoded >= 0); - // Modular inverse of PRIME mod 2^64 - BigInteger inversePrime = PRIME.modInverse(TWO_POW_64); - - // Undo scrambling - BigInteger combined = BigInteger.valueOf(encoded) - .multiply(inversePrime) - .mod(TWO_POW_64) - // Ensure it is positive. - .and(LONG_MAX_VALUE); - - // Extract version - return combined.shiftRight(20).longValue(); - } -} diff --git a/icebergShaded/src/main/java/org/apache/iceberg/unityCatalog/UnityCatalogTableOperations.java b/icebergShaded/src/main/java/org/apache/iceberg/unityCatalog/UnityCatalogTableOperations.java index fe287c488cf..3173452c0c0 100644 --- a/icebergShaded/src/main/java/org/apache/iceberg/unityCatalog/UnityCatalogTableOperations.java +++ b/icebergShaded/src/main/java/org/apache/iceberg/unityCatalog/UnityCatalogTableOperations.java @@ -58,13 +58,6 @@ public class UnityCatalogTableOperations extends BaseMetastoreTableOperations { private final Optional baseMetadataLocation; - // Pre-assigned snapshot ID to be used for the next snapshot creation. - // When set, this value will override the automatically generated snapshot ID - // during the next commit operation. This allows for deterministic snapshot ID - // generation for DBI based on delta version. - private Optional preAssignedNextSnapshotId; - private boolean preAssignedNextSnapshotIdConsumed; - private List metadataUpdates; private Optional> lastWrittenTableMetadataWithLocation; @@ -79,8 +72,6 @@ public UnityCatalogTableOperations( this.metadataUpdates = metadataUpdates; this.baseMetadataLocation = baseMetadataLocation; this.lastWrittenTableMetadataWithLocation = Optional.empty(); - this.preAssignedNextSnapshotId = Optional.empty(); - this.preAssignedNextSnapshotIdConsumed = false; } @Override @@ -228,17 +219,4 @@ private String loadTableMetadataLocation() { } - public void setPreAssignedNextSnapshotId(long preAssignedNextSnapshotId) { - this.preAssignedNextSnapshotId = Optional.of(preAssignedNextSnapshotId); - this.preAssignedNextSnapshotIdConsumed = false; - } - - @Override - public long newSnapshotId() { - if (preAssignedNextSnapshotId.isPresent()) { - this.preAssignedNextSnapshotIdConsumed = true; - return preAssignedNextSnapshotId.get(); - } - return super.newSnapshotId(); - } } diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/PartitionKeyType.java b/kernel/kernel-api/src/main/java/io/delta/kernel/PartitionKeyType.java new file mode 100644 index 00000000000..19a40dff132 --- /dev/null +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/PartitionKeyType.java @@ -0,0 +1,28 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.kernel; + +import io.delta.kernel.annotation.Evolving; + +/** + * Indicates whether the partition-column keys of a {@code partitionValues} map are logical or + * physical column names. + */ +@Evolving +public enum PartitionKeyType { + LOGICAL, + PHYSICAL +} diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/Transaction.java b/kernel/kernel-api/src/main/java/io/delta/kernel/Transaction.java index 97252f2cfa9..6813b640fef 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/Transaction.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/Transaction.java @@ -253,6 +253,23 @@ static void blockIfVariantDataTypeIsDefined(StructType tableSchema) { } } + /** + * Equivalent to {@link #getWriteContext(Engine, Row, Map, PartitionKeyType)} with {@link + * PartitionKeyType#LOGICAL}; {@code partitionValues} keys are logical partition column names. + * + * @param engine {@link Engine} instance to use. + * @param transactionState The transaction state + * @param partitionValues The partition values for the data, keyed by logical partition column + * name. If the table is un-partitioned, the map should be empty. + * @return {@link DataWriteContext} containing metadata about where and how the data for partition + * should be written. The target directory and the returned partition-value keys use the + * physical partition column names. + */ + static DataWriteContext getWriteContext( + Engine engine, Row transactionState, Map partitionValues) { + return getWriteContext(engine, transactionState, partitionValues, PartitionKeyType.LOGICAL); + } + /** * Get the context for writing data into a table. The context tells the connector where the data * should be written. For partitioned table context is generated per partition. So, the connector @@ -261,14 +278,19 @@ static void blockIfVariantDataTypeIsDefined(StructType tableSchema) { * * @param engine {@link Engine} instance to use. * @param transactionState The transaction state - * @param partitionValues The partition values for the data, keyed by logical partition column - * name. If the table is un-partitioned, the map should be empty. + * @param partitionValues The partition values for the data, keyed per {@code partitionKeyType}. + * If the table is un-partitioned, the map should be empty. + * @param partitionKeyType whether {@code partitionValues} keys are logical or physical partition + * column names. * @return {@link DataWriteContext} containing metadata about where and how the data for partition * should be written. The target directory and the returned partition-value keys use the * physical partition column names. */ static DataWriteContext getWriteContext( - Engine engine, Row transactionState, Map partitionValues) { + Engine engine, + Row transactionState, + Map partitionValues, + PartitionKeyType partitionKeyType) { // Column mapping check removed from getWriteContext: connectors that handle column // name translation themselves (e.g. using Spark's Parquet writer instead of Kernel's) // only need the target directory and partition metadata from this method. The column @@ -276,15 +298,18 @@ static DataWriteContext getWriteContext( StructType tableSchema = getLogicalSchema(transactionState); List partitionColNames = getPartitionColumnsList(transactionState); - partitionValues = - validateAndSanitizePartitionValues(tableSchema, partitionColNames, partitionValues); - - // The on-disk partition directory and AddFile partition-value keys use physical names, so - // translate the column-name list and the value-map keys to physical. ColumnMapping.ColumnMappingMode mode = getColumnMappingMode(transactionState); + // The on-disk partition directory and AddFile partition-value keys use physical names. + // Validate in the caller's key space, then ensure the values are physically keyed. + partitionValues = + validateAndSanitizePartitionValues( + tableSchema, partitionColNames, partitionValues, partitionKeyType); + if (partitionKeyType == PartitionKeyType.LOGICAL) { + partitionValues = + PartitionUtils.toPhysicalPartitionValues(tableSchema, partitionValues, mode); + } partitionColNames = PartitionUtils.toPhysicalPartitionColNames(tableSchema, partitionColNames, mode); - partitionValues = PartitionUtils.toPhysicalPartitionValues(tableSchema, partitionValues, mode); String targetDirectory = getTargetDirectory(getTablePath(transactionState), partitionColNames, partitionValues); diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/exceptions/VersionToLoadAfterLatestCommitException.java b/kernel/kernel-api/src/main/java/io/delta/kernel/exceptions/VersionToLoadAfterLatestCommitException.java new file mode 100644 index 00000000000..8ef55371caf --- /dev/null +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/exceptions/VersionToLoadAfterLatestCommitException.java @@ -0,0 +1,33 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.kernel.exceptions; + +import io.delta.kernel.annotation.Evolving; + +/** + * Thrown when the requested version to load is higher than the latest version available in the + * Delta log, i.e. it has not been materialized yet. This is typically recoverable: a consumer may + * retry once the requested version has been committed. + * + * @since 4.3.0 + */ +@Evolving +public class VersionToLoadAfterLatestCommitException extends KernelException { + + public VersionToLoadAfterLatestCommitException(String message) { + super(message); + } +} diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/actions/AddFile.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/actions/AddFile.java index 1e20a508636..5fe32fe38a1 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/actions/AddFile.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/actions/AddFile.java @@ -119,6 +119,35 @@ public static Row createAddFileRow( checkArgument(path != null, "path is not nullable"); checkArgument(partitionValues != null, "partitionValues is not nullable"); + return createAddFileRowWithStatsJson( + path, + partitionValues, + size, + modificationTime, + dataChange, + deletionVector, + tags, + baseRowId, + defaultRowCommitVersion, + stats.map(stat -> stat.serializeAsJson(physicalSchema))); + } + + /** Utility to generate an 'AddFile' row from fields with pre-serialized stats JSON. */ + public static Row createAddFileRowWithStatsJson( + String path, + MapValue partitionValues, + long size, + long modificationTime, + boolean dataChange, + Optional deletionVector, + Optional tags, + Optional baseRowId, + Optional defaultRowCommitVersion, + Optional statsJson) { + + checkArgument(path != null, "path is not nullable"); + checkArgument(partitionValues != null, "partitionValues is not nullable"); + Map fieldMap = new HashMap<>(); fieldMap.put(FULL_SCHEMA.indexOf("path"), path); fieldMap.put(FULL_SCHEMA.indexOf("partitionValues"), partitionValues); @@ -129,8 +158,7 @@ public static Row createAddFileRow( baseRowId.ifPresent(id -> fieldMap.put(FULL_SCHEMA.indexOf("baseRowId"), id)); defaultRowCommitVersion.ifPresent( version -> fieldMap.put(FULL_SCHEMA.indexOf("defaultRowCommitVersion"), version)); - stats.ifPresent( - stat -> fieldMap.put(FULL_SCHEMA.indexOf("stats"), stat.serializeAsJson(physicalSchema))); + statsJson.ifPresent(json -> fieldMap.put(FULL_SCHEMA.indexOf("stats"), json)); deletionVector.ifPresent( dv -> { Row dvRow = dv.toRow(); diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/replay/ConflictChecker.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/replay/ConflictChecker.java index fb77b75aaa6..3d2ce04d96a 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/replay/ConflictChecker.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/replay/ConflictChecker.java @@ -137,8 +137,14 @@ public TransactionRebaseState resolveConflicts(Engine engine) throws ConcurrentW AtomicReference> winningCommitInfoOpt = new AtomicReference<>(Optional.empty()); - // no winning commits. why did we get the transaction conflict? - checkState(!winningCommits.isEmpty(), "No winning commits found."); + if (winningCommits.isEmpty()) { + if (TableFeatures.isCatalogManagedSupported(transaction.getProtocol())) { + // A winning commit may be ratified before it is published to the log. Without its actions, + // conflict resolution is unsafe; let the caller refresh the catalog snapshot and retry. + throw new ConcurrentWriteException(); + } + checkState(false, "No winning commits found."); + } // Collect the data files removed by this (losing) transaction so we can detect delete-vs-delete // conflicts against the winning transactions below. @@ -449,7 +455,9 @@ private List getWinningCommitFiles(Engine engine) { return ensureNoGapsInWinningCommits(winningCommitFiles); } catch (FileNotFoundException nfe) { - // no winning commits. why did we get here? + if (TableFeatures.isCatalogManagedSupported(transaction.getProtocol())) { + return Collections.emptyList(); + } throw new IllegalStateException("No winning commits found.", nfe); } catch (IOException ioe) { throw new UncheckedIOException("Error listing files from " + firstWinningCommitFile, ioe); diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/PartitionUtils.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/PartitionUtils.java index c4c3d4ac445..2d321aca72f 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/PartitionUtils.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/PartitionUtils.java @@ -24,6 +24,7 @@ import static io.delta.kernel.internal.util.SchemaUtils.casePreservingPartitionColNames; import static java.util.Arrays.asList; +import io.delta.kernel.PartitionKeyType; import io.delta.kernel.data.*; import io.delta.kernel.engine.Engine; import io.delta.kernel.engine.ExpressionHandler; @@ -145,9 +146,9 @@ public static MapValue serializePartitionMap(Map partitionValue } /** - * Validate {@code partitionValues} contains values for every partition column in the table and - * the type of the value is correct. Once validated the partition values are sanitized to match - * the case of the partition column names in the table schema and returned + * Equivalent to {@link #validateAndSanitizePartitionValues(StructType, List, Map, + * PartitionKeyType)} with {@link PartitionKeyType#LOGICAL}; {@code partitionValues} keys are + * logical partition column names. * * @param tableSchema Schema of the table. * @param partitionColNames Partition column name. These should be from the table metadata that @@ -159,13 +160,50 @@ public static Map validateAndSanitizePartitionValues( StructType tableSchema, List partitionColNames, Map partitionValues) { + return validateAndSanitizePartitionValues( + tableSchema, partitionColNames, partitionValues, PartitionKeyType.LOGICAL); + } - if (!toLowerCaseSet(partitionColNames).equals(toLowerCaseSet(partitionValues.keySet()))) { + /** + * Validate {@code partitionValues} contains a value for every partition column and that each + * value's type matches its column, then return the values sanitized to the case of the partition + * column names. + * + * @param tableSchema Schema of the table. + * @param partitionColNames Logical partition column names, in the same case as in the table + * schema. + * @param partitionValues Map of partition column to value given by the connector, keyed per + * {@code partitionKeyType}. + * @param partitionKeyType Whether {@code partitionValues} keys are logical or physical column + * names. + * @return Sanitized partition values, keyed per {@code partitionKeyType}. + */ + public static Map validateAndSanitizePartitionValues( + StructType tableSchema, + List partitionColNames, + Map partitionValues, + PartitionKeyType partitionKeyType) { + + // Resolve the partition column names in partitionKeyType's name space, and the field for each + // name used to check the value's type. + List expectedColNames = new ArrayList<>(partitionColNames.size()); + Map colNameToField = new HashMap<>(); + for (String logicalName : partitionColNames) { + StructField field = tableSchema.get(logicalName); + String name = + partitionKeyType == PartitionKeyType.PHYSICAL + ? ColumnMapping.getPhysicalName(field) + : logicalName; + expectedColNames.add(name); + colNameToField.put(name, field); + } + + if (!toLowerCaseSet(expectedColNames).equals(toLowerCaseSet(partitionValues.keySet()))) { throw new IllegalArgumentException( String.format( "Partition values provided are not matching the partition columns. " + "Partition columns: %s, Partition values: %s", - partitionColNames, partitionValues)); + expectedColNames, partitionValues)); } // Convert the partition column names in given `partitionValues` to schema case. Schema @@ -175,7 +213,7 @@ public static Map validateAndSanitizePartitionValues( // (`partitionValues` in `AddFile`) or generating the target directory for writing the // data belonging to a partition. Map schemaCasePartitionValues = - casePreservingPartitionColNames(partitionColNames, partitionValues); + casePreservingPartitionColNames(expectedColNames, partitionValues); // validate types are the same schemaCasePartitionValues @@ -184,7 +222,7 @@ public static Map validateAndSanitizePartitionValues( entry -> { String partColName = entry.getKey(); Literal partValue = entry.getValue(); - StructField partColField = tableSchema.get(partColName); + StructField partColField = colNameToField.get(partColName); // this shouldn't happen as we have already validated the partition column names checkArgument( diff --git a/kernel/kernel-api/src/test/java/io/delta/kernel/internal/metrics/CounterTest.java b/kernel/kernel-api/src/test/java/io/delta/kernel/internal/metrics/CounterTest.java new file mode 100644 index 00000000000..e3eaba21623 --- /dev/null +++ b/kernel/kernel-api/src/test/java/io/delta/kernel/internal/metrics/CounterTest.java @@ -0,0 +1,68 @@ +/* + * Copyright (2025) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.kernel.internal.metrics; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +class CounterTest { + + @Test + void startsAtZero() { + assertEquals(0L, new Counter().value()); + } + + @Test + void incrementsByOne() { + Counter counter = new Counter(); + counter.increment(); + assertEquals(1L, counter.value()); + } + + @ParameterizedTest + @ValueSource(longs = {0L, 1L, 10L}) + void incrementsBySpecifiedAmount(long amount) { + Counter counter = new Counter(); + counter.increment(amount); + assertEquals(amount, counter.value()); + } + + @Test + void accumulatesAndResets() { + Counter counter = new Counter(); + counter.increment(); + counter.increment(); + counter.increment(10); + assertEquals(12L, counter.value()); + + counter.reset(); + assertEquals(0L, counter.value()); + + counter.increment(); + assertEquals(1L, counter.value()); + } + + @Test + void counterToStringRepresentation() { + Counter counter = new Counter(); + counter.increment(42); + + assertEquals("Counter(42)", counter.toString()); + } +} diff --git a/kernel/kernel-api/src/test/java/io/delta/kernel/types/StringTypeTest.java b/kernel/kernel-api/src/test/java/io/delta/kernel/types/StringTypeTest.java new file mode 100644 index 00000000000..e7f6e1671de --- /dev/null +++ b/kernel/kernel-api/src/test/java/io/delta/kernel/types/StringTypeTest.java @@ -0,0 +1,94 @@ +/* + * Copyright (2024) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.kernel.types; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.stream.Stream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +class StringTypeTest { + + private static Stream equalsCases() { + return Stream.of( + Arguments.of(StringType.STRING, StringType.STRING, true), + Arguments.of(StringType.STRING, new StringType("sPark.UTF8_bINary"), true), + Arguments.of(StringType.STRING, new StringType("SPARK.UTF8_LCASE"), false), + Arguments.of(new StringType("ICU.UNICODE"), new StringType("SPARK.UTF8_LCASE"), false), + Arguments.of(new StringType("ICU.UNICODE"), new StringType("ICU.UNICODE_CI"), false), + Arguments.of(new StringType("ICU.UNICODE_CI"), new StringType("icU.uniCODe_Ci"), true)); + } + + @ParameterizedTest(name = "{0}.equals({1}) => {2}") + @MethodSource("equalsCases") + void equalsHonorsCollationIdentity(StringType left, StringType right, boolean expected) { + assertEquals(expected, left.equals(right)); + // Collation identity comparison must be symmetric. + assertEquals(expected, right.equals(left)); + } + + @Test + void equalsIsNullOrForeignTypeSafeAndReflexive() { + StringType custom = new StringType("ICU.UNICODE_CI"); + + assertFalse(custom.equals(null)); + assertFalse(custom.equals(new Object())); + assertFalse(custom.equals(IntegerType.INTEGER)); + + assertTrue(custom.equals(custom)); + assertTrue(StringType.STRING.equals(StringType.STRING)); + } + + @Test + void equalInstancesHaveEqualHashCodes() { + assertEquals(StringType.STRING.hashCode(), new StringType("sPark.UTF8_bINary").hashCode()); + } + + private static Stream utf8BinaryCollationCases() { + return Stream.of( + Arguments.of(StringType.STRING, true), + Arguments.of(new StringType("sPark.UTF8_bINary"), true), + Arguments.of(new StringType("SPARK.UTF8_LCASE"), false), + Arguments.of(new StringType("ICU.UNICODE.72.2"), false), + Arguments.of(new StringType("ICU.UNICODE_CI"), false)); + } + + @ParameterizedTest + @MethodSource("utf8BinaryCollationCases") + void isUTF8BinaryCollated(StringType stringType, boolean expected) { + assertEquals(expected, stringType.isUTF8BinaryCollated()); + } + + private static Stream toStringCases() { + return Stream.of( + Arguments.of(StringType.STRING, "string"), + Arguments.of(new StringType("sPark.UTF8_bINary"), "string"), + Arguments.of(new StringType("SPARK.UTF8_LCASE"), "string collate UTF8_LCASE"), + Arguments.of(new StringType("ICU.uNICoDE.72.2"), "string collate UNICODE"), + Arguments.of(new StringType("ICU.UNICODE_CI"), "string collate UNICODE_CI")); + } + + @ParameterizedTest + @MethodSource("toStringCases") + void stringRepresentation(StringType stringType, String expected) { + assertEquals(expected, stringType.toString()); + } +} diff --git a/kernel/kernel-api/src/test/scala/io/delta/kernel/TransactionSuite.scala b/kernel/kernel-api/src/test/scala/io/delta/kernel/TransactionSuite.scala index 8abd7407626..575c98756fe 100644 --- a/kernel/kernel-api/src/test/scala/io/delta/kernel/TransactionSuite.scala +++ b/kernel/kernel-api/src/test/scala/io/delta/kernel/TransactionSuite.scala @@ -21,6 +21,7 @@ import java.util.Optional import scala.collection.JavaConverters._ +import io.delta.kernel.PartitionKeyType import io.delta.kernel.Transaction.{generateAppendActions, getWriteContext, transformLogicalData} import io.delta.kernel.data._ import io.delta.kernel.exceptions.KernelException @@ -158,6 +159,71 @@ class TransactionSuite extends AnyFunSuite with VectorTestUtils with MockEngineU } } + Seq("name", "id").foreach { cmMode => + test(s"getWriteContext: PHYSICAL keys used as-is on column-mapped table: cmMode=$cmMode") { + // Physical keys are used as-is: same physical target dir and keys as the LOGICAL path. + val ctx = getWriteContext( + mockEngine(), + testTxnState(columnMappedPartitionSchema, testPartitionColNames, cmMode = cmMode), + Map( + "col-state" -> Literal.ofString("CA"), + "col-country" -> Literal.ofString("USA")).asJava, + PartitionKeyType.PHYSICAL) + + assert(ctx.getTargetDirectory.endsWith("/col-state=CA/col-country=USA")) + val partitionKeys = + ctx.asInstanceOf[DataWriteContextImpl].getPartitionValues.keySet().asScala + assert(partitionKeys === Set("col-state", "col-country")) + } + } + + test("getWriteContext: PHYSICAL rejects a value whose type mismatches the column") { + val e = intercept[IllegalArgumentException] { + getWriteContext( + mockEngine(), + testTxnState(columnMappedPartitionSchema, testPartitionColNames, cmMode = "name"), + Map( + "col-state" -> Literal.ofInt(5), // state is StringType + "col-country" -> Literal.ofString("USA")).asJava, + PartitionKeyType.PHYSICAL) + } + assert( + e.getMessage === + "Partition column col-state is of type string but the value provided is of type integer") + } + + test("getWriteContext: PHYSICAL rejects an unknown physical partition column") { + val e = intercept[IllegalArgumentException] { + getWriteContext( + mockEngine(), + testTxnState(columnMappedPartitionSchema, testPartitionColNames, cmMode = "name"), + Map( + "col-bogus" -> Literal.ofString("CA"), + "col-country" -> Literal.ofString("USA")).asJava, + PartitionKeyType.PHYSICAL) + } + assert( + e.getMessage === + "Partition values provided are not matching the partition columns. " + + "Partition columns: [col-state, col-country], " + + "Partition values: {col-bogus=CA, col-country=USA}") + } + + test("getWriteContext: PHYSICAL rejects a missing partition column") { + val e = intercept[IllegalArgumentException] { + getWriteContext( + mockEngine(), + testTxnState(columnMappedPartitionSchema, testPartitionColNames, cmMode = "name"), + Map("col-state" -> Literal.ofString("CA")).asJava, // missing col-country + PartitionKeyType.PHYSICAL) + } + assert( + e.getMessage === + "Partition values provided are not matching the partition columns. " + + "Partition columns: [col-state, col-country], " + + "Partition values: {col-state=CA}") + } + withIcebergCompatVersions("generateAppendActions: iceberg comaptibily checks") { case (icebergCompatV2Enabled, icebergCompatV3Enabled) => val txnState = testTxnState( diff --git a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/CommitRangeBuilderSuite.scala b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/CommitRangeBuilderSuite.scala index 20dc9a1a79d..8301e6b0e65 100644 --- a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/CommitRangeBuilderSuite.scala +++ b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/CommitRangeBuilderSuite.scala @@ -31,7 +31,6 @@ import io.delta.kernel.test.{MockFileSystemClientUtils, MockListFromFileSystemCl import io.delta.kernel.test.MockSnapshotUtils.getMockSnapshot import io.delta.kernel.utils.FileStatus -import junit.runner.Version import org.scalatest.funsuite.AnyFunSuite class CommitRangeBuilderSuite extends AnyFunSuite with MockFileSystemClientUtils diff --git a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/actions/AddFileSuite.scala b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/actions/AddFileSuite.scala index c535c7925e7..50f5ef8d21e 100644 --- a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/actions/AddFileSuite.scala +++ b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/actions/AddFileSuite.scala @@ -263,6 +263,23 @@ class AddFileSuite extends AnyFunSuite with Matchers { assert(addFile1.hashCode === addFile1.hashCode) } + test("createAddFileRowWithStatsJson preserves serialized stats") { + val statsJson = "{\"numRecords\":42,\"minValues\":{\"id\":1},\"tightBounds\":false}" + val row = AddFile.createAddFileRowWithStatsJson( + "some/file.parquet", + stringStringMapValue(Map.empty[String, String].asJava), + 1024L, + 500L, + true, + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.of(statsJson)) + + assert(new AddFile(row).getStatsJson === Optional.of(statsJson)) + } + // Tests for toRemoveFileRow test("toRemoveFileRow: handles AddFile with all required fields") { val addFile = new AddFile(generateTestAddFileRow( diff --git a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/metrics/CounterSuite.scala b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/metrics/CounterSuite.scala deleted file mode 100644 index b56e00c6c81..00000000000 --- a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/metrics/CounterSuite.scala +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (2025) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.delta.kernel.internal.metrics - -import org.scalatest.funsuite.AnyFunSuite - -class CounterSuite extends AnyFunSuite { - - test("Counter class") { - val counter = new Counter() - assert(counter.value == 0) - counter.increment(0) - assert(counter.value == 0) - counter.increment() - assert(counter.value == 1) - counter.increment() - assert(counter.value == 2) - counter.increment(10) - assert(counter.value == 12) - counter.reset() - assert(counter.value == 0) - counter.increment() - assert(counter.value == 1) - } - - test("Counter toString representation") { - val counter = new Counter() - counter.increment(42) - - val stringRepresentation = counter.toString() - assert(stringRepresentation === "Counter(42)") - } -} diff --git a/kernel/kernel-api/src/test/scala/io/delta/kernel/types/StringTypeSuite.scala b/kernel/kernel-api/src/test/scala/io/delta/kernel/types/StringTypeSuite.scala deleted file mode 100644 index d89c34ecea7..00000000000 --- a/kernel/kernel-api/src/test/scala/io/delta/kernel/types/StringTypeSuite.scala +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright (2024) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.delta.kernel.types - -import org.scalatest.funsuite.AnyFunSuite - -class StringTypeSuite extends AnyFunSuite { - test("check equals") { - // Testcase: (instance1, instance2, expected value for `instance1 == instance2`) - Seq( - ( - StringType.STRING, - StringType.STRING, - true), - ( - StringType.STRING, - new StringType("sPark.UTF8_bINary"), - true), - ( - StringType.STRING, - new StringType("SPARK.UTF8_LCASE"), - false), - ( - new StringType("ICU.UNICODE"), - new StringType("SPARK.UTF8_LCASE"), - false), - ( - new StringType("ICU.UNICODE"), - new StringType("ICU.UNICODE_CI"), - false), - ( - new StringType("ICU.UNICODE_CI"), - new StringType("icU.uniCODe_Ci"), - true)).foreach { - case (st1, st2, expResult) => - assert(st1.equals(st2) == expResult) - } - } - - test("isUTF8BinaryCollated") { - assert(StringType.STRING.isUTF8BinaryCollated) - assert(new StringType("sPark.UTF8_bINary").isUTF8BinaryCollated) - assert(!new StringType("SPARK.UTF8_LCASE").isUTF8BinaryCollated) - assert(!new StringType("ICU.UNICODE.72.2").isUTF8BinaryCollated) - assert(!new StringType("ICU.UNICODE_CI").isUTF8BinaryCollated) - } - - test("toString") { - assert(StringType.STRING.toString == "string") - assert(new StringType("sPark.UTF8_bINary").toString == "string") - assert(new StringType("SPARK.UTF8_LCASE").toString == "string collate UTF8_LCASE") - assert(new StringType("ICU.uNICoDE.72.2").toString == "string collate UNICODE") - assert(new StringType("ICU.UNICODE_CI").toString == "string collate UNICODE_CI") - } -} diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/engine/fileio/PositionOutputStream.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/engine/fileio/PositionOutputStream.java index 026f22a42a1..4263e0db8bc 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/engine/fileio/PositionOutputStream.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/engine/fileio/PositionOutputStream.java @@ -23,10 +23,33 @@ * to write data into the file. */ public abstract class PositionOutputStream extends OutputStream { + private boolean aborted = false; + /** * Get the current position in the stream. * * @return the current position in bytes from the start of the stream */ public abstract long getPos() throws IOException; + + /** + * Record that this stream should discard, rather than publish, its output. abort() does not + * release resources on its own; the subsequent {@link #close()} does, and must honor this flag so + * that no file - partial or complete - becomes visible at the destination path. + * + *

Honored by temp-then-rename staging (temp file deleted) and by abortable streams. A backend + * that writes directly to the destination with a non-abortable stream cannot discard on close() + * and will publish. + */ + public void abort() { + this.aborted = true; + } + + /** + * @return true if {@link #abort()} has been called. Concrete {@link #close()} implementations + * must honor this by discarding the in-progress write instead of publishing it. + */ + protected final boolean isAborted() { + return aborted; + } } diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/engine/hadoopio/HadoopInputFile.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/engine/hadoopio/HadoopInputFile.java index 6fdebe24ea9..52b06003a8f 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/engine/hadoopio/HadoopInputFile.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/engine/hadoopio/HadoopInputFile.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.io.InterruptedIOException; import java.util.concurrent.ExecutionException; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FutureDataInputStreamBuilder; @@ -47,6 +48,11 @@ public String path() { return path.toString(); } + /** Returns the Hadoop {@link Configuration} this file is read with. */ + public Configuration configuration() { + return fs.getConf(); + } + @Override public SeekableInputStream newStream() throws IOException { // Pass the known length so the file system can skip the HEAD call to fetch the file size; diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/engine/hadoopio/HadoopOutputFile.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/engine/hadoopio/HadoopOutputFile.java index 77c6fc3f5de..e58da2cd5d6 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/engine/hadoopio/HadoopOutputFile.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/engine/hadoopio/HadoopOutputFile.java @@ -68,7 +68,12 @@ public PositionOutputStream create(boolean putIfAbsent) throws IOException { return new HadoopPositionOutputStream(fs.create(writePath)) { @Override public void close() throws IOException { + // super.close() aborts the temp stream if it is abortable, else closes it normally super.close(); + if (isAborted()) { + fs.delete(writePath, false /* recursive */); + return; + } if (useRename) { boolean renameDone = false; try { diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/engine/hadoopio/HadoopPositionOutputStream.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/engine/hadoopio/HadoopPositionOutputStream.java index 6b3e3f47a3d..9851490c3dc 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/engine/hadoopio/HadoopPositionOutputStream.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/engine/hadoopio/HadoopPositionOutputStream.java @@ -18,6 +18,7 @@ import io.delta.kernel.defaults.engine.fileio.PositionOutputStream; import java.io.IOException; import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.StreamCapabilities; public class HadoopPositionOutputStream extends PositionOutputStream { private final FSDataOutputStream delegateStream; @@ -48,7 +49,14 @@ public void flush() throws IOException { @Override public void close() throws IOException { - delegateStream.close(); + // If the write was aborted and the underlying stream supports a true abort (e.g. S3A cancels + // its multipart upload), abort so nothing is published to the destination. Otherwise fall + // back to a normal close; callers that stage via temp-then-rename skip the rename on abort. + if (isAborted() && delegateStream.hasCapability(StreamCapabilities.ABORTABLE_STREAM)) { + delegateStream.abort(); + } else { + delegateStream.close(); + } } @Override diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/parquet/ParquetBatchRowReader.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/parquet/ParquetBatchRowReader.java new file mode 100644 index 00000000000..e7d6977db70 --- /dev/null +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/parquet/ParquetBatchRowReader.java @@ -0,0 +1,264 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.kernel.defaults.internal.parquet; + +import static java.util.Objects.requireNonNull; + +import java.io.Closeable; +import java.io.IOException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Optional; +import java.util.PrimitiveIterator; +import java.util.Set; +import java.util.stream.LongStream; +import org.apache.parquet.ParquetReadOptions; +import org.apache.parquet.column.page.PageReadStore; +import org.apache.parquet.conf.ParquetConfiguration; +import org.apache.parquet.filter2.compat.FilterCompat; +import org.apache.parquet.hadoop.ParquetFileReader; +import org.apache.parquet.hadoop.api.InitContext; +import org.apache.parquet.hadoop.api.ReadSupport; +import org.apache.parquet.hadoop.metadata.FileMetaData; +import org.apache.parquet.io.ColumnIOFactory; +import org.apache.parquet.io.MessageColumnIO; +import org.apache.parquet.io.ParquetDecodingException; +import org.apache.parquet.io.RecordReader; +import org.apache.parquet.io.api.RecordMaterializer; +import org.apache.parquet.schema.MessageType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Iterates rows from an already-opened {@link ParquetFileReader}, using public parquet-mr APIs + * ({@code readNextFilteredRowGroup} + {@link ColumnIOFactory}) instead of package-private {@code + * InternalParquetRecordReader}. + * + *

Callers own footer reuse by opening the file reader with {@link + * ParquetFileReader#open(org.apache.parquet.io.InputFile, + * org.apache.parquet.hadoop.metadata.ParquetMetadata, ParquetReadOptions, + * org.apache.parquet.io.SeekableInputStream)} before calling {@link #open}. + * + *

Ownership of {@code fileReader} transfers to this class; {@link #close()} closes it. + */ +final class ParquetBatchRowReader implements Closeable { + private static final Logger logger = LoggerFactory.getLogger(ParquetBatchRowReader.class); + private static final String STRICT_TYPE_CHECKING = "parquet.strict.typing"; + private final ParquetFileReader fileReader; + private final ColumnIOFactory columnIOFactory; + private final MessageType fileSchema; + private final MessageType requestedSchema; + private final RecordMaterializer materializer; + private final boolean strictTypeChecking; + private final boolean filterRecords; + private final FilterCompat.Filter recordFilter; + private final long totalRows; + private long rowsConsumed = 0; + private long rowsLoadedFromGroups = 0; + private PageReadStore currentRowGroup; + private RecordReader recordReader; + private long currentRowIndex = -1; + private PrimitiveIterator.OfLong rowIndexInFile; + + private ParquetBatchRowReader( + ParquetFileReader fileReader, + ColumnIOFactory columnIOFactory, + MessageType fileSchema, + MessageType requestedSchema, + RecordMaterializer materializer, + boolean strictTypeChecking, + boolean filterRecords, + FilterCompat.Filter recordFilter, + long totalRows) { + this.fileReader = fileReader; + this.columnIOFactory = columnIOFactory; + this.fileSchema = fileSchema; + this.requestedSchema = requestedSchema; + this.materializer = materializer; + this.strictTypeChecking = strictTypeChecking; + this.filterRecords = filterRecords; + this.recordFilter = recordFilter; + this.totalRows = totalRows; + } + + /** + * Takes ownership of {@code fileReader}. Projects columns via {@code readSupport}, applies the + * row-group filter already configured on {@code fileReader}'s options, and prepares to iterate + * rows. + */ + static ParquetBatchRowReader open( + ParquetFileReader fileReader, ParquetReadOptions options, ReadSupport readSupport) { + requireNonNull(fileReader, "fileReader"); + requireNonNull(options, "options"); + requireNonNull(readSupport, "readSupport"); + // Mirror InternalParquetRecordReader: copy option properties onto the conf used by ReadSupport. + ParquetConfiguration conf = requireNonNull(options.getConfiguration(), "options.configuration"); + for (String property : options.getPropertyNames()) { + conf.set(property, options.getProperty(property)); + } + FileMetaData parquetFileMetadata = fileReader.getFooter().getFileMetaData(); + MessageType fileSchema = parquetFileMetadata.getSchema(); + Map fileMetadata = parquetFileMetadata.getKeyValueMetaData(); + ReadSupport.ReadContext readContext = + readSupport.init(new InitContext(conf, toSetMultiMap(fileMetadata), fileSchema)); + ColumnIOFactory columnIOFactory = new ColumnIOFactory(parquetFileMetadata.getCreatedBy()); + MessageType requestedSchema = readContext.getRequestedSchema(); + // Setting the projection schema before running any filtering (e.g. getting filtered record + // count) because projection impacts filtering + fileReader.setRequestedSchema(requestedSchema); + RecordMaterializer materializer = + readSupport.prepareForRead(conf, fileMetadata, fileSchema, readContext); + return new ParquetBatchRowReader( + fileReader, + columnIOFactory, + fileSchema, + requestedSchema, + materializer, + options.isEnabled(STRICT_TYPE_CHECKING, true), + options.useRecordFilter(), + options.getRecordFilter() == null ? FilterCompat.NOOP : options.getRecordFilter(), + fileReader.getFilteredRecordCount()); + } + + /** + * Advances to the next row. Returns {@code true} if a row is available; {@link + * #currentRowIndex()} is then valid until the next call. + * + *

Materialized values are written into the {@link ReadSupport}'s {@link RecordMaterializer} + * (Kernel's {@code BatchReadSupport} / {@code RowRecordCollector}). The returned boolean is the + * only signal the iterator needs — there is no {@code getCurrentValue()}. + */ + boolean next() throws IOException { + while (rowsConsumed < totalRows) { + loadNextRowGroupIfNeeded(); + rowsConsumed++; + try { + Object value = recordReader.read(); + advanceRowIndex(); + if (recordReader.shouldSkipCurrentRecord()) { + // this record is being filtered via the filter2 package + continue; + } + if (value == null) { + // only happens with FilteredRecordReader at end of block + rowsConsumed = rowsLoadedFromGroups; + continue; + } + return true; + } catch (RuntimeException e) { + throw new ParquetDecodingException( + String.format( + "Can not read value at %d in file %s", rowsConsumed, fileReader.getFile()), + e); + } + } + return false; + } + + /** File-level row index of the row made current by the last successful {@link #next()}. */ + long currentRowIndex() { + return currentRowIndex; + } + + @Override + public void close() throws IOException { + try { + if (currentRowGroup != null) { + currentRowGroup.close(); + currentRowGroup = null; + } + } finally { + fileReader.close(); + } + } + + private void loadNextRowGroupIfNeeded() throws IOException { + if (rowsConsumed != rowsLoadedFromGroups) { + return; + } + if (currentRowGroup != null) { + currentRowGroup.close(); + currentRowGroup = null; + } + logger.info("at row " + rowsConsumed + ". reading next block"); + // Applies stats / dictionary / bloom / column-index filters from ParquetReadOptions. + // (Plain readNextRowGroup() would ignore those.) + currentRowGroup = fileReader.readNextFilteredRowGroup(); + if (currentRowGroup == null) { + throw new IOException( + "expecting more rows but reached last block. Read " + + rowsConsumed + + " out of " + + totalRows); + } + resetRowIndexIterator(currentRowGroup); + MessageColumnIO columnIO = + columnIOFactory.getColumnIO(requestedSchema, fileSchema, strictTypeChecking); + recordReader = + columnIO.getRecordReader( + currentRowGroup, materializer, filterRecords ? recordFilter : FilterCompat.NOOP); + rowsLoadedFromGroups += currentRowGroup.getRowCount(); + } + + /** Resets the row index based on the current processed row group. */ + private void resetRowIndexIterator(PageReadStore pages) { + Optional rowGroupOffset = pages.getRowIndexOffset(); + if (!rowGroupOffset.isPresent()) { + rowIndexInFile = null; + currentRowIndex = -1; + return; + } + currentRowIndex = -1; + final PrimitiveIterator.OfLong rowIdxInRowGroup; + if (pages.getRowIndexes().isPresent()) { + rowIdxInRowGroup = pages.getRowIndexes().get(); + } else { + rowIdxInRowGroup = LongStream.range(0, pages.getRowCount()).iterator(); + } + final long offset = rowGroupOffset.get(); + rowIndexInFile = + new PrimitiveIterator.OfLong() { + @Override + public long nextLong() { + return offset + rowIdxInRowGroup.nextLong(); + } + + @Override + public boolean hasNext() { + return rowIdxInRowGroup.hasNext(); + } + }; + } + + private void advanceRowIndex() { + if (rowIndexInFile != null && rowIndexInFile.hasNext()) { + currentRowIndex = rowIndexInFile.nextLong(); + } else { + currentRowIndex = -1; + } + } + + private static Map> toSetMultiMap(Map map) { + Map> out = new HashMap<>(); + for (Map.Entry e : map.entrySet()) { + Set set = new HashSet<>(); + set.add(e.getValue()); + out.put(e.getKey(), set); + } + return out; + } +} diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/parquet/ParquetFileReader.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/parquet/ParquetFileReader.java index a8c1a2f34fc..91aed398f12 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/parquet/ParquetFileReader.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/parquet/ParquetFileReader.java @@ -33,10 +33,10 @@ import java.io.IOException; import java.util.*; import org.apache.hadoop.conf.Configuration; +import org.apache.parquet.ParquetReadOptions; +import org.apache.parquet.conf.ParquetConfiguration; import org.apache.parquet.filter2.compat.FilterCompat; import org.apache.parquet.filter2.predicate.FilterPredicate; -import org.apache.parquet.format.converter.ParquetMetadataConverter; -import org.apache.parquet.hadoop.ParquetReader; import org.apache.parquet.hadoop.api.InitContext; import org.apache.parquet.hadoop.api.ReadSupport; import org.apache.parquet.hadoop.metadata.ParquetMetadata; @@ -71,7 +71,7 @@ public CloseableIterator read( return new CloseableIterator() { private final BatchReadSupport readSupport = new BatchReadSupport(maxBatchSize, schema); - private ParquetReader reader; + private ParquetBatchRowReader reader; private boolean hasNotConsumedNextElement; @Override @@ -87,8 +87,7 @@ public boolean hasNext() { return true; } - Object next = reader.read(); - hasNotConsumedNextElement = next != null; + hasNotConsumedNextElement = reader.next(); return hasNotConsumedNextElement; } catch (IOException ex) { throw new KernelEngineException( @@ -106,7 +105,7 @@ public ColumnarBatch next() { hasNotConsumedNextElement = false; // hasNext reads to row to confirm there is a next element. // get the row index only if required by the read schema - long rowIndex = hasRowIndexCol ? reader.getCurrentRowIndex() : -1; + long rowIndex = hasRowIndexCol ? reader.currentRowIndex() : -1; readSupport.finalizeCurrentRow(rowIndex); batchSize++; } while (batchSize < maxBatchSize && hasNext()); @@ -126,24 +125,22 @@ private void initParquetReaderIfRequired() { org.apache.parquet.io.InputFile parquetInputFile = ParquetIOUtils.createParquetInputFile(inputFile); - ParquetMetadata footer = - org.apache.parquet.hadoop.ParquetFileReader.readFooter( - parquetInputFile, ParquetMetadataConverter.NO_FILTER); + // Seed both the footer read and the read options below with this configuration, so + // neither constructs a fresh Hadoop Configuration per file. See + // ParquetIOUtils#parquetConfiguration. + // + // This also means that the reader now inherits any parquet-mr options set on the + // file's Hadoop Configuration, rather than always using the defaults. + ParquetConfiguration parquetConf = ParquetIOUtils.parquetConfiguration(inputFile); + + ParquetMetadata footer = ParquetIOUtils.readFooter(parquetInputFile, parquetConf); MessageType parquetSchema = footer.getFileMetaData().getSchema(); Optional parquetPredicate = predicate.flatMap(predicate -> toParquetFilter(parquetSchema, predicate)); - // TODO: We can avoid reading the footer again if we can pass the footer, but there is - // no API to do that in the current version of parquet-mr which takes InputFile - // as input. - reader = - new ParquetReader.Builder(parquetInputFile) { - @Override - protected ReadSupport getReadSupport() { - return readSupport; - } - }.withFilter(parquetPredicate.map(FilterCompat::get).orElse(FilterCompat.NOOP)) + ParquetReadOptions readOptions = + ParquetReadOptions.builder(parquetConf) // Disable the record level filtering as the `parquet-mr` evaluates // the filter once the entire record has been materialized. Instead, // we use the predicate to prune the row groups which is more efficient. @@ -154,8 +151,16 @@ protected ReadSupport getReadSupport() { .useBloomFilter(false) .useDictionaryFilter(false) .useColumnIndexFilter(false) + .withRecordFilter( + parquetPredicate.map(FilterCompat::get).orElse(FilterCompat.NOOP)) .build(); + // Pass the already read footer to the reader to avoid reading it again. + fileReader = + org.apache.parquet.hadoop.ParquetFileReader.open( + parquetInputFile, footer, readOptions, parquetInputFile.newStream()); + // Ownership of fileReader transfers to the row reader, which closes it on close(). + reader = ParquetBatchRowReader.open(fileReader, readOptions, readSupport); } catch (IOException e) { Utils.closeCloseablesSilently(fileReader, reader); throw new KernelEngineException( diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/parquet/ParquetFileWriter.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/parquet/ParquetFileWriter.java index e54fd914847..f7d10c0ce47 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/parquet/ParquetFileWriter.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/parquet/ParquetFileWriter.java @@ -44,6 +44,8 @@ import org.apache.parquet.hadoop.metadata.CompressionCodecName; import org.apache.parquet.io.api.RecordConsumer; import org.apache.parquet.schema.MessageType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Implements writing data given as {@link FilteredColumnarBatch} to Parquet files. @@ -56,6 +58,8 @@ * ParquetWriter} through {@link RecordConsumer}. */ public class ParquetFileWriter { + private static final Logger logger = LoggerFactory.getLogger(ParquetFileWriter.class); + public static final String TARGET_FILE_SIZE_CONF = "delta.kernel.default.parquet.writer.targetMaxFileSize"; public static final long DEFAULT_TARGET_FILE_SIZE = 128 * 1024 * 1024; // 128MB @@ -176,11 +180,14 @@ private Optional writeNextFile() { return Optional.empty(); } - org.apache.parquet.io.OutputFile parquetOutputFile = + ParquetIOUtils.ParquetOutputFile parquetOutputFile = createParquetOutputFile(generateNextOutputFile(), atomicWrite); assert batchWriteSupport != null : "batchWriteSupport is not initialized"; long currentFileRowCount = 0; // tracks the number of rows written to the current file - try (ParquetWriter writer = createWriter(parquetOutputFile, batchWriteSupport)) { + ParquetWriter writer = null; + boolean committed = false; + try { + writer = createWriter(parquetOutputFile, batchWriteSupport); boolean maxFileSizeReached; do { if (consumeNextRow(writer)) { @@ -193,9 +200,33 @@ private Optional writeNextFile() { maxFileSizeReached = !writeAsSingleFile && writer.getDataSize() >= targetMaxFileSize; // Keep writing until max file is reached or no more data to write } while (!maxFileSizeReached && hasNextRow()); + committed = true; } catch (IOException e) { throw new UncheckedIOException( "Failed to write the Parquet file: " + parquetOutputFile.getPath(), e); + } finally { + if (!committed) { + // the read failed before completing; abort the write so no partial + // file is published in the subsequent close() + parquetOutputFile.abort(); + try { + if (writer != null) { + writer.close(); + } + } catch (Throwable closeFailure) { + logger.warn( + "Ignoring close failure while aborting {}", + parquetOutputFile.getPath(), + closeFailure); + } + } else if (writer != null) { + try { + writer.close(); + } catch (IOException e) { + throw new UncheckedIOException( + "Failed to close the Parquet file: " + parquetOutputFile.getPath(), e); + } + } } return Optional.of( diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/parquet/ParquetIOUtils.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/parquet/ParquetIOUtils.java index fad9571a1df..6b4163d0e87 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/parquet/ParquetIOUtils.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/parquet/ParquetIOUtils.java @@ -19,7 +19,14 @@ import io.delta.kernel.defaults.engine.fileio.OutputFile; import io.delta.kernel.defaults.engine.fileio.PositionOutputStream; import io.delta.kernel.defaults.engine.fileio.SeekableInputStream; +import io.delta.kernel.defaults.engine.hadoopio.HadoopInputFile; import java.io.IOException; +import org.apache.parquet.ParquetReadOptions; +import org.apache.parquet.conf.HadoopParquetConfiguration; +import org.apache.parquet.conf.ParquetConfiguration; +import org.apache.parquet.format.converter.ParquetMetadataConverter; +import org.apache.parquet.hadoop.ParquetFileReader; +import org.apache.parquet.hadoop.metadata.ParquetMetadata; import org.apache.parquet.io.DelegatingPositionOutputStream; import org.apache.parquet.io.DelegatingSeekableInputStream; @@ -30,6 +37,44 @@ public class ParquetIOUtils { private ParquetIOUtils() {} + /** + * Returns the {@link ParquetConfiguration} to read {@code inputFile} with. + * + *

This matters for concurrency, not just for honoring settings. `parquet-mr` entry points + * taking an {@link org.apache.parquet.io.InputFile} that isn't {@code + * org.apache.parquet.hadoop.util.HadoopInputFile} fall back to {@code new + * HadoopParquetConfiguration()}, which wraps a fresh {@code Configuration(loadDefaults=true)}. + * Hadoop loads such a configuration's properties lazily, so the first property read -- which + * `parquet-mr` issues immediately while building the read options -- scans the classpath for + * `core-default.xml` and friends under a JVM-global lock. This would make concurrent readers + * serialize on the lock. + * + *

Reusing the configuration the file is already being read with avoids the scan entirely: its + * properties are loaded once and memoized on the instance. A non-{@code HadoopInputFile} input + * carries no Hadoop configuration to reuse, so it falls back to a fresh {@code + * HadoopParquetConfiguration}. + */ + static ParquetConfiguration parquetConfiguration(InputFile inputFile) { + return inputFile instanceof HadoopInputFile + ? new HadoopParquetConfiguration(((HadoopInputFile) inputFile).configuration()) + : new HadoopParquetConfiguration(); + } + + /** + * Reads the footer of {@code parquetFile} using {@code conf}, so that `parquet-mr` doesn't + * construct a fresh Hadoop Configuration for the read. See {@link #parquetConfiguration}. + */ + static ParquetMetadata readFooter( + org.apache.parquet.io.InputFile parquetFile, ParquetConfiguration conf) throws IOException { + ParquetReadOptions readOptions = + ParquetReadOptions.builder(conf) + .withMetadataFilter(ParquetMetadataConverter.NO_FILTER) + .build(); + try (org.apache.parquet.io.SeekableInputStream stream = parquetFile.newStream()) { + return ParquetFileReader.readFooter(parquetFile, readOptions, stream); + } + } + /** Create a Parquet {@link org.apache.parquet.io.InputFile} from a Kernel's {@link InputFile}. */ static org.apache.parquet.io.InputFile createParquetInputFile(InputFile inputFile) { return new org.apache.parquet.io.InputFile() { @@ -59,46 +104,74 @@ public long getPos() throws IOException { /** * Create a Parquet {@link org.apache.parquet.io.OutputFile} from a Kernel's {@link OutputFile}. */ - static org.apache.parquet.io.OutputFile createParquetOutputFile( + static ParquetOutputFile createParquetOutputFile( OutputFile kernelOutputFile, boolean atomicWrite) { - return new org.apache.parquet.io.OutputFile() { - @Override - public org.apache.parquet.io.PositionOutputStream create(long blockSizeHint) - throws IOException { - // blockSizeHint is hint used in HDFS compliant file systems. In cloud storage systems - // it is irrelevant. So, we ignore it. - PositionOutputStream posOutputStream = kernelOutputFile.create(atomicWrite); - return new DelegatingPositionOutputStream(posOutputStream) { - @Override - public long getPos() throws IOException { - return posOutputStream.getPos(); - } - }; - } + return new ParquetOutputFile(kernelOutputFile, atomicWrite); + } - @Override - public org.apache.parquet.io.PositionOutputStream createOrOverwrite(long blockSizeHint) - throws IOException { - // In Kernel we never overwrite files, so this method is not used. - throw new UnsupportedOperationException("createOrOverwrite is not supported in Kernel"); - } + /** + * Parquet {@link org.apache.parquet.io.OutputFile} backed by a Kernel {@link OutputFile}. It + * remembers the {@link PositionOutputStream} it hands to the Parquet writer so the in-progress + * write can be aborted (via {@link #abort()}) if the row source fails. The subsequent close() + * driven by the Parquet writer then discards the write instead of publishing a partial file. + */ + static final class ParquetOutputFile implements org.apache.parquet.io.OutputFile { + private final OutputFile kernelOutputFile; + private final boolean atomicWrite; + private PositionOutputStream stream; - @Override - public boolean supportsBlockSize() { - return false; - } + ParquetOutputFile(OutputFile kernelOutputFile, boolean atomicWrite) { + this.kernelOutputFile = kernelOutputFile; + this.atomicWrite = atomicWrite; + } - @Override - public long defaultBlockSize() { - // blockSizeHint is hint used in HDFS compliant file systems. In cloud storage systems - // it is irrelevant. So, return some default value. - return 128 * 1024 * 1024; // 128MB - } + @Override + public org.apache.parquet.io.PositionOutputStream create(long blockSizeHint) + throws IOException { + // blockSizeHint is hint used in HDFS compliant file systems. In cloud storage systems + // it is irrelevant. So, we ignore it. + PositionOutputStream posOutputStream = kernelOutputFile.create(atomicWrite); + this.stream = posOutputStream; + return new DelegatingPositionOutputStream(posOutputStream) { + @Override + public long getPos() throws IOException { + return posOutputStream.getPos(); + } + }; + } - @Override - public String getPath() { - return kernelOutputFile.path(); + /** + * Abort the in-progress write. The stream is only marked; the Parquet writer's close() then + * discards rather than publishes. No-op if no stream has been created yet. + */ + void abort() { + if (stream != null) { + stream.abort(); } - }; + } + + @Override + public org.apache.parquet.io.PositionOutputStream createOrOverwrite(long blockSizeHint) + throws IOException { + // In Kernel we never overwrite files, so this method is not used. + throw new UnsupportedOperationException("createOrOverwrite is not supported in Kernel"); + } + + @Override + public boolean supportsBlockSize() { + return false; + } + + @Override + public long defaultBlockSize() { + // blockSizeHint is hint used in HDFS compliant file systems. In cloud storage systems + // it is irrelevant. So, return some default value. + return 128 * 1024 * 1024; // 128MB + } + + @Override + public String getPath() { + return kernelOutputFile.path(); + } } } diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/parquet/ParquetStatsReader.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/parquet/ParquetStatsReader.java index 8a12897a3e5..5e37931b525 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/parquet/ParquetStatsReader.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/parquet/ParquetStatsReader.java @@ -36,8 +36,6 @@ import org.apache.parquet.column.statistics.*; import org.apache.parquet.column.statistics.geospatial.BoundingBox; import org.apache.parquet.column.statistics.geospatial.GeospatialStatistics; -import org.apache.parquet.format.converter.ParquetMetadataConverter; -import org.apache.parquet.hadoop.ParquetFileReader; import org.apache.parquet.hadoop.metadata.BlockMetaData; import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData; import org.apache.parquet.hadoop.metadata.ParquetMetadata; @@ -57,11 +55,14 @@ public class ParquetStatsReader { public static DataFileStatistics readDataFileStatistics( InputFile kernelInputFile, StructType dataSchema, List statsColumns) throws IOException { - // Read the Parquet footer to compute the statistics + // Read the Parquet footer to compute the statistics. Pass the configuration explicitly so + // `parquet-mr` doesn't construct a fresh Hadoop Configuration here; see + // ParquetIOUtils#parquetConfiguration. org.apache.parquet.io.InputFile parquetFile = ParquetIOUtils.createParquetInputFile(kernelInputFile); ParquetMetadata footer = - ParquetFileReader.readFooter(parquetFile, ParquetMetadataConverter.NO_FILTER); + ParquetIOUtils.readFooter( + parquetFile, ParquetIOUtils.parquetConfiguration(kernelInputFile)); ImmutableMultimap.Builder metadataForColumn = ImmutableMultimap.builder(); diff --git a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/ConflictResolutionSuite.scala b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/ConflictResolutionSuite.scala index 7d52cd0d1a6..8c4aec06a7a 100644 --- a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/ConflictResolutionSuite.scala +++ b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/ConflictResolutionSuite.scala @@ -19,17 +19,21 @@ import java.util.{Collections, Optional} import scala.collection.JavaConverters._ +import io.delta.kernel.{Operation, TableManager} +import io.delta.kernel.commit.{CatalogCommitter, CommitFailedException, CommitMetadata, CommitResponse, Committer, PublishMetadata} import io.delta.kernel.data.Row -import io.delta.kernel.defaults.utils.{TestUtilsWithTableManagerAPIs, WriteUtils} +import io.delta.kernel.defaults.utils.{TestCommitterUtils, TestUtilsWithTableManagerAPIs, WriteUtils} import io.delta.kernel.engine.Engine import io.delta.kernel.exceptions.ConcurrentWriteException import io.delta.kernel.expressions.Literal import io.delta.kernel.internal.actions.{AddFile, DeletionVectorDescriptor, RemoveFile, SingleAction} import io.delta.kernel.internal.data.GenericRow +import io.delta.kernel.internal.tablefeatures.TableFeatures import io.delta.kernel.internal.util.PartitionUtils import io.delta.kernel.internal.util.Utils.toCloseableIterator import io.delta.kernel.utils.CloseableIterable import io.delta.kernel.utils.CloseableIterable.{emptyIterable, inMemoryIterable} +import io.delta.kernel.utils.CloseableIterator import org.scalatest.funsuite.AnyFunSuite @@ -45,7 +49,8 @@ import org.scalatest.funsuite.AnyFunSuite class ConflictResolutionSuite extends AnyFunSuite with WriteUtils - with TestUtilsWithTableManagerAPIs { + with TestUtilsWithTableManagerAPIs + with TestCommitterUtils { private def iterableOf(rows: Row*): CloseableIterable[Row] = inMemoryIterable(toCloseableIterator(rows.toSeq.asJava.iterator())) @@ -205,4 +210,70 @@ class ConflictResolutionSuite commitTransaction(losingTxn, engine, iterableOf(createAddFileRow("file3.parquet"))) } } + + test("conflict resolution - unavailable catalog-managed winning commits ask caller to retry") { + withTempDirAndEngine { (tablePath, engine) => + val catalogManagedProperties = Map( + TableFeatures.CATALOG_MANAGED_RW_FEATURE.getTableFeatureSupportKey -> "supported") + TableManager + .buildCreateTableTransaction(tablePath, testSchema, "test-engine") + .withTableProperties(catalogManagedProperties.asJava) + .withCommitter(customCatalogCommitter) + .build(engine) + .commit(engine, emptyIterable()) + + val conflictingCommitter = new CatalogCommitter { + override def commit( + engine: Engine, + finalizedActions: CloseableIterator[Row], + commitMetadata: CommitMetadata): CommitResponse = { + throw new CommitFailedException( + true, // retryable + true, // conflict + "Winning commit has not been published") + } + + override def getRequiredTableProperties: java.util.Map[String, String] = + customCatalogCommitter.getRequiredTableProperties + + override def publish(engine: Engine, publishMetadata: PublishMetadata): Unit = {} + } + val transaction = TableManager.loadSnapshot(tablePath) + .withCommitter(conflictingCommitter) + .withMaxCatalogVersionIfApplicable(true, 0) + .build(engine) + .buildUpdateTableTransaction("test-engine", Operation.WRITE) + .build(engine) + + val exception = intercept[ConcurrentWriteException] { + commitTransaction(transaction, engine, emptyIterable()) + } + assert(exception.getMessage.contains("re-executed using the latest version")) + } + } + + test("conflict resolution - unavailable filesystem winning commits remain an invariant error") { + withTempDirAndEngine { (tablePath, engine) => + commitTransaction(getCreateTxn(engine, tablePath, testSchema), engine, emptyIterable()) + + val conflictingCommitter = new Committer { + override def commit( + engine: Engine, + finalizedActions: CloseableIterator[Row], + commitMetadata: CommitMetadata): CommitResponse = { + throw new CommitFailedException(true, true, "Winning commit is missing") + } + } + val transaction = TableManager.loadSnapshot(tablePath) + .withCommitter(conflictingCommitter) + .build(engine) + .buildUpdateTableTransaction("test-engine", Operation.WRITE) + .build(engine) + + val exception = intercept[IllegalStateException] { + commitTransaction(transaction, engine, emptyIterable()) + } + assert(exception.getMessage.contains("No winning commits found")) + } + } } diff --git a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/CreateCheckpointSuite.scala b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/CreateCheckpointSuite.scala index 462b5d4ffbb..ca8d170af0b 100644 --- a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/CreateCheckpointSuite.scala +++ b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/CreateCheckpointSuite.scala @@ -16,20 +16,27 @@ package io.delta.kernel.defaults import java.io.File +import java.util.Optional import scala.collection.immutable.Seq import io.delta.golden.GoldenTableUtils.goldenTablePath import io.delta.kernel.{Table, TableManager} -import io.delta.kernel.defaults.engine.DefaultEngine +import io.delta.kernel.data.ColumnarBatch +import io.delta.kernel.defaults.engine.{DefaultEngine, DefaultJsonHandler} +import io.delta.kernel.defaults.engine.hadoopio.HadoopFileIO +import io.delta.kernel.defaults.internal.parquet.ParquetSuiteBase import io.delta.kernel.defaults.utils.{GeoTestUtils, TestRow, TestUtils, WriteUtils} -import io.delta.kernel.engine.Engine -import io.delta.kernel.exceptions.{CheckpointAlreadyExistsException, TableNotFoundException} -import io.delta.kernel.expressions.{Column, Literal} +import io.delta.kernel.engine.{Engine, JsonHandler} +import io.delta.kernel.exceptions.{CheckpointAlreadyExistsException, KernelEngineException, TableNotFoundException} +import io.delta.kernel.expressions.{Column, Literal, Predicate} import io.delta.kernel.internal.{SnapshotImpl, TableConfig} +import io.delta.kernel.internal.actions.SingleAction import io.delta.kernel.internal.tablefeatures.TableFeatures.GEOSPATIAL_RW_FEATURE +import io.delta.kernel.internal.util.Utils.toCloseableIterator import io.delta.kernel.statistics.DataFileStatistics import io.delta.kernel.types.{GeometryType, StructType => KernelStructType} +import io.delta.kernel.utils.{CloseableIterator, FileStatus} import org.apache.spark.sql.delta.{DeltaLog, VersionNotFoundException} import org.apache.spark.sql.delta.DeltaOperations.ManualUpdate @@ -45,7 +52,7 @@ import org.scalatest.funsuite.AnyFunSuite /** * Test suite for `io.delta.kernel.Table.checkpoint(engine, version)` */ -class CreateCheckpointSuite extends CheckpointBase with GeoTestUtils { +class CreateCheckpointSuite extends CheckpointBase with GeoTestUtils with ParquetSuiteBase { /////////// // Tests // @@ -592,6 +599,49 @@ class CreateCheckpointSuite extends CheckpointBase with GeoTestUtils { "Log cleanup should NOT happen with checkpointProtection enabled") } } + + test("checkpoint write must not leave a partial checkpoint " + + "file when log replay fails mid-write") { + withTempDirAndEngine { (tablePath, tc) => + setupTestTable(tc, tablePath, tableProperties = Map.empty[String, String], commits = 5) + + val snapshot = TableManager.loadSnapshot(tablePath).build(tc).asInstanceOf[SnapshotImpl] + val checkpointVersion = snapshot.getVersion + val v0CommitName = f"${0L}%020d.json" + + val fileIO = new HadoopFileIO(new Configuration()) + val failingEngine = new DefaultEngine(fileIO) { + override def getJsonHandler: JsonHandler = new DefaultJsonHandler(fileIO) { + override def readJsonFiles( + scanFileIter: CloseableIterator[FileStatus], + physicalSchema: KernelStructType, + predicate: Optional[Predicate]): CloseableIterator[ColumnarBatch] = { + val files = new java.util.ArrayList[FileStatus]() + while (scanFileIter.hasNext) { + val fileStatus = scanFileIter.next() + if (fileStatus.getPath.endsWith(v0CommitName)) { + throw new RuntimeException("simulating read failure") + } + files.add(fileStatus) + } + super.readJsonFiles(toCloseableIterator(files.iterator()), physicalSchema, predicate) + } + } + } + + intercept[KernelEngineException] { + snapshot.writeCheckpoint(failingEngine) + } + + // With the abort-on-read-failure fix, a checkpoint whose log replay fails part-way through + // must NOT publish any file at the destination: the write is aborted, not finalized. + val checkpointFile = new File(checkpointFilePath(tablePath, checkpointVersion)) + if (checkpointFile.exists()) { + fail(s"aborted checkpoint write must not publish a file, " + + s"but one was left at ${checkpointFile.getPath}") + } + } + } } /** diff --git a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/engine/hadoopio/HadoopOutputFileSuite.scala b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/engine/hadoopio/HadoopOutputFileSuite.scala new file mode 100644 index 00000000000..ebd6ce45e10 --- /dev/null +++ b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/engine/hadoopio/HadoopOutputFileSuite.scala @@ -0,0 +1,103 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.kernel.defaults.engine.hadoopio + +import java.io.File +import java.nio.file.Files + +import io.delta.kernel.defaults.utils.TestUtils +import io.delta.storage.{CloseableIterator, LogStore} + +import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.fs.{FileStatus, Path} +import org.scalatest.funsuite.AnyFunSuite + +class HadoopOutputFileSuite extends AnyFunSuite with TestUtils { + + /** + * Registers [[NonPartialWriteVisibleLogStore]] as the LogStore for the `file` scheme, so that + * `HadoopOutputFile.create(putIfAbsent = true)` resolves `useRename = false`, i.e. the returned + * stream writes directly to the target path instead of a temp-then-rename path. + */ + private def confWithNonRenamingLogStore(): Configuration = { + val conf = new Configuration() + conf.set( + "io.delta.kernel.logStore.file.impl", + classOf[NonPartialWriteVisibleLogStore].getName) + conf + } + + test("abort() with useRename=false discards the write and publishes no file") { + withTempDir { tempDir => + val targetFile = new File(tempDir, "target.txt") + val outputFile = + new HadoopOutputFile(confWithNonRenamingLogStore(), targetFile.toURI.toString) + + val stream = outputFile.create( /* putIfAbsent = */ true) + stream.write("partial-content".getBytes("UTF-8")) + stream.abort() + stream.close() + + assert( + !targetFile.exists(), + "an aborted write with useRename=false must not publish a file at the target path") + } + } + + test("normal write with useRename=false publishes the file") { + withTempDir { tempDir => + val targetFile = new File(tempDir, "target.txt") + val outputFile = + new HadoopOutputFile(confWithNonRenamingLogStore(), targetFile.toURI.toString) + + val content = "full-content".getBytes("UTF-8") + val stream = outputFile.create( /* putIfAbsent = */ true) + stream.write(content) + stream.close() + + assert(targetFile.exists(), "a non-aborted write must publish a file at the target path") + assertResult(content.toSeq)(Files.readAllBytes(targetFile.toPath).toSeq) + } + } +} + +/** A [[LogStore]] whose `isPartialWriteVisible` is `false`, e.g. mimicking GCS/S3-like stores. */ +class NonPartialWriteVisibleLogStore(initHadoopConf: Configuration) + extends LogStore(initHadoopConf) { + + override def read( + path: Path, + hadoopConf: Configuration): CloseableIterator[String] = + throw new UnsupportedOperationException("not used in this test") + + override def write( + path: Path, + actions: java.util.Iterator[String], + overwrite: java.lang.Boolean, + hadoopConf: Configuration): Unit = + throw new UnsupportedOperationException("not used in this test") + + override def listFrom( + path: Path, + hadoopConf: Configuration): java.util.Iterator[FileStatus] = + throw new UnsupportedOperationException("not used in this test") + + override def resolvePathOnPhysicalStorage(path: Path, hadoopConf: Configuration): Path = + throw new UnsupportedOperationException("not used in this test") + + override def isPartialWriteVisible(path: Path, hadoopConf: Configuration): java.lang.Boolean = + false +} diff --git a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/internal/parquet/ParquetFileReaderSuite.scala b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/internal/parquet/ParquetFileReaderSuite.scala index 88a7baf2462..77a1308c14e 100644 --- a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/internal/parquet/ParquetFileReaderSuite.scala +++ b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/internal/parquet/ParquetFileReaderSuite.scala @@ -16,14 +16,23 @@ package io.delta.kernel.defaults.internal.parquet import java.math.BigDecimal -import java.util.TimeZone +import java.nio.file.{Files, Paths} +import java.util.{Optional, TimeZone} +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicInteger import io.delta.golden.GoldenTableUtils.{goldenTableFile, goldenTablePath} +import io.delta.kernel.defaults.engine.fileio.{FileIO, InputFile, OutputFile, SeekableInputStream} +import io.delta.kernel.defaults.engine.hadoopio.HadoopFileIO import io.delta.kernel.defaults.utils.{ExpressionTestUtils, TestRow} +import io.delta.kernel.expressions.Literal.ofLong +import io.delta.kernel.expressions.Predicate import io.delta.kernel.test.VectorTestUtils import io.delta.kernel.types._ -import io.delta.kernel.utils.MetadataColumnTestUtils +import io.delta.kernel.utils.{CloseableIterator, MetadataColumnTestUtils} +import io.delta.kernel.utils.{FileStatus => KernelFileStatus} +import org.apache.hadoop.conf.Configuration import org.apache.spark.sql.internal.SQLConf import org.scalatest.funsuite.AnyFunSuite import org.slf4j.LoggerFactory @@ -33,6 +42,18 @@ class ParquetFileReaderSuite extends AnyFunSuite with MetadataColumnTestUtils { private val logger = LoggerFactory.getLogger(classOf[ParquetFileReaderSuite]) + test("parquetConfiguration reuses the HadoopInputFile's configuration") { + // Disable the FileSystem cache for this scheme so newInputFile initializes a fresh + // FileSystem carrying this exact Configuration, rather than a cached one whose conf + // (the cache keys on scheme+authority+UGI, not conf) would lack the sentinel key. + val conf = new Configuration(false) + conf.setBoolean("fs.file.impl.disable.cache", true) + conf.set("my.sentinel.key", "sentinel-value") + val input = new HadoopFileIO(conf).newInputFile("file:/tmp/does-not-need-to-exist", 0) + val parquetConf = ParquetIOUtils.parquetConfiguration(input) + assertResult("sentinel-value")(parquetConf.get("my.sentinel.key")) + } + test("decimals encoded using dictionary encoding ") { // Below golden tables contains three decimal columns // each stored in a different physical format: int32, int64 and fixed binary @@ -329,6 +350,41 @@ class ParquetFileReaderSuite extends AnyFunSuite checkAnswer(actResult2, expResult2) } + ///////////////////////////////////////////////////////////////////////////////////////////////// + // Tests for footer reuse: the footer read up-front to build the filter is reused by the reader // + // instead of being read (and parsed) a second time. // + ///////////////////////////////////////////////////////////////////////////////////////////////// + + test("footer is read only once when reading a Parquet file") { + // Multiple row groups + a predicate that prunes some of them exercises the full record-read + // loop (row-group iteration, stats filtering and row-index tracking) while still needing only + // a single footer read. + val filePath = getTestResourceFilePath("parquet/row_index_multiple_row_groups.parquet") + val readSchema = new StructType() + .add("id", LongType.LONG) + .add(ROW_INDEX) + val fileStatus = KernelFileStatus.of( + filePath, + Files.size(Paths.get(filePath)), + /* modificationTime = */ 0L) + + // Read once with no predicate and once with a predicate that prunes row groups; the footer + // should be read exactly once in both cases. + Seq[Optional[Predicate]]( + Optional.empty(), + Optional.of(gt(col("id"), ofLong(5000L)))).foreach { predicate => + val fileIO = new FooterReadCountingFileIO(new HadoopFileIO(configuration)) + val reader = new ParquetFileReader(fileIO) + // Drain the iterator (the implicit `forEach` closes it) so all row groups are read. + reader.read(fileStatus, readSchema, predicate).forEach(_ => ()) + + assert( + fileIO.footerReadCount(filePath) === 1, + s"Expected the footer to be read exactly once for predicate=$predicate, " + + s"but it was read ${fileIO.footerReadCount(filePath)} times") + } + } + ///////////////////////////////////////////////////////////////////////////////////////////////// // Test compatibility with Parquet legacy format files // ///////////////////////////////////////////////////////////////////////////////////////////////// @@ -386,3 +442,68 @@ class ParquetFileReaderSuite extends AnyFunSuite readParquetFilesUsingSpark(parquetFilePath, readSchema) /* expected */ ) } } + +/** + * A [[FileIO]] that delegates to an underlying [[FileIO]] but counts how many times the Parquet + * footer is read for each file. A footer read is detected by a seek to the footer-length index + * (`fileLength - 8`), which is where `parquet-mr` starts reading the footer trailer. + */ +class FooterReadCountingFileIO(delegate: FileIO) extends FileIO { + private val footerReadCounts = new ConcurrentHashMap[String, AtomicInteger]() + + def footerReadCount(path: String): Int = + Option(footerReadCounts.get(path)).map(_.get()).getOrElse(0) + + private def recordFooterRead(path: String): Unit = { + footerReadCounts.putIfAbsent(path, new AtomicInteger(0)) + footerReadCounts.get(path).incrementAndGet() + } + + override def newInputFile(path: String, fileSize: Long): InputFile = { + val delegateInputFile = delegate.newInputFile(path, fileSize) + new InputFile { + override def length(): Long = delegateInputFile.length() + + override def path(): String = delegateInputFile.path() + + override def newStream(): SeekableInputStream = { + val delegateStream = delegateInputFile.newStream() + // `fileLength - 8` == fileLength - MAGIC(4) - FOOTER_LENGTH_SIZE(4), the position + // parquet-mr seeks to when it reads the footer. + val footerIndex = delegateInputFile.length() - 8 + new SeekableInputStream { + override def getPos(): Long = delegateStream.getPos() + + override def seek(newPos: Long): Unit = { + if (newPos == footerIndex) { + recordFooterRead(path) + } + delegateStream.seek(newPos) + } + + override def readFully(b: Array[Byte], off: Int, len: Int): Unit = + delegateStream.readFully(b, off, len) + + override def read(): Int = delegateStream.read() + + override def read(b: Array[Byte], off: Int, len: Int): Int = + delegateStream.read(b, off, len) + + override def close(): Unit = delegateStream.close() + } + } + } + } + + // All other operations simply delegate. + override def listFrom(filePath: String): CloseableIterator[KernelFileStatus] = + delegate.listFrom(filePath) + override def getFileStatus(path: String): KernelFileStatus = delegate.getFileStatus(path) + override def resolvePath(path: String): String = delegate.resolvePath(path) + override def mkdirs(path: String): Boolean = delegate.mkdirs(path) + override def newOutputFile(path: String): OutputFile = delegate.newOutputFile(path) + override def delete(path: String): Boolean = delegate.delete(path) + override def getConf(confKey: String): Optional[String] = delegate.getConf(confKey) + override def copyFileAtomically(srcPath: String, destPath: String, overwrite: Boolean): Unit = + delegate.copyFileAtomically(srcPath, destPath, overwrite) +} diff --git a/kernel/unitycatalog/src/main/java/io/delta/kernel/unitycatalog/UCCatalogManagedClient.java b/kernel/unitycatalog/src/main/java/io/delta/kernel/unitycatalog/UCCatalogManagedClient.java index 7f91392297c..ece32c9a299 100644 --- a/kernel/unitycatalog/src/main/java/io/delta/kernel/unitycatalog/UCCatalogManagedClient.java +++ b/kernel/unitycatalog/src/main/java/io/delta/kernel/unitycatalog/UCCatalogManagedClient.java @@ -27,6 +27,7 @@ import io.delta.kernel.annotation.Experimental; import io.delta.kernel.commit.Committer; import io.delta.kernel.engine.Engine; +import io.delta.kernel.exceptions.VersionToLoadAfterLatestCommitException; import io.delta.kernel.internal.annotation.VisibleForTesting; import io.delta.kernel.internal.files.ParsedCatalogCommitData; import io.delta.kernel.internal.files.ParsedLogData; @@ -88,8 +89,10 @@ public UCCatalogManagedClient(UCClient ucClient) { * mutually exclusive with timestampOpt. * @param timestampOpt The optional timestamp to time-travel to when loading the table. This must * be mutually exclusive with versionOpt. - * @throws IllegalArgumentException if a negative version or timestamp is provided + * @throws IllegalArgumentException if a negative version is provided * @throws IllegalArgumentException if both versionOpt and timestampOpt are defined + * @throws VersionToLoadAfterLatestCommitException if the requested version is greater than the + * latest version ratified by UC */ public Snapshot loadSnapshot( Engine engine, @@ -273,7 +276,7 @@ public CreateTableTransactionBuilder buildCreateTableTransaction( * @throws IllegalArgumentException if neither startVersionOpt nor startTimestampOpt is provided * @throws IllegalArgumentException if both startVersionOpt and startTimestampOpt are defined * @throws IllegalArgumentException if both endVersionOpt and endTimestampOpt are defined - * @throws IllegalArgumentException if either startVersionOpt or endVersionOpt is provided and is + * @throws VersionToLoadAfterLatestCommitException if either startVersionOpt or endVersionOpt is * greater than the latest ratified version from UC */ public CommitRange loadCommitRange( @@ -486,7 +489,7 @@ private GetCommitsResponse getRatifiedCommitsFromUC( private void validateTimeTravelVersionNotPastMax( String ucTableId, long tableVersionToLoad, long maxRatifiedVersion) { if (tableVersionToLoad > maxRatifiedVersion) { - throw new IllegalArgumentException( + throw new VersionToLoadAfterLatestCommitException( String.format( "[%s] Cannot load table version %s as the latest version ratified by UC is %s", ucTableId, tableVersionToLoad, maxRatifiedVersion)); @@ -501,7 +504,7 @@ private void validateVersionBoundariesExist( BiConsumer validateVersion = (version, type) -> { if (version > maxRatifiedVersion) { - throw new IllegalArgumentException( + throw new VersionToLoadAfterLatestCommitException( String.format( "[%s] Cannot load commit range with %s version %d as the latest version " + "ratified by UC is %d", diff --git a/kernel/unitycatalog/src/test/scala/io/delta/kernel/unitycatalog/UCCatalogManagedClientCommitRangeSuite.scala b/kernel/unitycatalog/src/test/scala/io/delta/kernel/unitycatalog/UCCatalogManagedClientCommitRangeSuite.scala index 8bcfffc69e2..ee49b190e0d 100644 --- a/kernel/unitycatalog/src/test/scala/io/delta/kernel/unitycatalog/UCCatalogManagedClientCommitRangeSuite.scala +++ b/kernel/unitycatalog/src/test/scala/io/delta/kernel/unitycatalog/UCCatalogManagedClientCommitRangeSuite.scala @@ -18,7 +18,7 @@ package io.delta.kernel.unitycatalog import java.util.Optional import io.delta.kernel.engine.Engine -import io.delta.kernel.exceptions.KernelException +import io.delta.kernel.exceptions.{KernelException, VersionToLoadAfterLatestCommitException} import io.delta.storage.commit.uccommitcoordinator.InvalidTargetTableException import org.scalatest.funsuite.AnyFunSuite @@ -162,33 +162,31 @@ class UCCatalogManagedClientCommitRangeSuite extends AnyFunSuite with UCCatalogM "Cannot provide a start timestamp greater than the end timestamp")) } - test("loadCommitRange throws if startVersion is greater than max ratified version") { - val ucClient = new InMemoryUCClient("ucMetastoreId") - val ucCatalogManagedClient = new UCCatalogManagedClient(ucClient) - - val ex = intercept[IllegalArgumentException] { - testLoadCommitRange( - expectedStartVersion = 0, - expectedEndVersion = 2, - startVersionOpt = Optional.of(9L)) + Seq(3L, 9L).foreach { versionToLoad => + test(s"loadCommitRange throws a typed exception for start version $versionToLoad above max") { + val ex = intercept[VersionToLoadAfterLatestCommitException] { + testLoadCommitRange( + expectedStartVersion = 0, + expectedEndVersion = 2, + startVersionOpt = Optional.of(versionToLoad)) + } + assert(ex.getMessage == + s"[testUcTableId] Cannot load commit range with start version $versionToLoad " + + "as the latest version ratified by UC is 2") } - assert(ex.getMessage.contains( - "Cannot load commit range with start version 9 as the latest version ratified by UC is 2")) - } - test("loadCommitRange throws if endVersion is greater than max ratified version") { - val ucClient = new InMemoryUCClient("ucMetastoreId") - val ucCatalogManagedClient = new UCCatalogManagedClient(ucClient) - - val ex = intercept[IllegalArgumentException] { - testLoadCommitRange( - expectedStartVersion = 0, - expectedEndVersion = 2, - startVersionOpt = Optional.of(0L), - endVersionOpt = Optional.of(9L)) + test(s"loadCommitRange throws a typed exception for end version $versionToLoad above max") { + val ex = intercept[VersionToLoadAfterLatestCommitException] { + testLoadCommitRange( + expectedStartVersion = 0, + expectedEndVersion = 2, + startVersionOpt = Optional.of(0L), + endVersionOpt = Optional.of(versionToLoad)) + } + assert(ex.getMessage == + s"[testUcTableId] Cannot load commit range with end version $versionToLoad " + + "as the latest version ratified by UC is 2") } - assert(ex.getMessage.contains( - "Cannot load commit range with end version 9 as the latest version ratified by UC is 2")) } test("loadCommitRange throws when no start boundary is provided") { diff --git a/kernel/unitycatalog/src/test/scala/io/delta/kernel/unitycatalog/UCCatalogManagedClientSuite.scala b/kernel/unitycatalog/src/test/scala/io/delta/kernel/unitycatalog/UCCatalogManagedClientSuite.scala index 2e1d509d80e..2b781133fe2 100644 --- a/kernel/unitycatalog/src/test/scala/io/delta/kernel/unitycatalog/UCCatalogManagedClientSuite.scala +++ b/kernel/unitycatalog/src/test/scala/io/delta/kernel/unitycatalog/UCCatalogManagedClientSuite.scala @@ -20,7 +20,7 @@ import java.util.Optional import scala.collection.JavaConverters._ -import io.delta.kernel.exceptions.KernelException +import io.delta.kernel.exceptions.{KernelException, VersionToLoadAfterLatestCommitException} import io.delta.kernel.internal.CreateTableTransactionBuilderImpl import io.delta.kernel.internal.tablefeatures.TableFeatures import io.delta.kernel.internal.tablefeatures.TableFeatures.{CATALOG_MANAGED_RW_FEATURE, TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION} @@ -190,12 +190,16 @@ class UCCatalogManagedClientSuite extends AnyFunSuite with UCCatalogManagedTestU testCatalogManagedTable(versionToLoad = Optional.of(0L)) } - test("loadTable throws if version to load is greater than max ratified version") { - val exMsg = intercept[IllegalArgumentException] { - testCatalogManagedTable(versionToLoad = Optional.of(9L)) - }.getMessage + Seq(3L, 9L).foreach { versionToLoad => + test(s"loadTable throws a typed exception for version $versionToLoad above max ratified") { + val ex = intercept[VersionToLoadAfterLatestCommitException] { + testCatalogManagedTable(versionToLoad = Optional.of(versionToLoad)) + } - assert(exMsg.contains("Cannot load table version 9 as the latest version ratified by UC is 2")) + assert(ex.getMessage == + s"[$testUcTableId] Cannot load table version $versionToLoad " + + "as the latest version ratified by UC is 2") + } } /* ---- Time-travel-by-timestamp tests --- */ diff --git a/kernel/unitycatalog/src/test/scala/io/delta/kernel/unitycatalog/UCCatalogManagedCommitterSuite.scala b/kernel/unitycatalog/src/test/scala/io/delta/kernel/unitycatalog/UCCatalogManagedCommitterSuite.scala index e17ba447218..38f2d34b75c 100644 --- a/kernel/unitycatalog/src/test/scala/io/delta/kernel/unitycatalog/UCCatalogManagedCommitterSuite.scala +++ b/kernel/unitycatalog/src/test/scala/io/delta/kernel/unitycatalog/UCCatalogManagedCommitterSuite.scala @@ -750,6 +750,7 @@ class UCCatalogManagedCommitterSuite // ===== THEN ===== assert(ex.isRetryable && !ex.isConflict) assert(ex.getMessage.contains("Failed to write delta file due to: Network hiccup")) + assert(ucClient.getLastFinalizeCreateRecord.isEmpty) } } diff --git a/project/CrossFlinkVersions.scala b/project/CrossFlinkVersions.scala new file mode 100644 index 00000000000..93a1c104506 --- /dev/null +++ b/project/CrossFlinkVersions.scala @@ -0,0 +1,127 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.{File, PrintWriter} + +import sbt._ +import sbt.Keys._ +import sbtrelease.ReleasePlugin.autoImport.ReleaseStep + +case class FlinkVersionSpec(fullVersion: String) { + + /** Returns the Flink compatibility version used in the Maven artifact name. */ + def shortVersion: String = { + val (major, minor, _) = Mima.getMajorMinorPatch(fullVersion) + s"$major.$minor" + } +} + +object FlinkVersionSpec { + private val defaultVersion = "2.3.0" + + val DEFAULT = FlinkVersionSpec(defaultVersion) + + val ALL_SPECS = + Seq("2.0.2", "2.1.3", "2.2.1", "2.3.0", defaultVersion) + .distinct + .map(FlinkVersionSpec.apply) + + require( + ALL_SPECS.map(_.shortVersion).distinct.size == ALL_SPECS.size, + "Only one patch release can be published for each Flink compatibility line") +} + +object CrossFlinkVersions extends AutoPlugin { + + /** Returns the selected Flink version specification. */ + def getFlinkVersionSpec(): FlinkVersionSpec = { + val input = sys.props.getOrElse("flinkVersion", FlinkVersionSpec.DEFAULT.fullVersion) + FlinkVersionSpec.ALL_SPECS.find { spec => + spec.fullVersion == input || spec.shortVersion == input + }.getOrElse { + val validInputs = FlinkVersionSpec.ALL_SPECS.flatMap { spec => + Seq(spec.fullVersion, spec.shortVersion) + } + throw new IllegalArgumentException( + s"Invalid flinkVersion: $input. Valid values: ${validInputs.mkString(", ")}") + } + } + + /** Returns the Maven artifact version used for Apache Flink dependencies. */ + def getFlinkArtifactVersion(): String = getFlinkVersionSpec().fullVersion + + override def trigger = allRequirements + + /** Adds one release step for every supported Flink version. */ + def crossFlinkReleaseSteps(task: String): Seq[ReleaseStep] = { + FlinkVersionSpec.ALL_SPECS.map { spec => + { (state: State) => + val extracted = Project.extract(state) + val baseDir = extracted.get(ThisBuild / Keys.baseDirectory) + val command = Seq( + s"${baseDir.getAbsolutePath}/build/sbt", + s"-DflinkVersion=${spec.fullVersion}", + s"flink/$task") + + println(s"[info] Publishing Flink module for Flink ${spec.fullVersion}") + val exitCode = scala.sys.process.Process(command, baseDir).! + if (exitCode != 0) { + sys.error( + s"Publishing Flink module for Flink ${spec.fullVersion} failed with exit code $exitCode") + } + state + }: ReleaseStep + } + } + + override lazy val projectSettings = Seq( + // Runs the same Flink publication steps used by releaseProcess. This command lets CI exercise + // the real release path with a non-releasing task such as publishM2. + commands += Command.args("publishFlinkVersions", "") { (state, args) => + if (args.isEmpty) { + sys.error( + "Usage: publishFlinkVersions \n" + + "Example: build/sbt \"publishFlinkVersions publishM2\"") + } + crossFlinkReleaseSteps(args.mkString(" ")).foldLeft(state) { (currentState, step) => + step(currentState) + } + }, + // Exports the supported Flink versions so CI can build its test matrix from the SBT source of + // truth instead of maintaining a separate version list in the workflow. + commands += Command.command("exportFlinkVersionsJson") { state => + val outputFile = new File("target/flink-versions.json") + outputFile.getParentFile.mkdirs() + + val writer = new PrintWriter(outputFile) + try { + writer.println("[") + FlinkVersionSpec.ALL_SPECS.zipWithIndex.foreach { case (spec, index) => + val comma = if (index < FlinkVersionSpec.ALL_SPECS.size - 1) "," else "" + writer.println(" {") + writer.println(s""" "fullVersion": "${spec.fullVersion}",""") + writer.println(s""" "compatibilityVersion": "${spec.shortVersion}"""") + writer.println(s" }$comma") + } + writer.println("]") + } finally { + writer.close() + } + + println(s"[info] Flink version information exported to: ${outputFile.getAbsolutePath}") + state + }) +} diff --git a/project/CrossSparkVersions.scala b/project/CrossSparkVersions.scala index 202c2ee6921..9c27d58489c 100644 --- a/project/CrossSparkVersions.scala +++ b/project/CrossSparkVersions.scala @@ -135,7 +135,8 @@ import Unidoc._ * Step 2: Publish Spark-dependent modules WITH suffix for each non-master Spark version * build/sbt -DsparkVersion=4.0 "runOnlyForReleasableSparkModules publishSigned" * build/sbt -DsparkVersion=4.1 "runOnlyForReleasableSparkModules publishSigned" - * # Publishes: delta-spark_4.0_2.13, delta-spark_4.1_2.13, etc. + * build/sbt -DsparkVersion=4.2 "runOnlyForReleasableSparkModules publishSigned" + * # Publishes: delta-spark_4.0_2.13, delta-spark_4.1_2.13, delta-spark_4.2_2.13, etc. * * This workflow is automated via crossSparkReleaseSteps() in the release process. * See releaseProcess in build.sbt for integration. @@ -153,6 +154,7 @@ import Unidoc._ * build/sbt -DskipSparkSuffix=true publishM2 # Without suffix (backward compat) * build/sbt -DsparkVersion=4.0 "runOnlyForReleasableSparkModules publishM2" * build/sbt -DsparkVersion=4.1 "runOnlyForReleasableSparkModules publishM2" + * build/sbt -DsparkVersion=4.2 "runOnlyForReleasableSparkModules publishM2" * # Verify JARs in ~/.m2/repository/io/delta/ * * ======================================================== @@ -194,7 +196,7 @@ import Unidoc._ * Example: * build/sbt exportSparkVersionsJson * # Generates: target/spark-versions.json - * # Output: [{"fullVersion": "4.0.1", "shortVersion": "4.0", "isMaster": false, "isDefault": true, "targetJvm": "17", "packageSuffix": "_4.0"}, ...] + * # Output: [{"fullVersion": "4.0.1", ...}, ..., {"fullVersion": "4.2.0", "isDefault": true, ...}] * * Use with Python utilities to extract specific fields: * python3 project/scripts/get_spark_version_info.py --all-spark-versions @@ -318,7 +320,7 @@ object SparkVersionSpec { ) /** Default Spark version */ - val DEFAULT = spark41 + val DEFAULT = spark42 /** Spark master branch version (optional). Release branches should not build against master */ val MASTER: Option[SparkVersionSpec] = None @@ -516,12 +518,13 @@ object CrossSparkVersions extends AutoPlugin { * Generates release steps for cross-Spark publishing. * * Returns a sequence of release steps that: - * 1. Publishes all modules WITHOUT Spark suffix (backward compatibility) + * 1. Publishes non-Flink modules WITHOUT Spark suffix (backward compatibility) * 2. Publishes Spark-dependent modules WITH Spark suffix for each non-master version * - * For example, with Spark versions 4.0 (default) and 4.1: + * For example, with Spark versions 4.0, 4.1, and 4.2 (default): * - Step 1 publishes: delta-spark_2.13, delta-storage, delta-kernel-api, etc. (no suffix) - * - Step 2 publishes: delta-spark_4.0_2.13, delta-spark_4.1_2.13, etc. (with suffix) + * - Step 2 publishes: delta-spark_4.0_2.13, delta-spark_4.1_2.13, + * delta-spark_4.2_2.13, etc. (with suffix) * * Each step runs as a separate SBT subprocess so the build reloads with * the correct sparkVersion/skipSparkSuffix settings (SBT settings like @@ -557,13 +560,13 @@ object CrossSparkVersions extends AutoPlugin { state } - // Step 1: Publish ALL modules WITHOUT Spark suffix (backward compatibility) - // Uses skipSparkSuffix=true to get artifact names like delta-spark_2.13 + // Step 1: Publish non-Flink modules WITHOUT Spark suffix (backward compatibility). + // Flink is published separately for every supported version. val backwardCompatStep: ReleaseStep = { (state: State) => runSbtSubprocess( state, - Seq("-DskipSparkSuffix=true", task), - "Publishing all modules without Spark suffix (backward compat)" + Seq("-DskipSparkSuffix=true", "-DskipFlinkPublish=true", task), + "Publishing non-Flink modules without Spark suffix (backward compat)" ) } diff --git a/project/scripts/setup_unitycatalog_main.sh b/project/scripts/setup_unitycatalog_main.sh index 020a84d54bd..f951d2c4321 100755 --- a/project/scripts/setup_unitycatalog_main.sh +++ b/project/scripts/setup_unitycatalog_main.sh @@ -83,15 +83,15 @@ set -euo pipefail # The pin. Bump both lines together if UC's version.sbt changed at the new SHA. build.sbt's # `unityCatalogVersion` is obtained by running this script with `--print-version`, so these two # values are the single source of truth. -UC_PIN_SHA=b11188d2a92c9bf1395d03be4235706af26b2c45 -UC_BASE_VERSION=0.6.0-SNAPSHOT +UC_PIN_SHA=e36e9ca0259d13fdd8cb43edd57a12f22aead321 +UC_BASE_VERSION=0.7.0-SNAPSHOT # --------------------------------------------------------------------------------------------- UC_DIR="${UC_DIR:-/tmp/unitycatalog}" UC_REPO="${UC_REPO:-https://github.com/unitycatalog/unitycatalog.git}" UC_REF="${UC_REF:-$UC_PIN_SHA}" UC_FORCE="${UC_FORCE:-0}" -SPARK_VERSION="${SPARK_VERSION:-4.1}" +SPARK_VERSION="${SPARK_VERSION:-4.2}" DELTA_RELEASE_MODE="${DELTA_RELEASE_MODE:-0}" # Compose version coordinate. When UC_VERSION is set from env, use it verbatim (no SHA suffix). diff --git a/project/tests/test_cross_spark_publish.py b/project/tests/test_published_artifacts.py similarity index 89% rename from project/tests/test_cross_spark_publish.py rename to project/tests/test_published_artifacts.py index 95e173ad06b..fedc50255b4 100755 --- a/project/tests/test_cross_spark_publish.py +++ b/project/tests/test_published_artifacts.py @@ -1,19 +1,19 @@ #!/usr/bin/env python3 """ -Cross-Spark Version Build Testing +Published Artifact Build Testing -Tests the Delta Lake build system by validating JAR file names for: +Tests the Delta Lake build system by validating published JAR file names for: 1. Default publish (publishM2) - publishes ALL modules WITH Spark suffix 2. Backward-compat publish (skipSparkSuffix=true) - publishes WITHOUT suffix -3. Full cross-version workflow publishes both with and without suffix +3. Full release workflow publishes every Spark and Flink artifact Usage: - python project/tests/test_cross_spark_publish.py + python project/tests/test_published_artifacts.py The script will: 1. Test default publishM2 command publishes all modules WITH Spark suffix 2. Test skipSparkSuffix=true publishes WITHOUT suffix (backward compatibility) -3. Test full cross-version build workflow (both with and without suffix) +3. Test the full Spark and Flink release workflow 4. Exit with status 0 on success, 1 on failure """ @@ -61,7 +61,6 @@ "delta-kernel-defaults-{version}.jar", "delta-storage-s3-dynamodb-{version}.jar", "delta-kernel-unitycatalog-{version}.jar", - "delta-flink-{version}.jar", "delta-contribs_2.13-{version}.jar", ] @@ -105,6 +104,12 @@ def all_jars(self) -> List[str]: return self.spark_related_jars + self.non_spark_related_jars + self.iceberg_jars + self.hudi_jars +@dataclass +class FlinkVersionSpec: + """Expected artifact for a supported Flink version.""" + jar_template: str + + # Spark versions to test (key = full version string, value = spec with suffix) # By default, ALL versions get a Spark suffix (e.g., delta-spark_4.0_2.13) # skipSparkSuffix=true removes the suffix (used during release for backward compat) @@ -115,9 +120,20 @@ def all_jars(self) -> List[str]: "4.2.0": SparkVersionSpec(suffix="_4.2", support_iceberg=False, support_hudi=False) } +# Keep this map independent from CrossFlinkVersions so this test detects missing artifacts. +FLINK_VERSIONS: Dict[str, FlinkVersionSpec] = { + "2.0.2": FlinkVersionSpec(jar_template="delta-flink_2.0-{version}.jar"), + "2.1.3": FlinkVersionSpec(jar_template="delta-flink_2.1-{version}.jar"), + "2.2.1": FlinkVersionSpec(jar_template="delta-flink_2.2-{version}.jar"), + "2.3.0": FlinkVersionSpec(jar_template="delta-flink_2.3-{version}.jar"), +} + # The default Spark version # This is intentionally hardcoded here to explicitly test the default version. -DEFAULT_SPARK = "4.1.0" +DEFAULT_SPARK = "4.2.0" + +# The default Flink version. This is intentionally hardcoded to test the default version. +DEFAULT_FLINK = "2.3.0" def substitute_xversion(jar_templates: List[str], delta_version: str) -> Set[str]: @@ -127,8 +143,8 @@ def substitute_xversion(jar_templates: List[str], delta_version: str) -> Set[str return {jar.format(version=delta_version) for jar in jar_templates} -class CrossSparkPublishTest: - """Tests cross-Spark version builds.""" +class PublishedArtifactsTest: + """Tests the artifacts produced by default and cross-version builds.""" def __init__(self, delta_root: Path): self.delta_root = delta_root @@ -183,6 +199,35 @@ def run_sbt_command(self, description: str, command: List[str]) -> bool: print(f" ✗ Command failed: {' '.join(command)}") return False + def test_flink_version_selection(self) -> bool: + """Flink selection accepts a compatibility version and rejects unknown versions.""" + result = subprocess.run( + ["build/sbt", "-no-colors", "-DflinkVersion=2.1", "show flink/moduleName", "show flink/assembly/assemblyJarName"], + cwd=self.delta_root, + capture_output=True, + text=True, + ) + output = result.stdout + result.stderr + if (result.returncode != 0 or "[info] delta-flink_2.1" not in output or + "[info] delta-flink-2.1.3-" not in output): + print(" ✗ Flink 2.1 did not select artifact version 2.1.3") + return False + + result = subprocess.run( + ["build/sbt", "-no-colors", "-DflinkVersion=9.9.9", "show flink/name"], + cwd=self.delta_root, + capture_output=True, + text=True, + ) + output = result.stdout + result.stderr + expected_error = "Invalid flinkVersion: 9.9.9." + if result.returncode == 0 or expected_error not in output: + print(" ✗ Unsupported Flink version was not rejected as expected") + return False + + print(" ✓ Flink compatibility-version selection is valid") + return True + def validate_jars(self, expected: Set[str], test_name: str) -> bool: """Validates that found JARs match expected JARs exactly.""" found = self.find_all_jars() @@ -233,6 +278,7 @@ def test_default_publish(self) -> bool: # Default behavior: all Spark-dependent modules have suffix (e.g., delta-spark_4.0_2.13) expected = substitute_xversion(spark_spec.all_jars, self.delta_version) + expected.add(FLINK_VERSIONS[DEFAULT_FLINK].jar_template.format(version=self.delta_version)) return self.validate_jars(expected, "Default publishM2 (with suffix)") def test_backward_compat_publish(self) -> bool: @@ -256,23 +302,28 @@ def test_backward_compat_publish(self) -> bool: # Expect artifacts WITHOUT suffix (e.g., delta-spark_2.13 instead of delta-spark_4.0_2.13) expected = substitute_xversion(spark_spec_no_suffix.all_jars, self.delta_version) + expected.add(FLINK_VERSIONS[DEFAULT_FLINK].jar_template.format(version=self.delta_version)) return self.validate_jars(expected, "skipSparkSuffix=true (backward compat)") - def test_cross_spark_workflow(self) -> bool: - """Full cross-Spark workflow: backward-compat (no suffix) + all versions (with suffix).""" + def test_publish_workflow(self) -> bool: + """Full publish workflow: backward compatibility plus Spark and Flink cross-builds.""" print("\n" + "="*70) - print("TEST: Cross-Spark Workflow (backward-compat + all non-master with suffix)") + print("TEST: Published Artifact Workflow") print("="*70) self.clean_maven_cache() - # Step 1: Publish all modules WITHOUT suffix (backward compatibility) + # Step 1: Publish non-Flink modules WITHOUT a Spark suffix. if not self.run_sbt_command( - "Step 1: build/sbt -DskipSparkSuffix=true publishM2 (backward compat, no suffix)", - ["build/sbt", "-DskipSparkSuffix=true", "publishM2"] + "Step 1: Publish base modules", + ["build/sbt", "-DskipSparkSuffix=true", "-DskipFlinkPublish=true", "publishM2"] ): return False + if any(jar.startswith("delta-flink_") for jar in self.find_all_jars()): + print(" ✗ Base release unexpectedly published a Flink artifact") + return False + # Step 2: Publish Spark-dependent modules WITH suffix for each non-master version for spark_version, spark_spec in SPARK_VERSIONS.items(): # Skip master/snapshot versions @@ -285,10 +336,18 @@ def test_cross_spark_workflow(self) -> bool: ): return False + # Step 3: Publish every supported Flink version through the release implementation. + if not self.run_sbt_command( + "Step 3: build/sbt \"publishFlinkVersions publishM2\"", + ["build/sbt", "publishFlinkVersions publishM2"] + ): + return False + # Build expected JARs: - # 1. All modules WITHOUT suffix (from Step 1 - backward compat) + # 1. Non-Flink modules WITHOUT a Spark suffix (from Step 1) # 2. Spark-dependent modules WITH suffix for each non-master version (from Step 2) # 3. Iceberg/Hudi JARs for supported versions (no Spark suffix) + # 4. Flink JARs for every supported compatibility version (from Step 3) expected = set() # Step 1: All modules without suffix (uses default Spark version's iceberg support) @@ -305,7 +364,9 @@ def test_cross_spark_workflow(self) -> bool: expected.update(substitute_xversion(spark_spec.iceberg_jars, self.delta_version)) expected.update(substitute_xversion(spark_spec.hudi_jars, self.delta_version)) - return self.validate_jars(expected, "Cross-Spark Workflow") + expected.update(substitute_xversion([spec.jar_template for spec in FLINK_VERSIONS.values()], self.delta_version)) + + return self.validate_jars(expected, "Published Artifact Workflow") def validate_spark_versions(self) -> None: """ @@ -850,7 +911,7 @@ def main(): sys.exit(1) print("="*70) - print("Cross-Spark Build Test Suite") + print("Published Artifact Build Test Suite") print("="*70) print() @@ -869,17 +930,18 @@ def main(): script_test6c_passed = script_test.test_source_build_artifact_version_format() script_test7_passed = script_test.test_get_field() - # Test cross-Spark build workflow + # Test published artifact workflows print("\n" + "="*70) - print("PART 2: Cross-Spark Build Tests") + print("PART 2: Published Artifact Tests") print("="*70) - build_test = CrossSparkPublishTest(delta_root) + build_test = PublishedArtifactsTest(delta_root) build_test.validate_spark_versions() # Run all build tests + build_test0_passed = build_test.test_flink_version_selection() build_test1_passed = build_test.test_default_publish() build_test2_passed = build_test.test_backward_compat_publish() - build_test3_passed = build_test.test_cross_spark_workflow() + build_test3_passed = build_test.test_publish_workflow() # Summary print("\n" + "="*70) @@ -895,17 +957,18 @@ def main(): print(f" Source-Build Cache Key (non-block): {'✓ PASSED' if script_test6b_passed else '✗ FAILED'}") print(f" Source-Build Artifact Version: {'✓ PASSED' if script_test6c_passed else '✗ FAILED'}") print(f" Get Field Functionality: {'✓ PASSED' if script_test7_passed else '✗ FAILED'}") - print("\nPart 2: Cross-Spark Build Tests") + print("\nPart 2: Published Artifact Tests") + print(f" Flink version selection: {'✓ PASSED' if build_test0_passed else '✗ FAILED'}") print(f" Default publishM2 (with suffix): {'✓ PASSED' if build_test1_passed else '✗ FAILED'}") print(f" skipSparkSuffix (backward compat): {'✓ PASSED' if build_test2_passed else '✗ FAILED'}") - print(f" Cross-Spark Workflow (both): {'✓ PASSED' if build_test3_passed else '✗ FAILED'}") + print(f" Full publish workflow: {'✓ PASSED' if build_test3_passed else '✗ FAILED'}") print("="*70) all_tests_passed = ( script_test1_passed and script_test2_passed and script_test3_passed and script_test4_passed and script_test5_passed and script_test6_passed and script_test6b_passed and script_test6c_passed and script_test7_passed and - build_test1_passed and build_test2_passed and build_test3_passed + build_test0_passed and build_test1_passed and build_test2_passed and build_test3_passed ) if all_tests_passed: diff --git a/protocol_rfcs/README.md b/protocol_rfcs/README.md index 4f84fc02f3d..633924c769b 100644 --- a/protocol_rfcs/README.md +++ b/protocol_rfcs/README.md @@ -23,9 +23,12 @@ Here is the history of all the RFCs propose/accepted/rejected since Feb 6, 2024, | 2025-03-13 | [checkpoint-protection.md](https://github.com/delta-io/delta/blob/master/protocol_rfcs/checkpoint-protection.md) | https://github.com/delta-io/delta/issues/4152 | Checkpoint Protection | | 2025-03-18 | [iceberg-writer-compat-v1.md](https://github.com/delta-io/delta/blob/master/protocol_rfcs/iceberg-writer-compat-v1.md) | https://github.com/delta-io/delta/issues/4284 | IcebergWriterCompatV1 | | 2025-05-19 | [iceberg-compat-v3.md](https://github.com/delta-io/delta/blob/master/protocol_rfcs/iceberg-compat-v3.md) | https://github.com/delta-io/delta/issues/4574 | IcebergCompatV3 | -| 2025-11-20 | [materialize-partition-columns.md](https://github.com/delta-io/delta/blob/master/protocol_rfcs/materialize-partition-columns.md) | https://github.com/delta-io/delta/issues/5555 | Materialize Partition Columns | +| 2026-02-19 | [nanosecond-timestamps.md](https://github.com/delta-io/delta/blob/master/protocol_rfcs/nanosecond-timestamps.md) | https://github.com/delta-io/delta/issues/6081 | Nanosecond Timestamp Primitive Types | | 2026-04-22 | [iceberg-v4-metadata.md](https://github.com/delta-io/delta/blob/master/protocol_rfcs/iceberg-v4-metadata.md) | https://github.com/delta-io/delta/issues/6640 | Iceberg V4 Adaptive Metadata Tree | | 2026-06-23 | [interval-types.md](https://github.com/delta-io/delta/blob/master/protocol_rfcs/interval-types.md) | https://github.com/delta-io/delta/issues/7077 | Interval Types | +| 2026-07-06 | [file-type.md](https://github.com/delta-io/delta/blob/master/protocol_rfcs/file-type.md) | https://github.com/delta-io/delta/issues/7147 | File Data Type | +| 2026-08-27 | [user-defined-types.md](https://github.com/delta-io/delta/blob/master/protocol_rfcs/user-defined-types.md) | https://github.com/delta-io/delta/issues/7559 | User-Defined Types (UDT) | +| 2026-09-01 | [concurrent-identity-columns.md](https://github.com/delta-io/delta/blob/master/protocol_rfcs/concurrent-identity-columns.md) | https://github.com/delta-io/delta/issues/7572 | Concurrent Identity Columns | ### Accepted RFCs @@ -37,6 +40,7 @@ Here is the history of all the RFCs propose/accepted/rejected since Feb 6, 2024, | 2023-02-09 | 2025-01-28 |[type-widening.md](https://github.com/delta-io/delta/blob/master/protocol_rfcs/accepted/type-widening.md) | https://github.com/delta-io/delta/issues/2623 | Type Widening | | 2023-04-24 | 2025-02-14 |[variant-type.md](https://github.com/delta-io/delta/blob/master/protocol_rfcs/accepted/variant-type.md) | https://github.com/delta-io/delta/issues/2864 | Variant Data Type | | 2025-05-06 | 2026-05-01 |[variant-shredding.md](https://github.com/delta-io/delta/blob/master/protocol_rfcs/accepted/variant-shredding.md) | https://github.com/delta-io/delta/issues/4032 | Variant Shredding | +| 2025-11-20 | 2026-09-11 |[materialize-partition-columns.md](https://github.com/delta-io/delta/blob/master/protocol_rfcs/accepted/materialize-partition-columns.md) | https://github.com/delta-io/delta/issues/5555 | Materialize Partition Columns | ### Rejected RFCs diff --git a/protocol_rfcs/materialize-partition-columns.md b/protocol_rfcs/accepted/materialize-partition-columns.md similarity index 96% rename from protocol_rfcs/materialize-partition-columns.md rename to protocol_rfcs/accepted/materialize-partition-columns.md index a2a08453486..96268fbdc23 100644 --- a/protocol_rfcs/materialize-partition-columns.md +++ b/protocol_rfcs/accepted/materialize-partition-columns.md @@ -40,6 +40,8 @@ and Iceberg compatibility. - When the writer feature `materializePartitionColumns` is not set in the table protocol, writers are not required to write partition columns to data files. Note that other features might still require materialization of partition values, such as [IcebergCompatV1](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#iceberg-compatibility-v1) +When [Column Mapping](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#column-mapping) is enabled, materialized partition columns are written to the Parquet data file using their assigned physical column names and field IDs, the same as data columns. + This feature does not impose any requirements on readers. All Delta readers must be able to read the table regardless of whether partition columns are materialized in the data files. If partition values are present in both parquet and AddFile metadata, Delta readers should continue to read partition values from AddFile metadata. Also, [file-level statistics](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#per-file-statistics) should not be written for the partition column as it would repeat information already present in an AddFile's `partitionValues`. Note that this table feature, as well as [icebergCompatV1](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#iceberg-compatibility-v1) (and related table features that require partition column materialization), if enabled, take priority over the `delta.writePartitionColumnsToParquet` table property. In other words, if a table feature is enabled that requires materialization of partition columns, and table metadata contains a `false` value for `delta.writePartitionColumnsToParquet`, partition columns must be materialized. diff --git a/protocol_rfcs/concurrent-identity-columns.md b/protocol_rfcs/concurrent-identity-columns.md new file mode 100644 index 00000000000..947352068f9 --- /dev/null +++ b/protocol_rfcs/concurrent-identity-columns.md @@ -0,0 +1,228 @@ +# Concurrent Identity Columns + +**Associated Github issue for discussions: https://github.com/delta-io/delta/issues/7572** + +## Overview + +Delta already supports [Identity Columns](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#identity-columns). +A writer generates unique `start + k * step` values for a column and records the highest value it +emitted in the column's `delta.identity.highWaterMark` schema metadata, bumping that key in the same +commit that writes the rows. That serializes identity generation through the commit. A writer cannot +know which values are safe to assign until it has read the current mark, and two writers that read the +same mark generate overlapping values. An identity-column table therefore admits only one concurrent +writer of new identity values, making it unsuitable for parallel workload. + +This RFC proposes a new **writer-only** table feature, `concurrentIdentityColumns`, that removes the +high-water mark from the write path. An identity column is instead bound to a **monotonic sequence** +owned by the table's catalog. Writers *reserve* disjoint ranges of values from the sequence and assign +them locally. Many writers (and many tasks within a writer) can then generate identity values in +parallel without coordinating through the Delta commit. Because ranges are disjoint by construction, +uniqueness no longer depends on winning the commit or on reading a shared mark. + +The sequence lives in, and is allocated by, the same catalog that coordinates the table's commits, so +the feature is restricted to [Catalog-Managed Tables](https://github.com/delta-io/delta/blob/master/protocol_rfcs/accepted/catalog-managed.md). +This RFC does **not** standardize the sequence-allocation RPC (that is a catalog concern); it +standardizes only what is persisted in the Delta log, so that any writer that understands the feature, +talking to a compatible catalog, produces a table any reader can read with no changes. + +## Motivation + +Each writer, and each task within a write job, holds its own reserved range, so identity generation +scales with write parallelism instead of funneling through one high-water-mark update. The highest +generated value is tracked by the sequence service, so it is no longer rewritten into schema metadata +on every commit. The trade-off is a dependency on an external, catalog-hosted allocator and the extra +communication required to reserve ranges. + +-------- + + + +> ***Add the following key to the column-metadata list in [Identity Columns](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#identity-columns), after `delta.identity.allowExplicitInsert`.*** + +When the `concurrentIdentityColumns` feature is supported, the `metadata` for an identity column MAY +additionally contain: + +- `delta.identity.concurrent.sequenceId`: The identifier of the catalog-hosted sequence (a monotonic + counter, see [Concurrent Identity Columns](#concurrent-identity-columns)) that currently allocates + values for this column. This is a string type value. Its presence marks the column as a + **concurrent identity column**: its values are allocated from the named sequence rather than derived from + `delta.identity.highWaterMark`, and the two keys are mutually exclusive. It is an opaque pointer to + the column's *current* sequence, not a stable column identifier. A writer must not change it + during ordinary writes, it is replaced when the column is re-bound to a fresh sequence, and it is + removed when the column is converted back to a classic identity column. + +> ***New Section after the [Identity Columns](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#identity-columns) section*** + +## Concurrent Identity Columns + +A **concurrent** identity column draws its values from a monotonic sequence hosted by the table's +catalog instead of from `delta.identity.highWaterMark`. An identity column is concurrent if and +only if its schema metadata contains `delta.identity.concurrent.sequenceId`. + +To support this feature: +- Since this table feature depends on [Catalog-Managed Tables](https://github.com/delta-io/delta/blob/master/protocol_rfcs/accepted/catalog-managed.md), + the table must be on Reader Version 3 and the feature `catalogManaged` must exist in the + `protocol`'s `readerFeatures` and `writerFeatures`. +- The table must be on Writer Version 7. +- The feature `concurrentIdentityColumns` must exist in the table `protocol`'s `writerFeatures`. +- The feature `identityColumns` must exist in the table `protocol`'s `writerFeatures`. A + concurrent identity column is still an [Identity Column](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#identity-columns) + and must satisfy the identity-column requirements (`start`, `step`, `allowExplicitInsert`); only + value generation and the highest-value bookkeeping are delegated to the sequence. + +A valid table that supports `concurrentIdentityColumns` must satisfy the following invariant: +every identity column carries `delta.identity.concurrent.sequenceId`, and no identity column carries +`delta.identity.highWaterMark`. The feature's presence in `writerFeatures` is what turns on concurrent +generation for the table. `delta.identity.concurrent.sequenceId` contains the information a writer +needs to address the column's counter in the sequence service, which is why it is stored per column +and replaced whenever the column is re-bound. Consequently: +- A writer that adds the feature to a table must bind every existing identity column to a sequence in + the same operation. +- A writer creating or altering an identity column on a feature-supporting table must bind it to a + sequence. +- A writer must not fall back to high-water-mark generation for an identity column that lacks a + `sequenceId`. Such a table is invalid. + +### The sequence + +A sequence is a catalog-hosted counter identified by `delta.identity.concurrent.sequenceId` and +addressed by `(table, sequenceId)`, so a sequence belongs to exactly one table. The same `sequenceId` +stamped on another table addresses a different counter, and two tables cannot draw from one sequence. +The contract is that a persisted `sequenceId` resolves, via the table's catalog, to the sequence the +column's values are drawn from. Each identity column is bound to its own sequence, mirroring the +per-column `delta.identity.highWaterMark` it replaces. + +The sequence service hands out values `start + k * step` for strictly increasing, non-negative integers `k` +(`start` and `step` are the column's `delta.identity.start` and `delta.identity.step`). It tracks the +largest `k` it has allocated, not a numeric extreme, so a negative `step` works unchanged. Every +reservation must return a range disjoint from every range that sequence has ever returned, and must +advance the allocation frontier atomically and durably before returning it. Reservations are not +idempotent. Retrying after a timeout or lost response is a new request that must allocate a new range. + +Any allocation or transition that would produce a value outside the signed 64-bit `BIGINT` range must +fail rather than wrap. + +**Gaps are acceptable.** The sequence guarantees only that a generated value is never reused, not that +generated values are contiguous. A range a writer reserves but does not fully use (it crashes, aborts, +or under-fills) leaves those values permanently unallocated, so a concurrent identity column's values may +contain holes. Writers can use the range, even persist it, as long as they can guarantee that each +id is used only once. + +### Writer Requirements for Concurrent Identity Columns + +A writer to a table that supports `concurrentIdentityColumns` must: + +- Obtain each concurrent identity column's values by **reserving ranges** from that column's + sequence via the catalog, and assign only values drawn from ranges it has reserved. The catalog is + responsible for serving **disjoint ranges**. Writers must not generate a concurrent identity + column's values from `delta.identity.highWaterMark` or from any local counter, and must never + write or advance that key during ordinary writes. +- Honor `delta.identity.allowExplicitInsert` exactly as for classic identity columns. Explicitly + inserted values are not drawn from the sequence and the catalog does not account for them, just as + they do not advance the high-water mark today, so when `allowExplicitInsert` is `true` a + user-supplied value may collide with a generated one. Writers neither detect nor report such a + collision, exactly as for classic identity columns. An identity column is not a uniqueness + constraint, and deduplicating explicit inserts remains the user's responsibility. +- When creating a concurrent identity column, or converting a classic identity column to a + concurrent one, bind the column to a sequence and persist its `sequenceId` in the column + metadata in the same operation. During ordinary writes a writer must not change a column's + `sequenceId`; it changes only when the binding is repaired or the feature is removed (see + [Establishing, Repairing, and Removing a Sequence Binding](#establishing-repairing-and-removing-a-sequence-binding)). + +### Establishing, Repairing, and Removing a Sequence Binding + +Three operations establish, re-establish, or tear down a column's binding to a sequence. Each is a +table-metadata transaction that must conflict with every concurrent transaction on the table. +Whichever commits first, the loser aborts and retries against the new table state, and a retried +writer must obtain new reservations rather than commit identity values generated from a binding that +is no longer current. + +- **Establishing a binding.** In the version that adds `concurrentIdentityColumns` to the table's + `writerFeatures`, a writer must, for every identity column, create a sequence seeded at the first + value past the column's current `delta.identity.highWaterMark` (`highWaterMark + step`, or `start` + when the column has never emitted a value), persist its `sequenceId`, and remove + `delta.identity.highWaterMark` from the column in the same commit. The writer trusts the mark and + does not scan the data. +- **Repairing a binding.** A writer may re-bind a concurrent identity column to a **fresh** sequence + seeded strictly past the column's current extreme in the data (max for ascending `step`, min for + descending), persisting the new `sequenceId`. This scans the data, and repairs a sequence that + drifted from it, e.g. after explicit inserts the sequence never saw. It does not convert a column + between the classic and concurrent implementations. (delta-spark exposes this as + `ALTER TABLE ... ALTER COLUMN c SYNC IDENTITY`.) +- **Removing the binding.** In the version that removes `concurrentIdentityColumns` from the table's + `writerFeatures`, a writer must convert every concurrent identity column back to a classic identity + column. Rather than to scan the data, it reserves a **single** value from the column's sequence, + writes that value into `delta.identity.highWaterMark`, and removes + `delta.identity.concurrent.sequenceId`. The reserved value is past everything the sequence has handed + out, so the next classic value (`highWaterMark + step`) clears them all. + +Because all three seed strictly past the existing extreme (or preserve it in the high-water mark), no +transition can cause a previously-written identity value to be regenerated. + +### Reader Requirements for Concurrent Identity Columns + +None beyond those already imposed by the required `catalogManaged` reader-writer feature. Identity +values are fully materialized into the data files by writers, exactly as for classic identity columns, +and `delta.identity.concurrent.sequenceId` is write-path bookkeeping. + +### Compatibility with other Delta Features + +`concurrentIdentityColumns` changes where a column's values come from, not what an identity column +is. It adds two requirements of its own, and inherits everything else from +[Identity Columns](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#identity-columns). + +Feature | Support for Concurrent Identity Columns +-|- +[Identity Columns](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#identity-columns) | **Required.** On a table that supports `concurrentIdentityColumns`, every identity column must be a concurrent identity column. +[Catalog-Managed Tables](https://github.com/delta-io/delta/blob/master/protocol_rfcs/accepted/catalog-managed.md) | **Required.** The catalog owns and allocates the sequence, so a table must not support `concurrentIdentityColumns` without `catalogManaged`. + +**Everything below follows from requiring `identityColumns`: these are the existing identity-column +rules, which `concurrentIdentityColumns` neither relaxes nor extends.** + +Feature | Support for Concurrent Identity Columns +-|- +Partition Columns | **Unsupported:** an identity column cannot be a partition column, and binding it to a sequence does not change that.
**Supported:** a concurrent identity column may be a non-partition column of a partitioned table. +Clustered Tables | **Supported:** a concurrent identity column may be a clustering column. Clustering determines data layout, not value generation, and a reserved range is independent of which file its values land in. +Generated Columns | **Unsupported:** a column must not be both an identity column and a generated column.
**Supported:** a table may contain concurrent identity columns and generated columns as separate columns, and a generated column's expression may read a concurrent identity column. +Default Column Values | **Unsupported:** a column must not be both an identity column and a column with a default value.
**Supported:** a table may contain concurrent identity columns and columns with default values as separate columns. +Type Widening | **Unsupported:** `ALTER TABLE ALTER COLUMN` is not supported on identity columns, so the type of a concurrent identity column cannot be changed. Widening other columns of the table is unaffected. The column type must be `BIGINT`. +Deletion Vectors, Row Tracking, Change Data Feed, Time Travel | **Supported, with no interaction.** Identity values are materialized into the data files by the writer, so deleting, tracking, or replaying rows reads the values that were written, with no dependency on the current state of the sequence. +Per-file Statistics | **Supported.** Statistics for a concurrent identity column are computed from the materialized values exactly as for any other `BIGINT` column. +Iceberg Compatibility | **Read compatibility.** A concurrent identity column is a plain `BIGINT` column in the data files, and `delta.identity.concurrent.sequenceId` is Delta metadata that these features do not read. + +## Valid Feature Names in Table Features + +> ***Add the following row after In-Commit Timestamps in [Valid Feature Names in Table Features](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#valid-feature-names-in-table-features).*** + +Feature | Name | Readers or Writers? +-|-|- +[Concurrent Identity Columns](#concurrent-identity-columns) | `concurrentIdentityColumns` | Writers only + +## Non-Goals + +- **The sequence-allocation wire protocol.** How a writer creates a sequence and reserves ranges from + the catalog (RPC surface, batching, buffering, retry policy) is a catalog concern. This RFC + standardizes only what is persisted in the Delta log and the resulting value guarantees. +- **Lifecycle and garbage collection of sequences.** A sequence left unreferenced by a dropped column, + table, or feature downgrade may be retired by the catalog; this RFC does not mandate a reclamation + protocol. A `sequenceId` that no longer resolves means no further values can be generated for that + column: inserts fail until the column is re-bound by a repair, while reads, deletes and updates + continue to work. +- **Cross-table or shared sequences.** A sequence is addressed relative to the table it is stamped on. + A later revision may lift this restriction. +- **Support on path-based / non-catalog-managed tables.** + +## Appendix: The Sequence Service (non-normative) + +A catalog that backs this feature is expected to expose three operations on a sequence, keyed by +`(table, sequenceId)`. This sketch is **informative**: a concrete catalog is free to shape, batch, or +name these calls differently as long as it honors the contract above. + +- **`batchCreate`**: register new sequences for a column, seeded as described in + [Establishing, Repairing, and Removing a Sequence Binding](#establishing-repairing-and-removing-a-sequence-binding). + Idempotent create-or-get, so a post-commit retry is safe. +- **`batchReserveRanges`**: for multiple existing sequences reserve a contiguous range of `count` values from + an and return the inclusive bounds. +- **`batchDelete`**: retire sequences the table no longer references. Best-effort: a failed drop + only strands a sequences the table no longer points at, never leaves a live table pointing at + dropped sequences. diff --git a/protocol_rfcs/file-type.md b/protocol_rfcs/file-type.md new file mode 100644 index 00000000000..b8778fc82c4 --- /dev/null +++ b/protocol_rfcs/file-type.md @@ -0,0 +1,186 @@ +# File Data Type +**Associated Github issue for discussions: https://github.com/delta-io/delta/issues/7147** + +This protocol change adds support for the `file` data type. +The `file` data type stores a reference to a range of bytes, stored either inline in the value or in an external file. +It is intended for use cases such as file inventories, manifests, and unstructured-data references (for example, images or audio stored in object storage), which are increasingly common with AI/ML workloads. + +The `file` data type is the Delta mapping of the Parquet [`FILE` logical type](https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#file). Delta follows that specification for the physical representation and field set, with one Delta-specific restriction defined below — a `uri` must be absolute. The sections below define how the type is represented in the Delta schema and how it interacts with Delta features. + +-------- + +> ***New Section after the [Clustered Table](#clustered-table) section*** + +# File Data Type + +This feature enables support for the `file` data type, which stores a reference to a range of bytes. +A `file` value resolves to bytes that are located in one of two ways: +- **inline** — the bytes are stored directly in the value (the `inline` field), or +- **external** — the bytes are stored in a separate file at an absolute `uri` (optionally a byte range within it, via `offset`/`size`). + +These are the only two forms the Parquet `FILE` type provides: `offset`/`size` designate a byte range **within the file referenced by `uri`**, and there is no form that addresses a byte range in the data file that physically contains the value. + +The schema serialization method is described in [Schema Serialization Format](#schema-serialization-format), and the physical encoding is described in [File data in Parquet](#file-data-in-parquet). + +To support this feature: +- The table must be on Reader Version 3 and Writer Version 7. +- The feature `fileType` must exist in the table `protocol`'s `readerFeatures` and `writerFeatures`. + +## Example JSON-Encoded Delta Table Schema with File types + +``` +{ + "type" : "struct", + "fields" : [ { + "name" : "profile_image", + "type" : "file", + "nullable" : true, + "metadata" : { } + }, { + "name" : "attachments", + "type" : { + "type" : "array", + "elementType" : { + "type" : "file" + }, + "containsNull" : false + }, + "nullable" : false, + "metadata" : { } + } ] +} +``` + +## File data in Parquet + +Delta follows the Parquet `FILE` logical type. A `file` column is stored in Parquet as a group annotated with the `FILE` logical type; its physical field set, the byte-resolution rules, the `checksum` encoding, and validation are exactly as defined in the [Parquet `FILE` specification](https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#file). Those rules are not restated here; Delta adds a single restriction, defined below: a `uri` must be absolute. + +For reference, the `FILE` group may contain the following optional fields: `uri`, `offset`, `size`, `content_type`, `checksum`, and `inline`. In a Delta table a value resolves to bytes either **inline** (the `inline` field) or from an **external** file at an absolute `uri` (optionally a byte range via `offset`/`size`); `content_type` and `checksum` are metadata describing the resolved bytes. If both `inline` and a `uri` are set, then per the Parquet specification a reader may resolve the value from either (producers are expected to write the same bytes in both); reading `inline` avoids external access. See the Parquet specification for the exact field semantics. + +## Writer Requirements for File Data Type + +When File type is supported (`writerFeatures` field of a table's `protocol` action contains `fileType`), writers: +- must write a column of type `file` to Parquet as a group annotated with the Parquet `FILE` logical type, conforming to the [Parquet `FILE` specification](https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#file) (field names and types, `checksum` encoding, and validation), subject to the Delta restrictions below. +- must write an **absolute** `uri` ([RFC 3986](https://datatracker.ietf.org/doc/html/rfc3986)) wherever a `uri` is set — whether it is an external reference or a locator carried alongside an `inline` value — because a reader may resolve the value from that `uri`. Relative URIs are not permitted in Delta tables, because a relative reference has no defined resolution base: `SHALLOW CLONE` leaves data files under the source table's directory, and `OPTIMIZE`/compaction and `DEEP CLONE` move rows into files under a different directory, so a relative `uri` would resolve differently after ordinary operations. +- must produce values that **resolve** as either **inline** (`inline` set) or **external** (`uri` set), per the Parquet `FILE` resolution rules. For resolution, `offset`/`size` apply only together with a `uri` (designating a byte range within the referenced file); the Parquet type provides no form that resolves to a byte range in the containing data file, so no such value can be written. (An *inline* value may also carry a locator, from which a reader may alternatively resolve — see the next point.) +- may write inline values (`inline` set); doing so is optional. An inline value may additionally carry `uri`/`offset`/`size` locator fields that record where the bytes came from; per the Parquet specification a reader may resolve the value from `inline` or from the locator, whichever suits it (producers are expected to write the same bytes in both). Because that locator may be used for resolution, the absolute-`uri` requirement above applies to its `uri` as well. +- must represent a value that does not resolve to any referent as a column null. +- must store additional metadata about a file (for example, a modification timestamp) adjacent to the `file` column, not inside the `FILE`-annotated group. + +## Reader Requirements for File Data Type + +When File type is supported (`readerFeatures` field of a table's `protocol` action contains `fileType`), readers: +- must recognize and tolerate a `file` data type in a Delta schema. +- must read the `file` column from its Parquet `FILE`-annotated group and resolve each value to bytes per the [Parquet `FILE` specification](https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#file), supporting both **inline** values (the `inline` field) and **external** references (including a byte range when `offset`/`size` are set). Note that although writers are not required to produce inline values, readers must support reading them. +- must, for a row whose value does not resolve to any referent (an invalid value per the Parquet resolution rules — for example neither `inline` nor `uri` set, or `offset` set without `uri`) or that sets a **relative** `uri` (which Delta does not permit — see [Writer Requirements for File Data Type](#writer-requirements-for-file-data-type)), **fail the read**. A reader must **not** attempt to resolve a relative `uri` by choosing a base (such as the table root or the data file's directory), because Delta defines no such base, nor silently return `null` in place of a malformed reference. A conforming writer never produces either case, so this only arises from a non-conforming writer or corrupt data. +- must make the column available to the engine: + - [Recommended] Expose and interpret the group as a single `file` value, resolving inline and external bytes on access. + - [Alternate] Expose the raw physical group as a struct-shaped value carrying its present FILE fields (`uri`, `offset`, `size`, `content_type`, `checksum`, `inline`), leaving byte resolution to the caller — for example if the engine does not natively support the `file` type. + +## Compatibility with other Delta Features + +Feature | Support for File Data Type +-|- +Partition Columns | A `file` column cannot be chosen as a partition column (a `file` value is a group and cannot be serialized to a partition-value string), but it can be used as a data column of a partitioned table. +Clustered Tables | A `file` column cannot itself be chosen as a clustering column (a `file` value is a group and is not a comparable data type as a whole), but it can be used as a non-clustering data column of a clustered table. Its comparable leaf fields (for example, `size` or `content_type`) may be used as clustering columns, addressed by the leaf path defined in [Statistics for File Columns](#statistics-for-file-columns) — encoded in the `clusteringColumns` list as a path-segment array (for example `[["", "size"]]`), the same logical leaf path used for its required per-column statistics. +Delta Column Statistics | **Supported:** `nullCount` on **all** the `file` column's leaf fields (`uri`, `offset`, `size`, `content_type`, `checksum`, `inline`), and `minValues` / `maxValues` on its comparable, skipping-useful leaf fields (`uri`, `offset`, `size`, `content_type`). See [Statistics for File Columns](#statistics-for-file-columns).
**Unsupported:** The `file` value as a whole is not a comparable data type; and `minValues` / `maxValues` are not collected for `checksum` or `inline`. +Generated Columns | **Supported:** A `file` column is allowed to be used as a source in a generated column expression, via its addressable leaf fields (all except `inline`) addressed by logical name (see the leaf-addressing carve-out in [Statistics for File Columns](#statistics-for-file-columns)).
**Open question:** Whether `file` may be the *result* type of a generated column expression (for example, constructing a `file` reference from other columns) is left open for discussion on the associated issue, and is not specified here. +Delta CHECK Constraints | A `file` column may be used in a CHECK constraint expression through its **addressable** leaf fields (`uri`, `offset`, `size`, `content_type`, `checksum`), addressed by logical name (for example, `f.size > 0`). The `inline` field is not addressable in expressions. Because a FILE leaf is not a struct field of the Delta schema, this is an explicit carve-out from the usual requirement that referenced columns exist in the schema — see the leaf-addressing rules in [Statistics for File Columns](#statistics-for-file-columns). +Default Column Values | A `file` column must default to `NULL`. There is no Delta-defined way to construct a non-null `file` literal as a default expression, so `NULL` is the only permitted default (as with the Variant type). +Change Data Feed | **Supported:** A table using the `file` data type is allowed to enable the Delta Change Data Feed. A `file` value flows through Change Data Feed and time travel like any other column value. See [Time Travel and Change Data Feed](#time-travel-and-change-data-feed) for the distinction between the reference and the referenced bytes. +Iceberg Compatibility | **Not supported in currently-released Iceberg versions.** Iceberg has no equivalent type today, so a `file` column cannot be represented in an IcebergCompatV1 or IcebergCompatV2 table: under [IcebergCompatV2](#writer-requirements-for-icebergcompatv2) it is blocked by the type allow-list (which excludes `file`), and [IcebergCompatV1](#writer-requirements-for-icebergcompatv1) has no type allow-list (it only blocks `Map`/`Array`/`Void`), so a `file` column is additionally not permitted there. The same holds for the in-flight IcebergCompatV3. **Support is targeted for Iceberg V4**, via the in-flight IcebergNativeV4 RFC and the Iceberg `FILE` type spec, where an equivalent capability is expected. The FILE sub-fields carry reserved field IDs matching the Iceberg spec (see [Statistics for File Columns](#statistics-for-file-columns)); the remaining conversion details are out of scope here and will follow that work. +Type Widening | **Unsupported:** No type change to or from `file` is supported. +Map Keys | **Unsupported:** A `file` value is not comparable, so `file` cannot be used as a map key type. `file` is allowed as an array element type — for example the `attachments` field in the schema example above — and as a map value type. + +## Statistics for File Columns + +A `file` value is physically a group with the fixed leaf fields defined by the Parquet `FILE` type (`uri`, `offset`, `size`, `content_type`, `checksum`, `inline`). The five metadata leaves — `uri`, `offset`, `size`, `content_type`, and `checksum` — are **addressable** in expressions (clustering columns, CHECK constraints, and generated columns); the `inline` field holds the raw referenced bytes and is **not** addressable in expressions. Per-file statistics, however, are collected for **all** the sub-fields (`inline` included), aligning with the Iceberg `FILE` type, which tracks per-sub-field statistics including a null/value count for `inline`. Although `file` is serialized with a single type-name string in the Delta schema (like `variant`, and not a [Primitive Type](#primitive-types)), for [Per-file Statistics](#per-file-statistics) it is treated as that physical group: statistics descend into the FILE leaf fields, exactly as they do for a struct column. (A `file` is a distinct type and is not interchangeable with a `struct` that has the same sub-fields.) + +**Leaf addressing.** A FILE leaf is named by extending Delta's [field path](#field-path) formalism — "the ordered sequence of field names along that path" — by one final segment naming the literal `FILE` sub-field (`uri`, `offset`, `size`, `content_type`, `checksum`, or `inline`). The FILE group's inner field names are fixed literals: they are **not** subject to [Column Mapping](#column-mapping) (the Parquet spec requires that they not be renamed) and have no assigned physical name, and they are **not** [struct fields](#struct-field) of the Delta schema. This one logical leaf path is *encoded differently at each site* where a leaf is referenced: + +- **Per-file statistics** are nested JSON objects keyed by physical names, so a leaf statistic is keyed by the file column's physical name followed by the literal FILE field name — for example `minValues..uri` (see the example below). +- **Clustering columns** are stored in the `delta.clustering` domain as a list of path-segment arrays (a `Seq[Seq[String]]`), using physical names when Column Mapping is enabled — so clustering on a FILE leaf is encoded as, for example, `[["", "size"]]`, **not** as a dotted string. +- **CHECK constraints and generated columns** are SQL expression strings over the *logical* schema, so they address a FILE leaf by logical name — for example `f.size`. + +Because a FILE leaf is not a struct field of the Delta schema, addressing one in a `CHECK` constraint or generated-column expression is an explicit carve-out from the usual requirement that a referenced column exist in the table schema: the addressable `FILE` field names (`uri`, `offset`, `size`, `content_type`, `checksum` — not `inline`) may be used in SQL expressions by logical name despite not being schema fields. + +**Field IDs.** The FILE sub-fields have reserved field IDs at fixed offsets from the `file` column's own field ID: `uri` = +1, `offset` = +2, `size` = +3, `content_type` = +4, `checksum` = +5, `inline` = +6. A `file` column therefore reserves seven consecutive field IDs (its own plus these six); a writer must advance the table's maximum column id (`delta.columnMapping.maxColumnId`) past all seven and must not assign these IDs to any other field. These offsets match the Iceberg `FILE` type so that field IDs align when the table is read as Iceberg or uses field-ID-keyed statistics. + +> Note: the per-leaf addressing above describes the current per-file statistics format — `add.stats`, a JSON object keyed by physical name. When field-ID-keyed statistics are used (the in-flight Iceberg V4 Adaptive Metadata Tree RFC), each sub-field's statistics are keyed by its reserved field ID above. + +The following statistics are collected per leaf: +- `nullCount` — on **each** leaf field, `inline` included, counting rows in which that leaf is null (a per-sub-field null/value count, aligning with the Iceberg `FILE` spec). (For example, `nullCount` on `uri` counts rows whose value is stored inline — which have no `uri` — **plus** rows in which the entire `file` value is null; and `nullCount` on `inline` counts rows with no inline bytes. It is real data-skipping information.) Whole-value nullness of the `file` column is **not** separately captured, as is also the case for a struct column: the per-leaf encoding has no group-level `nullCount` slot, so `WHERE IS NULL` cannot be data-skipped from these statistics. (This differs from the Variant type, which keeps a deliberate column-level scalar `nullCount`.) +- `minValues` / `maxValues` — on the comparable, skipping-useful leaves only: `uri` (STRING), `offset` (INT64), `size` (INT64), and `content_type` (STRING). Standard per-type rules apply (for example, STRING leaves such as `uri` are truncated to a fixed prefix length). +- `minValues` / `maxValues` are **not** collected for `checksum` (a digest, or an opaque ETAG, is effectively uniformly distributed, so its min/max cannot skip anything) or for `inline` (binary content with no skipping value, and potentially large). `nullCount` is still collected for both. + +Collecting `minValues` / `maxValues` on `uri` in particular enables data skipping on file-inventory and manifest tables that filter by URI prefix. + +**Indexed-column budget.** Each indexed FILE leaf counts individually toward `delta.dataSkippingNumIndexedCols` (leaf columns are counted structurally, as with any nested leaf, whether or not `minValues`/`maxValues` are collected for them) — so a `file` column contributes up to **six** indexed leaves: all six carry `nullCount`, and four of them (`uri`, `offset`, `size`, `content_type`) additionally carry `minValues`/`maxValues`. Writers should account for this when a `file` column appears near the front of a wide schema, so it does not silently displace statistics for other columns. + +For a `file` column `f`, the per-file statistics therefore take the following shape (leaves without `minValues`/`maxValues` omitted from those structs): + +``` +|-- stats: struct +| |-- numRecords: long +| |-- nullCount: struct +| | |-- f: struct +| | | |-- uri: long +| | | |-- offset: long +| | | |-- size: long +| | | |-- content_type: long +| | | |-- checksum: long +| | | |-- inline: long +| |-- minValues: struct +| | |-- f: struct +| | | |-- uri: string +| | | |-- offset: long +| | | |-- size: long +| | | |-- content_type: string +| |-- maxValues: struct +| | |-- f: struct +| | | |-- uri: string +| | | |-- offset: long +| | | |-- size: long +| | | |-- content_type: string +``` + +## Time Travel and Change Data Feed + +A `file` value is a reference, and it is stored in the table's data files like any other column value. Delta therefore time-travels and change-data-feeds the **reference**: querying a historical version of the table, or reading `file` columns through the Change Data Feed, returns the reference values exactly as they were written at that version. + +For **inline** values, the bytes are stored within the value itself, so they are versioned and time-travel with the table like any other column data. + +For **external** references (a `uri` is set), Delta makes **no guarantee about the referenced bytes**, because the referenced files live outside the Delta table (they are not tracked by its transaction log): + +- The bytes may be overwritten or deleted independently of the table, so dereferencing a reference read from a historical version (via time travel or Change Data Feed) may fail or may return different bytes than when the reference was written. The `checksum` field, when present, allows a reader to detect that the bytes have changed, but does not allow it to recover the original bytes. +- Availability of the externally-referenced bytes is orthogonal to which table version is queried: time travel of the reference does not imply time travel of the external bytes. + +## Non-Goals + +A `file` value is a reference, and the referenced bytes are **external to the table and not under its management.** In particular, `VACUUM` does not treat referenced files as table data (and may delete files that happen to reside under the table's directory but are not tracked in the transaction log), and `CLONE` (shallow or deep) does not copy referenced files. + +The following are additionally out of scope: + +- **Lifecycle and garbage collection of referenced bytes.** The `file` type is a reference only; the protocol does not specify how, or whether, the referenced bytes are created, retained, or reclaimed. Referenced bytes are handled out-of-band by the writer or an external system. +- **Access brokering and governance** of the referenced bytes (for example, catalog-vended credentials or signed URLs). +- **Resolution of a `uri` to a physical location.** A `uri` is an absolute, resolvable reference; how it maps to a physical location (for example, environment- or catalog-level path mapping or mounts) is owned by the engine or catalog, not defined here. + +-------- + +> ***New Sub-Section after the [Variant Type](#variant-type) sub-section within the [Schema Serialization Format](#schema-serialization-format) section*** + +### File Type + +File data uses the Delta type name `file` for Delta schema serialization. + +Field Name | Description +-|- +type | Always the string "file" + +Like `variant`, `file` is a distinct top-level Delta type — it is **not** one of the [Primitive Types](#primitive-types). It is serialized with a single type-name string (`"type": "file"`) but is physically a group (see [File Data Type](#file-data-type)). To use this type, a table must support the feature `fileType`. + +-------- + +> ***Add a row to the [Valid Feature Names in Table Features](#valid-feature-names-in-table-features) table*** + +> [File Data Type](#file-data-type) | `fileType` | Readers and writers diff --git a/protocol_rfcs/iceberg-v4-metadata.md b/protocol_rfcs/iceberg-v4-metadata.md index fd017861827..e95c993d423 100644 --- a/protocol_rfcs/iceberg-v4-metadata.md +++ b/protocol_rfcs/iceberg-v4-metadata.md @@ -45,7 +45,7 @@ This design enables: | Field Name | Data Type | Description | | - | - | - | -| backReference | Struct | Reference to the existing leaf-manifest entry that this add supersedes (e.g., stats backfill, DV update). Null when the file has no leaf-manifest entry to supersede — either it has no entry in the tree, or its entry is inline in the root manifest. Contains `manifest` (String) and `pos` (Long). See [Backreferences](#backreferences). | +| backReference | Struct | Reference to the leaf-manifest entry this add supersedes in place, without a paired `remove` (e.g., a stats backfill). Null otherwise, including a DV update, where the backreference is on the paired `remove`. Contains `manifest` (String) and `pos` (Int). See [Backreferences](#backreferences). | ### Remove File @@ -56,8 +56,8 @@ This design enables: | Field Name | Data Type | Description | | - | - | - | | deletionTimestamp | Long | Must be null. Metadata cleanup uses tree reachability instead of timestamp-based expiration. | -| extendedFileMetadata | Boolean | Must be true. `partitionValues`, `size`, and `tags` are always present on the `remove`. | -| backReference | Struct | Reference to the file's entry in a leaf manifest. Null when the file has no leaf-manifest entry — either it has no entry in the tree, or its entry is inline in the root manifest. Contains `manifest` (String) and `pos` (Long). See [Backreferences](#backreferences). | +| extendedFileMetadata | Boolean | Must be true. `partitionValues` and `size` are always present on the `remove`. | +| backReference | Struct | Reference to the file's entry in a leaf manifest. Null when the file has no leaf-manifest entry — either it has no entry in the tree, or its entry is inline in the root manifest. Contains `manifest` (String) and `pos` (Int). See [Backreferences](#backreferences). | | stats | String | Must be present. Statistics of the removed file, with `numRecords` required at minimum; column statistics are included when recorded for the file. Copied from the matching `add.stats`, or converted from the file's tree entry (`record_count`, `content_stats`). | `remove` actions are transient. During log replay a `remove` cancels the matching `add` (or, via its `backReference`, marks the corresponding tree entry deleted) and is then discarded. Removes are **not** retained as tombstones in checkpoints or in the reconstructed table state. There is no timestamp-based tombstone expiration; physical file cleanup is driven by tree reachability (see [Metadata Cleanup](#metadata-cleanup)). @@ -119,6 +119,18 @@ This design enables: Files not in the reachable set may be deleted once past the retention period. Reachability is derived from the live tree, not from `remove` tombstones, so no tombstone tracking is required (see [Remove File](#remove-file)). +### Commit Provenance Information + +> ***Change to [existing section](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#commit-provenance-information)*** + +The `commitInfo` action supports a `dataChange` field that summarizes, at the commit level, whether the commit changed the data of the table: + +| Field Name | Data Type | Description | +| - | - | - | +| dataChange | Boolean | Whether the commit changes the logical records of the table. This must be `false` when the commit only rearranges existing data or adds new statistics without changing the table's logical records; it must be `true` otherwise. **Required when the `adaptiveMetadata` table feature is enabled**; optional otherwise, in which case readers fall back to the `dataChange` flags of the individual [file actions](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#add-file-and-remove-file) when it is absent. | + +When the `adaptiveMetadata` table feature is enabled, writers must include the `dataChange` field in the `commitInfo` action of every commit, and readers must treat it as the source of truth for whether the commit changed data. + -------- > ***Add a new section at the [Table Features](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#table-features) section*** @@ -175,7 +187,7 @@ When a manifest commit occurs, the Delta log entry contains a self-contained `ch { "checkpoint": [ { "checkpointMetadata": { "version": 42 } }, - { "contentRoot": { "path": "metadata/a3d1f7e2-v42.parquet", "sizeInBytes": 1024, "version": 42 } }, + { "contentRoot": { "path": "metadata/a3d1f7e2-v42.parquet", "sizeInBytes": 1024, "version": 42, "tags": {} } }, { "protocol": { "minReaderVersion": 3, "minWriterVersion": 7, "readerFeatures": ["columnMapping", "deletionVectors", "adaptiveMetadata"], "writerFeatures": ["columnMapping", "deletionVectors", "domainMetadata", "rowTracking", "adaptiveMetadata"] } }, { "metaData": { "id": "af23c9d7-fff1-4a5a-a2c8-55c59bd782aa", "name": "my_table", "schemaString": "{...}", "partitionColumns": [], "configuration": {}, "createdTime": 1234567890000 } }, { "domainMetadata": { "domain": "delta.rowTracking", "configuration": "{\"rowIdHighWaterMark\": 1000000}", "removed": false } }, @@ -194,7 +206,7 @@ The `checkpoint` action is an array of action entries. Each entry is one of: | Action | Description | |--------|-------------| | `checkpointMetadata` | Contains `version`: the table version up to which the checkpoint is complete. May be less than or equal to the commit version (e.g., commit v100 may checkpoint v50). Checkpoint versions must be strictly monotonically increasing across all checkpoint actions in the log. | -| `contentRoot` | Reference to the root manifest: `path` (relative to the table root, or an absolute URI), `sizeInBytes`, and `version` — the table version the root reflects. `version` must be `<= checkpointMetadata.version`; the two are equal in a manifest commit, and less in a standalone checkpoint (the gap is covered by inline file actions). | +| `contentRoot` | Reference to the root manifest: `path` (relative to the table root, or an absolute URI), `sizeInBytes`, `version`, and `tags` (`Map[String, String]`). `version` is the table version the root reflects and must be `<= checkpointMetadata.version`; the two are equal in a manifest commit, and less in a standalone checkpoint (the gap is covered by inline file actions). `tags` allow writers to record additional optional metadata about the manifest tree. | | `protocol` | The Protocol action at this checkpoint version. | | `metaData` | The Metadata action at this checkpoint version. | | `domainMetadata` | A DomainMetadata action. System domains (keys prefixed with `delta.`) must appear here; user domains may appear here or in a sidecar. | @@ -211,6 +223,14 @@ When `adaptiveMetadata` is enabled, `remove` and `add` actions carry a `backRefe A backreference is therefore non-null only when the file has a live entry in a leaf manifest. It is null when the file has no manifest entry (it exists only in the Delta log) or when its entry is inline in the root manifest. +The action that invalidates the existing tree entry carries the backreference: + +- **Pure delete**: the `remove` carries it; the old entry becomes `DELETED`. +- **DV update**: expressed as a `remove` of the file with its old DV plus an `add` of the same file with the new DV. The `remove` carries the backreference (old entry becomes `REPLACED`); the new `add` carries none. +- **Re-add** (e.g., a stats backfill): there is no paired `remove`, so the `add` re-adds the file in place and carries the backreference (old entry becomes `REPLACED`). + +An `add` thus carries a backreference only when it re-adds a file with no paired `remove`. + A backreference is meaningful only relative to the tree it was computed from, identified by that tree's `contentRoot.version`. A commit's backreferences are valid only if they target the current `contentRoot.version`; if a concurrent manifest commit has advanced the tree (e.g., compaction moved entries between manifests), they are stale and must be recomputed against the new tree before the commit can proceed (see [Conflict Resolution](#conflict-resolution)). Backreferences enable efficient [Manifest Deletion Vector (MDV)](#manifest-deletion-vectors-mdvs) creation during manifest commits: writers can directly construct MDVs from backreferences without scanning leaf manifests. The engine must propagate (manifest, position) metadata through the planning pipeline so it is available at commit time. @@ -239,7 +259,7 @@ Backreferences enable efficient [Manifest Deletion Vector (MDV)](#manifest-delet ### Add with Backreference (Re-add) -When an `add` supersedes an existing manifest entry (e.g., `OPTIMIZE` backfilling stats on a file), the backreference points to the old entry: +When an `add` re-adds a file in place with no paired `remove` (e.g., `OPTIMIZE` backfilling stats on a file), the backreference points to the old entry. (A DV update instead carries the backreference on its `remove`.) ```json { @@ -260,7 +280,7 @@ When an `add` supersedes an existing manifest entry (e.g., `OPTIMIZE` backfillin | Field | Type | Description | |-------|------|-------------| | `manifest` | String | Path to the leaf manifest containing this file, relative to the table root (e.g., `metadata/leaf-m1.parquet`) | -| `pos` | Long | Row position (0-indexed) of the file entry within the manifest | +| `pos` | Int | Row position (0-indexed) of the file entry within the manifest | ## Content Entry Schema @@ -369,7 +389,7 @@ Summary information for `content_type` = DATA_MANIFEST entries. Includes file/ro | 513 | `existing_rows_count` | Long | Required | Number of rows in existing files | | 514 | `deleted_rows_count` | Long | Required | Number of rows in deleted files | | 521 | `replaced_rows_count` | Long | Required | Number of rows in replaced files | -| 516 | `min_sequence_number` | Long | Required | Minimum sequence number of files in this manifest | +| 516 | `min_sequence_number` | Long | Required | Minimum data sequence number of all live entries in the manifest | | 522 | `dv` | Binary | Optional | MDV bitmap marking deleted positions in leaf manifest. Must be non-null if and only if `dv_cardinality` is non-null. | | 523 | `dv_cardinality` | Long | Optional | Number of entries marked as deleted in the MDV. Must be non-null if and only if `dv` is non-null. | @@ -513,8 +533,8 @@ Manifest commits have the following characteristics: 1. **File actions may be logged**: A manifest commit may also write `add` and `remove` actions to the Delta log, in addition to updating the metadata tree. The `checkpointMetadata.version` may be less than the commit version (the tree covers up to `checkpointMetadata.version`; remaining changes are in the log). Checkpoint versions must be strictly monotonically increasing across all checkpoint actions in the log. -2. **Non-file actions must be logged**: A manifest commit must always - write non-file actions (`metadata`, `protocol`, `txn`, `domainMetadata`, `commitInfo`) to the Delta log. These actions are not stored in the metadata tree. +2. **Non-file actions**: Non-file actions (`metaData`, `protocol`, `txn`, `domainMetadata`, `commitInfo`) are + never stored in the metadata tree. A manifest commit's `checkpoint` action already carries the current `protocol`, `metaData`, `domainMetadata`, and `txn`s (see [Checkpoint Action](#checkpoint-action)), so a checkpoint-based reader needs nothing else to reconstruct the table state up to `checkpointMetadata.version`. When one of these values changes, the commit that changes it must write a *separate* top-level non-file action, following standard Delta commit semantics. 3. **Incorporates preceding commits**: Manifest commits must incorporate all preceding log commits (since the last checkpoint) into the new metadata tree. @@ -562,7 +582,10 @@ When `adaptiveMetadata` is supported and active, writers must: - Record a `backReference` for every file read from the tree, and use the accumulated backreferences to build MDVs and re-add entries when producing a manifest commit (see [Backreferences](#backreferences) and [Manifest Deletion Vectors](#manifest-deletion-vectors-mdvs)). - Populate manifest entries with partition values, content stats, deletion vectors, and tracking and sequence numbers (see [Content Entry Schema](#content-entry-schema) and [Row Tracking Compatibility](#row-tracking-compatibility)). - Materialize row-tracking and partition columns in data files, tagged with their Iceberg `field_id`s (see [Materialized Row Tracking Columns](#materialized-row-tracking-columns) and [Partition Values](#partition-values)). +- Write timestamp columns in data files as `int64` `TIMESTAMP(MICROS)`, not `int96`, with `isAdjustedToUTC = true` for `timestamp` and `false` for `timestampNtz`. +- Write timestamp values in manifests as `int64` `TIMESTAMP(MICROS)`, not `int96`, with `isAdjustedToUTC = true` for `timestamp` and `false` for `timestampNtz`. This covers the `partition` tuple (field 102) and the `lower_bound` / `upper_bound` of [Content Stats](#content-stats) (field 146). - Resolve conflicts with commits that land concurrently, per [Conflict Resolution](#conflict-resolution). +- Include the `dataChange` field in the `commitInfo` action of every commit (see [Commit Provenance Information](#commit-provenance-information)). ### Manifest Commit Procedure @@ -571,13 +594,15 @@ When `adaptiveMetadata` is supported and active, writers must: 2. Collect all log commits since previous checkpointMetadata.version 3. For removes and re-adds with backreferences: - Group by manifest path - - Add positions to manifest_info.dv bitmap (accumulates all - deletions/replacements) - - For removes: add position to tracking.deleted_positions - bitmap (this commit only) - - For re-adds: add position to tracking.replaced_positions - bitmap (this commit only) - - Add new DATA entries with updated info for re-added files + - Add each superseded position to the manifest_info.dv bitmap + (cumulative; readers use it to skip invalidated entries) + - Set a per-commit CDF tracking bitmap only for data changes: + tracking.deleted_positions for a pure delete (remove with no + paired add), tracking.replaced_positions for a DV update (remove + + add). A stats backfill (add with no paired remove) is not a + data change and sets neither. + - For re-adds (DV update, or stats backfill), add the new DATA + entry with the updated info 4. For adds from preceding log commits (versions <= checkpointMetadata.version): set status=EXISTING with explicit snapshot_id and sequence numbers. These files are already diff --git a/protocol_rfcs/interval-types.md b/protocol_rfcs/interval-types.md index 223e4ce5683..181d6bb8241 100644 --- a/protocol_rfcs/interval-types.md +++ b/protocol_rfcs/interval-types.md @@ -1,31 +1,30 @@ # Interval Types **Associated Github issue for discussions: https://github.com/delta-io/delta/issues/7077** -This protocol change adds support for interval types (as defined [here](https://spark.apache.org/docs/latest/sql-ref-datatypes.html)). It consists of two changes to the protocol: +This protocol change documents support for interval types (as defined [here](https://spark.apache.org/docs/latest/sql-ref-datatypes.html)). It consists of one change to the protocol: -- One new reader/writer table feature -- Two distinct new primitive types (year-month and day-second) +- Two distinct primitive types (year-month and day-second) + +Like [`void`](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#void-type), interval types are documented here post-facto: interval columns already exist in tables written by earlier clients before this behavior was specified. Because such columns predate any table feature, interval types are **not** gated by a table feature and apply to all tables. + +Adopting this behavior in Delta-Spark is expected to be straightforward, since the Parquet encoding, statistics handling, and partition-value serialization specified here already match what Spark produces for these types, leaving essentially only the schema-level type check to relax. -------- > ***Add a new section in front of the [Primitive Types](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#primitive-types) section.*** -# Interval Types Table Feature +# Interval Types -This table feature (`intervalTypes`) adds two distinct Delta logical types, one for each [ANSI SQL interval type](https://spark.apache.org/docs/latest/sql-ref-datatypes.html) family: +This section defines two distinct Delta logical types, one for each [ANSI SQL interval type](https://spark.apache.org/docs/latest/sql-ref-datatypes.html) family: 1. **interval year to month**: A signed number of months, e.g. `INTERVAL '1-6' YEAR TO MONTH` represents 18 (1 year + 6 months = 18 months). 2. **interval day to second**: A signed number of microseconds, e.g. `INTERVAL '1 12:24:36.000022' DAY TO SECOND` represents 131,076,000,022 (1 day + 12 hours + 24 minutes + 36 seconds + 22 microseconds = 86,400,000,000 + 43,200,000,000 + 1,440,000,000 + 36,000,000 + 22 = 131,076,000,022). These are separate logical types with different units and physical encodings, rather than parameters of a single interval type. Each supported interval type-name spelling described below resolves to exactly one of these two logical types. -To support this feature: -- The table must be on Reader Version 3 and Writer Version 7. -- The feature `intervalTypes` must be listed in the table `protocol`'s `readerFeatures` and `writerFeatures`. - ## Type Definitions -Interval type names in `Metadata.schemaString` are case-sensitive. The canonical type-name strings are: +Interval type names in `metaData.schemaString` are case-sensitive. The canonical type-name strings are: - `interval year to month` - `interval day to second` @@ -34,7 +33,7 @@ Intervals have two families: year-month (made up of the fields `year` and `month ANSI SQL also permits narrowed spellings that denote the same two families: for year-month, `interval year` and `interval month`; for day-second, `interval day`, `interval hour`, `interval minute`, `interval second`, and any ` to ` range over the ordered fields `day`, `hour`, `minute`, `second` (e.g. `interval day to minute`, `interval hour to second`). A spelling is not valid if it is multi-family — that is, it combines fields from both families (for example, `interval month to second`). A spelling is also not valid if its fields are in the wrong order, going from a shorter unit to a longer one (for example, `interval second to day` or `interval month to year`). -Both the canonical names and all valid narrowed spellings listed above are permitted in `Metadata.schemaString`. Readers must accept every permitted spelling, while writers must always serialize the corresponding canonical name. +Both the canonical names and all valid narrowed spellings listed above are permitted in `metaData.schemaString`. Readers must accept every permitted spelling, while writers must always serialize the corresponding canonical name. Regardless of which spelling is used, the stored value is the same: every year-month spelling stores a signed count of months, and every day-second spelling stores a signed count of microseconds. The spelling affects only how a value is displayed, not how it is stored. @@ -58,17 +57,16 @@ Interval types are permitted anywhere a primitive type is permitted: as a top-le ### Reader Requirements -When this table feature is supported, readers must: +To support interval types, readers must: - Interpret `interval year to month` as a signed count of months, and `interval day to second` as a signed count of microseconds. - Accept the narrowed spellings above and normalize each to its family: any year-month spelling is treated as `interval year to month`, and any day-second spelling is treated as `interval day to second`. ### Writer Requirements -When this table feature is supported, writers must: +To support interval types, writers must: -- Serialize an interval field's type in `Metadata.schemaString` using the canonical `interval year to month` or `interval day to second` form. -- Never write an interval type to a table unless the `intervalTypes` feature is present in the table `protocol`'s `readerFeatures` and `writerFeatures`. When a writer introduces an interval type into the schema of a table that does not yet support the feature — whether at table creation or via schema evolution — it must add `intervalTypes` to both lists in the same commit, so that the feature and the interval type are committed together. +- Serialize an interval field's type in `metaData.schemaString` using the canonical `interval year to month` or `interval day to second` form. ## Partition Value Serialization @@ -96,16 +94,17 @@ Interval values are stored using a raw Parquet physical type with no logical-typ - `interval year to month` is stored as a Parquet `int32` holding the signed count of months. - `interval day to second` is stored as a Parquet `int64` holding the signed count of microseconds. -Because no Parquet logical type is written, an interval column is physically indistinguishable from a Parquet `int32`/`int64` (i.e. a Delta `integer`/`long`); the interval semantics are carried solely by the Delta schema in `Metadata.schemaString`. This representation supports signed intervals and microsecond precision. +Because no Parquet logical type is written, an interval column is physically indistinguishable from a Parquet `int32`/`int64` (i.e. a Delta `integer`/`long`); the interval semantics are carried solely by the Delta schema in `metaData.schemaString`. This representation supports signed intervals and microsecond precision. -## Feature Interactions +Because interval types are not gated by a table feature, clients that do not recognize them may fail or otherwise behave unexpectedly when reading a table with intervals in its `metaData.schemaString`. -Beyond the partition-value and statistics behavior described above, and the restrictions listed in [Error Conditions](#error-conditions), interval types have no special interactions with other table features. +## Schema Evolution and Type Changes + +Interval columns do not participate in type changes. Because interval values are materialized as raw `int32`/`int64` counts with no logical-type annotation (see [Parquet Format](#parquet-format)), and because neither interval family has a wider type it could be promoted to, there is no meaningful widening for an interval. A writer must not change an interval column to any other type, and must not change any other type to an interval type. Interval type changes are therefore never permitted, and the [Type Widening](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#type-widening) table feature does not apply to interval columns. ## Error Conditions - **Unrecognized type-name strings.** Type-name matching is case-sensitive. A reader that encounters an interval type-name string that is not one of the recognized canonical or narrowed spellings, including a multi-family spelling such as `interval month to second`, or a case variant such as `INTERVAL Year To Month`, must reject the schema with an error rather than silently coercing it to a supported type. -- **Feature not present.** A writer must add `intervalTypes` to the table `protocol`'s `readerFeatures` and `writerFeatures` whenever it introduces an interval type into the table's schema — whether at table creation or via schema evolution — if the feature is not already present, committing the feature and the interval type together (see [Writer Requirements](#writer-requirements)). Re-serializing an already-committed schema, such as when writing a checkpoint, does not introduce an interval type and must not add the `intervalTypes` feature. - **Value overflow on write.** An `interval year to month` value is stored as a signed `int32` count of months, so it must lie in the inclusive range `INTERVAL '-178956970-8' YEAR TO MONTH` to `INTERVAL '178956970-7' YEAR TO MONTH` (roughly ±179 million years). An `interval day to second` value is stored as a signed `int64` count of microseconds, so it must lie in the inclusive range `INTERVAL '-106751991 04:00:54.775808' DAY TO SECOND` to `INTERVAL '106751991 04:00:54.775807' DAY TO SECOND` (roughly ±106,751,991 days, or about 292 thousand years). A writer must reject any value that falls outside these bounds. - **Malformed or out-of-range partition values.** When reading, a partition value that is not a valid ANSI interval literal, or whose decoded value does not fit the column's underlying `int32`/`int64` range, must be rejected with an error. - **IcebergCompat incompatibility.** Apache Iceberg has no interval type. When any of the `icebergCompatV1`, `icebergCompatV2`, or `icebergCompatV3` features is enabled, a writer must reject any schema containing an interval type. @@ -124,6 +123,18 @@ Beyond the partition-value and statistics behavior described above, and the rest | interval year to month | `int32` | | | interval day to second | `int64` | | +> ***Add a note after the [Delta Data Type to Parquet Type Mappings](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#delta-data-type-to-parquet-type-mappings) table.*** + +Note that interval columns are stored using a raw Parquet `int32`/`int64` physical type with no logical-type annotation. See section [Interval Types](#interval-types). + +> ***Update the [IcebergCompatV1/V2/V3](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#icebergcompatv1) writer requirements to block interval types, alongside the existing restriction on `Map`/`Array`/`Void` types. Replace the line:*** +> +> - Block adding `Map`/`Array`/`Void` types to the table schema (and, thus, block writing them, too) +> +> ***with:*** +> +> - Block adding `Map`/`Array`/`Void`/`Interval` types to the table schema (and, thus, block writing them, too) + # References [1] https://spark.apache.org/docs/latest/sql-ref-literals.html#interval-literal diff --git a/protocol_rfcs/nanosecond-timestamps.md b/protocol_rfcs/nanosecond-timestamps.md new file mode 100644 index 00000000000..2b7e41481d0 --- /dev/null +++ b/protocol_rfcs/nanosecond-timestamps.md @@ -0,0 +1,54 @@ +# timestampNanos / Nanosecond timestamp primitive types +**Associated Github issue for discussions: https://github.com/delta-io/delta/issues/6081** + +Nanosecond resolution timestamps are widely supported in related formats (Arrow, Parquet) and tools (Pandas, Iceberg, etc). However, currently the protocol only supports microsecond resolution timestamps. This proposes adding two additional primitive type for nanosecond timestamps: UTC, and without timezones, corresponding to the two already existing microsecond timestamp types. + + +-------- + +## Add a new feature: Nanosecond timestamps (TimestampNanos and TimestampNanosNtz) + +Add a new feature: + +> This feature introduces two new data types to support timestamps with nanosecond resolution. One UTC, one without a timezone corresponding to `TimestampNtz`. For example: `1970-01-01T00:00:00.123456789Z` and `1970-01-01 00:00:00.123456789`, respectively. +> The serialization method is described in Sections [Partition Value Serialization](#partition-value-serialization) and [Schema Serialization Format](#schema-serialization-format). +> +> To support this feature: +> - To have a column of `TimestampNanos` or `TimestampNanosNtz` type in a table, the table must have Reader Version 3 and Writer Version 7. A feature name `timestampNanos` must exist in the table's `readerFeatures` and `writerFeatures`. +> - The `timestampNtz` feature must also be enabled in `readerFeatures` and `writerFeatures`. + +## Add nanosecond timestamps to the Schema Serialization Format + +Add entries for nanosecond timestamps to the Primitive Types table: + +> nanosecond timestamp: Nanosecond precision timestamp elapsed since the Unix epoch, 1970-01-01 00:00:00 UTC. When this is stored in a parquet file, its `isAdjustedToUTC` must be set to `true`. To use this type, a table must support feature `timestampNanos` and `timestampNtz`. + +> nanosecond timestamp without timezone: Nanosecond precision timestamp elapsed since the Unix epoch, 1970-01-01 00:00:00 UTC, in local timezone. It doesn't have the timezone information, and a value of this type can map to multiple physical time instants. It should always be displayed in the same way, regardless of the local time zone in effect. When this is stored in a parquet file, its `isAdjustedToUTC` must be set to `false`. To use this type, a table must support feature `timestampNanos` and `timestampNtz`. + +## Add nanosecond timestamps to the Partition Value Serialization + +Add entries for nanosecond timestamps: + +> nanosecond timestamp: Timestamps should encoded as an ISO8601 formatted timestamp adjusted to UTC timestamp such as `1970-01-01T00:00:00.123456789Z`. They may also be encoded as `{year}-{month}-{day} {hour}:{minute}:{second}.{nanosecond}`, for example: `1970-01-01 00:00:00.123456789`. + +> nanosecond timestamp without timezone: Encoded as `{year}-{month}-{day} {hour}:{minute}:{second}.{nanosecond}`. For example: `1970-01-01 00:00:00.123456789`. + +## Add Parquet Type mapping + +Add table entries for nanosecond timestamps: + +Nanosecond timestamp: + +> Physical type: `int64` +> +> Logical type: `TIMESTAMP(isAdjustedToUTC = true, units = nanoseconds)` + +Nanosecond timestamp without timezone: + +> Physical type: `int64` +> +> Logical type: `TIMESTAMP(isAdjustedToUTC = false, units = nanoseconds)` + +## Add entry to the Valid Feature Names in Table Features + +In the appendix for Valid Feature Names, add `timestampNanos` as a valid feature for readers and writers. diff --git a/protocol_rfcs/user-defined-types.md b/protocol_rfcs/user-defined-types.md new file mode 100644 index 00000000000..d6fc16b8e31 --- /dev/null +++ b/protocol_rfcs/user-defined-types.md @@ -0,0 +1,101 @@ +# User-Defined Types (UDT) +**Associated Github issue for discussions: https://github.com/delta-io/delta/issues/7559** + +This protocol change documents how Spark currently defines `UserDefinedType` (`udt`) columns and proposes adding them to the Delta protocol so they are available to all connectors. Spark writes `udt` columns into `metaData.schemaString`, but the protocol's schema type system defines only primitive / struct / array / map (plus `variant`), so a `udt` field is non-conformant under the current spec even though such columns already exist in tables written by earlier clients, and a reader that rejects the unknown type fails to read the entire table. + +Like [`void`](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#primitive-types) and interval types, `udt` is documented here post-facto and is **not** gated by a table feature. A `udt` introduces no new physical representation: it is an engine-specific *annotation* over an existing physical type (its `sqlType`). A reader that does not run the engine's deserialization code and simply reads the `sqlType` reads correct data. Unlike `timestampNtz` / `variant`, which introduced new physical semantics that readers must opt into, there is nothing here for a reader to opt into, and a feature gate could only fragment behavior for columns that already exist unguarded. This mirrors the `void` precedent. + +Notes for reviewers (not part of the proposed spec text): + +- The annotation members (`class` / `pyClass` / `serializedClass`) are Spark-specific and shown only as an example; the protocol fixes only `type` and `sqlType` and treats the rest as an opaque, engine-defined set. +- Generated columns and column defaults are forbidden for UDT columns (see Constraints). This is a new restriction: Delta Spark does not reject them today (it accepts a UDT generated column that copies another UDT column), so enforcing it needs a writer-side check. +- Nested UDTs are forbidden because they do not round-trip: an engine that reconstructs a UDT from its declared type ignores the serialized inner `sqlType`, silently dropping the nested annotation. +- On adoption, the corresponding `PROTOCOL.md` sections (statistics, type widening, column mapping, IcebergCompat) should be updated to reference UDT, rather than only this standalone section. + +-------- + +> ***Add a new "User-Defined Types" section after the [Map Type](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#map-type) section within [Schema Serialization Format](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#schema-serialization-format).*** + +# User-Defined Types + +A user-defined type (UDT) is an engine-specific logical annotation over an ordinary physical type. It introduces no new physical representation: a UDT's *physical* type is exactly its `sqlType`. The annotation identifies engine-specific code that converts between the stored `sqlType` value and a richer in-memory object (for example, a Spark ML vector); that code is outside the scope of this protocol, and the protocol specifies neither how to load nor to run it. A `udt` is not gated by a table feature; a client that does not recognize `udt` should read the column as its `sqlType` rather than reject the schema. + +A UDT field's `type` in `metaData.schemaString` is a JSON object. At its simplest, a `udt` over a `long` physical type, shown as it appears within a table's `schemaString`: + +```json +{ + "type": "struct", + "fields": [ + { "name": "id", "type": { "type": "udt", "class": "com.example.IdUDT", "sqlType": "long" }, "nullable": true, "metadata": {} } + ] +} +``` + +A fuller field-level example is Spark ML's `VectorUDT`, whose `sqlType` is a struct: + +```json +{ + "name": "features", + "type": { + "type": "udt", + "class": "org.apache.spark.ml.linalg.VectorUDT", + "pyClass": "pyspark.ml.linalg.VectorUDT", + "sqlType": { + "type": "struct", + "fields": [ + { "name": "type", "type": "byte", "nullable": false, "metadata": {} }, + { "name": "size", "type": "integer", "nullable": true, "metadata": {} }, + { "name": "indices", "type": { "type": "array", "elementType": "integer", "containsNull": false }, "nullable": true, "metadata": {} }, + { "name": "values", "type": { "type": "array", "elementType": "double", "containsNull": false }, "nullable": true, "metadata": {} } + ] + } + }, + "nullable": true, + "metadata": {} +} +``` + +The `udt` object MUST contain: + +- `type`: the string `"udt"`. +- `sqlType`: any valid Delta type other than `udt`. This is the physical, on-disk representation of the column. + +`sqlType` is expressed in the same serialization used everywhere else in `metaData.schemaString`, not a Parquet type. It may be a primitive or a nested struct / array / map, subject to two constraints: + +- **Feature conformance.** A `udt`'s `sqlType` (recursively) MUST consist only of types supported by the table's protocol version and enabled table features, exactly as if the column were declared with that type directly. If the `sqlType` contains a feature-gated type (for example `timestampNtz` or `variant`), that feature MUST be enabled on the table, and table-feature detection MUST descend into the `sqlType`. A `udt` does not exempt its `sqlType` from any type's requirements. +- **No nesting.** A `udt`'s `sqlType` MUST NOT contain another `udt`. + +A UDT is permitted anywhere its `sqlType` is permitted: as a top-level column, a nested struct field, an array element type, or a map key or value type. As a map key, a UDT is allowed only when its `sqlType` is itself a valid map-key type. + +Every other member forms the **annotation**: an open, engine-defined set that identifies the engine-specific type and where its conversion code lives. Delta does not define or interpret these members. Each value MUST be a JSON string or JSON null. An engine emits only the members meaningful to it, and MAY emit none: `{type, sqlType}` alone is valid, and is then indistinguishable to a reader from its `sqlType`. A reader that does not recognize the members reads the column as its `sqlType`. Whether a missing engine-specific member is an error is the engine's decision, not the protocol's. + +**Spark's members (example).** Spark records where its conversion code lives as: + +- `class`: the JVM type implementing the conversion (e.g. `org.apache.spark.ml.linalg.VectorUDT`). Present for JVM-defined UDTs and Python UDTs with a JVM peer. +- `pyClass`: the Python type (e.g. `pyspark.ml.linalg.VectorUDT`), or `null` when there is no Python pairing (the key is still present). +- `serializedClass`: a base64-encoded Python type, used when there is no JVM `class`. + +Each is only an identifier Spark uses to find its code; Delta does not interpret any of them. A Spark `udt` therefore takes one of two shapes, `{type, sqlType, class, pyClass}` or `{type, sqlType, pyClass, serializedClass}` (Python-only). These are an example of the mechanism, not the set of members the protocol requires. A different engine uses its own annotation members, or none, and does not set Spark's `class` / `pyClass`. + +The Parquet column data carries no `udt` marker: a UDT column is physically indistinguishable from a column of its `sqlType`, and the UDT semantics are carried solely by the annotation in `metaData.schemaString`. (Spark also copies its full schema, annotation included, into the Parquet file footer's key-value metadata under `org.apache.spark.sql.parquet.row.metadata`; that copy is Spark-specific and not required. A reader obtains the schema from `metaData.schemaString` and MUST NOT depend on the Parquet footer for UDT information.) + +## Reader and writer requirements + +A reader has no UDT-specific requirements: a UDT's physical type is its `sqlType`, so a client reads the column as its `sqlType` for all physical operations (Parquet read, projection, expression evaluation) whether or not it recognizes the annotation. A `udt` missing its `sqlType`, or carrying an annotation member whose value is neither a JSON string nor JSON null, is not a valid `udt`. + +A writer: + +- MUST store a UDT column's data physically as its `sqlType`. +- MUST preserve `udt` columns: it MUST NOT drop the annotation or downgrade a column to its bare `sqlType`, MUST retain `type`, `sqlType`, and every annotation member, and MUST NOT introduce an annotation member whose value is neither a JSON string nor JSON null. + +## Constraints and interactions with other features + +Except where stated below, a UDT column is subject to the same protocol rules as a column of its `sqlType`. + +- **Column mapping.** A UDT is a leaf: the enclosing field receives a `physicalName` and field id, but column mapping does not descend into the `sqlType`. The `sqlType`'s internal fields carry no `physicalName` or field id and are matched by the names defined in the `sqlType` itself, exactly as the same type would be in a table without column mapping. This holds in both `id` and `name` modes. +- **Statistics and data skipping.** A UDT column is not eligible for `minValues` / `maxValues`, and readers MUST NOT perform min/max data skipping over one. A per-column `nullCount` is recorded, with the UDT treated as a single statistics leaf (statistics do not descend into the `sqlType`). Per-file `numRecords` is unaffected. +- **Partitioning and clustering.** A UDT column MUST NOT be a partition column or a clustering column: partition values have no serialization for a UDT, and clustering requires the `minValues` / `maxValues` a UDT does not have. +- **Identity columns.** A UDT column MUST NOT be an identity column. +- **Generated columns and column defaults.** A UDT column MUST NOT be a generated column, and MUST NOT carry a column default. +- **Type widening.** A UDT does not participate in type changes: a writer MUST NOT widen a UDT to another type, nor another type to a UDT, nor widen any type inside a UDT's `sqlType`. +- **IcebergCompat.** Iceberg has no UDT concept; when any of `icebergCompatV1`, `icebergCompatV2`, or `icebergCompatV3` is enabled, a writer MUST reject a schema containing a UDT column. diff --git a/python/delta/version.py b/python/delta/version.py index 6aa21ab4a75..1d3d5a836c4 100644 --- a/python/delta/version.py +++ b/python/delta/version.py @@ -18,4 +18,4 @@ # Do not edit manually - edit version.sbt instead and run: # build/sbt sparkV1/generatePythonVersion -__version__ = "4.4.0" +__version__ = "4.5.0" diff --git a/run-integration-tests.py b/run-integration-tests.py index 8aed3e14e26..b4813bb236f 100755 --- a/run-integration-tests.py +++ b/run-integration-tests.py @@ -129,19 +129,20 @@ def get_spark_variants(spark_specs): Builds the list of artifact variants to test from the Spark version specs. Each variant is a dict with: - - suffix: Maven artifact suffix, e.g. "" (unsuffixed), "_4.0", "_4.1" - - spark_version: full Spark version, e.g. "4.1.0", "4.0.1" + - suffix: Maven artifact suffix, e.g. "" (unsuffixed), "_4.0", "_4.1", "_4.2" + - spark_version: full Spark version, e.g. "4.0.1", "4.1.0", "4.2.0" - support_iceberg: "true" or "false" - support_hudi: "true" or "false" The first variant is always unsuffixed (backward compat) using the DEFAULT spec's metadata. Remaining variants are suffixed, one per non-master Spark version. - Example return value (given Spark 4.0 and 4.1 specs, with 4.1 as default): + Example return value (given Spark 4.0, 4.1, and 4.2 specs, with 4.2 as default): [ - {"suffix": "", "spark_version": "4.1.0", "support_iceberg": "false", "support_hudi": "false"}, + {"suffix": "", "spark_version": "4.2.0", "support_iceberg": "false", "support_hudi": "false"}, {"suffix": "_4.0", "spark_version": "4.0.1", "support_iceberg": "true", "support_hudi": "true"}, - {"suffix": "_4.1", "spark_version": "4.1.0", "support_iceberg": "false", "support_hudi": "false"}, + {"suffix": "_4.1", "spark_version": "4.1.0", "support_iceberg": "true", "support_hudi": "false"}, + {"suffix": "_4.2", "spark_version": "4.2.0", "support_iceberg": "false", "support_hudi": "false"}, ] """ variants = [] diff --git a/sharing/src/main/scala/io/delta/sharing/spark/DeltaFormatSharingSource.scala b/sharing/src/main/scala/io/delta/sharing/spark/DeltaFormatSharingSource.scala index e7f8795789b..eeea519ed2a 100644 --- a/sharing/src/main/scala/io/delta/sharing/spark/DeltaFormatSharingSource.scala +++ b/sharing/src/main/scala/io/delta/sharing/spark/DeltaFormatSharingSource.scala @@ -17,7 +17,7 @@ package io.delta.sharing.spark import java.lang.ref.WeakReference -import java.util.UUID +import java.util.{Locale, UUID} import java.util.concurrent.TimeUnit import org.apache.spark.sql.delta.{ @@ -134,7 +134,7 @@ case class DeltaFormatSharingSource( spark = spark, deltaLog = localDeltaLog, catalogTableOpt = None, - options = new DeltaOptions(parameters, sqlConf), + options = new DeltaOptions(parametersForDeltaSource, sqlConf), snapshotAtSourceInit = snapshotDescriptor, metadataPath = metadataPath, metadataTrackingLog = schemaTrackingLogOpt @@ -1088,6 +1088,38 @@ case class DeltaFormatSharingSource( } } + /** + * The parameters for the wrapped DeltaSource, with startingTimestamp replaced by the version + * [[getStartingVersion]] resolved it to. Otherwise the wrapped DeltaSource resolves it again on + * the local delta log, where commits with no file actions have a modificationTime of 0 and an + * empty version range fails with DELTA_TIMESTAMP_GREATER_THAN_COMMIT. Replaced rather than + * dropped, since without a starting option it would read a snapshot of the latest version. + */ + private def parametersForDeltaSource: Map[String, String] = { + val convertStartingTimestamp = sqlConf.getConf( + DeltaSQLConf.DELTA_SHARING_STREAMING_CONVERT_STARTING_TIMESTAMP_TO_VERSION) + if (!convertStartingTimestamp || options.startingTimestamp.isEmpty) { + return parameters + } + getStartingVersion match { + case Some(version) => + logInfo(s"Replacing startingTimestamp with the resolved startingVersion($version) for " + + "the wrapped DeltaSource," + getTableInfoForLogging) + // `parameters` keeps the user's casing, so drop the key case-insensitively. + val startingTimestampKey = + DeltaOptions.STARTING_TIMESTAMP_OPTION.toLowerCase(Locale.ROOT) + parameters.filterNot { + case (key, _) => key.toLowerCase(Locale.ROOT) == startingTimestampKey + } + (DeltaOptions.STARTING_VERSION_OPTION -> version.toString) + case None => + // Unreachable unless code changes: getStartingVersion always resolves a defined + // startingTimestamp to Some. Log and fall back to the original parameters. + logWarning("getStartingVersion returned None despite startingTimestamp being defined," + + getTableInfoForLogging) + parameters + } + } + override def stop(): Unit = { legacyDeltaSharingSourceOpt.foreach(_._2.stop()) deltaSource.stop() diff --git a/sharing/src/test/scala/io/delta/sharing/spark/DeltaFormatSharingSourceSuite.scala b/sharing/src/test/scala/io/delta/sharing/spark/DeltaFormatSharingSourceSuite.scala index 03d16475a48..83f2400632f 100644 --- a/sharing/src/test/scala/io/delta/sharing/spark/DeltaFormatSharingSourceSuite.scala +++ b/sharing/src/test/scala/io/delta/sharing/spark/DeltaFormatSharingSourceSuite.scala @@ -32,6 +32,7 @@ import io.delta.sharing.client.model.{Table => DeltaSharingTable} import org.apache.hadoop.fs.Path import org.apache.spark.SparkEnv +import org.apache.spark.sql.QueryTest import org.apache.spark.sql.Row import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.delta.sharing.DeltaSharingTestSparkUtils @@ -55,10 +56,11 @@ import org.apache.spark.sql.types.{ } class DeltaFormatSharingSourceSuite - extends StreamTest + extends QueryTest with DeltaSQLCommandTest with DeltaSharingTestSparkUtils - with DeltaSharingDataSourceDeltaTestUtils { + with DeltaSharingDataSourceDeltaTestUtils + with StreamTest { import testImplicits._ @@ -3673,4 +3675,83 @@ class DeltaFormatSharingSourceSuite assertBlocksAreCleanedUp() } } + + /** + * Runs a startingTimestamp streaming query over a version range with no file actions, and + * returns the JSON of the last offset it committed. + */ + private def runEmptyRangeStartingTimestampQuery( + testName: String, + convertToVersion: Boolean): Option[String] = { + var offsetJson: Option[String] = None + withTempDirs { (inputDir, outputDir, checkpointDir) => + val deltaTableName = s"delta_table_empty_range_$testName" + withTable(deltaTableName) { + createTableForStreaming(deltaTableName) + sql(s"INSERT INTO $deltaTableName VALUES ('a')") + sql(s"INSERT INTO $deltaTableName VALUES ('b')") + // Version 3's actions all have dataChange = false, so [3, 3] has no file actions. + sql(s"OPTIMIZE $deltaTableName") + + val sharedTableName = s"shared_empty_range_$testName" + prepareMockedClientMetadata(deltaTableName, sharedTableName) + // The mocked client returns this version for any timestamp. + prepareMockedClientGetTableVersion(deltaTableName, sharedTableName, Some(3L)) + prepareMockedClientAndFileSystemResultForStreaming( + deltaTableName, sharedTableName, 3, 3) + + val startingTimestamp = new java.sql.Timestamp( + getTimeStampForVersion(deltaTableName, 3L)).toInstant.toString + + val profileFile = prepareProfileFile(inputDir) + val tablePath = profileFile.getCanonicalPath + s"#share1.default.$sharedTableName" + + withSQLConf( + (getDeltaSharingClassesSQLConf.toSeq :+ + (DeltaSQLConf.DELTA_SHARING_STREAMING_CONVERT_STARTING_TIMESTAMP_TO_VERSION.key -> + convertToVersion.toString)): _*) { + val q = spark.readStream + .format("deltaSharing") + .option("responseFormat", "delta") + .option(SKIP_CHANGE_COMMITS_OPTION, "true") + .option("startingTimestamp", startingTimestamp) + .load(tablePath) + .writeStream + .format("delta") + .option("checkpointLocation", checkpointDir.toString) + .trigger(Trigger.Once()) + .start(outputDir.toString) + try { + q.processAllAvailable() + } finally { + q.stop() + } + + checkAnswer(spark.read.format("delta").load(outputDir.getCanonicalPath), Seq.empty[Row]) + + val offsetLog = new OffsetSeqLog( + spark, s"${checkpointDir.getCanonicalPath}/offsets") + offsetJson = offsetLog.getLatest().flatMap(_._2.offsets.head).map(_.json()) + } + } + } + offsetJson + } + + test("startingTimestamp with an empty first batch yields an empty batch and advances offset") { + val offsetJson = runEmptyRangeStartingTimestampQuery("fixed", convertToVersion = true) + + assert(offsetJson.isDefined, "the query should have committed an offset") + assert(offsetJson.get.contains("\"reservoirVersion\":4"), + s"offset should have advanced past the empty range, but got ${offsetJson.get}") + assert(offsetJson.get.contains("\"isStartingVersion\":false"), + s"offset should not be in the initial snapshot, but got ${offsetJson.get}") + } + + test("startingTimestamp with an empty first batch fails when the fix is disabled") { + val e = intercept[StreamingQueryException] { + runEmptyRangeStartingTimestampQuery("legacy", convertToVersion = false) + } + assert(e.getMessage.contains("DELTA_TIMESTAMP_GREATER_THAN_COMMIT")) + } } diff --git a/sharing/src/test/scala/io/delta/sharing/spark/DeltaSharingDataSourceCMSuite.scala b/sharing/src/test/scala/io/delta/sharing/spark/DeltaSharingDataSourceCMSuite.scala index b9c7700c1da..1b951a0886e 100644 --- a/sharing/src/test/scala/io/delta/sharing/spark/DeltaSharingDataSourceCMSuite.scala +++ b/sharing/src/test/scala/io/delta/sharing/spark/DeltaSharingDataSourceCMSuite.scala @@ -27,6 +27,7 @@ import org.apache.spark.sql.delta.{ import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest +import org.apache.spark.sql.QueryTest import org.apache.spark.sql.delta.sharing.DeltaSharingTestSparkUtils import org.apache.spark.sql.functions.col import org.apache.spark.sql.streaming.{StreamingQueryException, StreamTest, Trigger} @@ -34,10 +35,11 @@ import org.apache.spark.sql.types.{IntegerType, StringType, StructType} // Unit tests to verify that delta format sharing support column mapping (CM). class DeltaSharingDataSourceCMSuite - extends StreamTest + extends QueryTest with DeltaSQLCommandTest with DeltaSharingTestSparkUtils - with DeltaSharingDataSourceDeltaTestUtils { + with DeltaSharingDataSourceDeltaTestUtils + with StreamTest { import testImplicits._ diff --git a/sharing/src/test/scala/io/delta/sharing/spark/DeltaSharingDataSourceDeltaTestUtils.scala b/sharing/src/test/scala/io/delta/sharing/spark/DeltaSharingDataSourceDeltaTestUtils.scala index 06d631b825e..74a680d72c6 100644 --- a/sharing/src/test/scala/io/delta/sharing/spark/DeltaSharingDataSourceDeltaTestUtils.scala +++ b/sharing/src/test/scala/io/delta/sharing/spark/DeltaSharingDataSourceDeltaTestUtils.scala @@ -84,7 +84,9 @@ trait DeltaSharingDataSourceDeltaTestUtils extends SharedSparkSession { deletionVector: DeletionVectorDescriptor): (DeletionVectorDescriptor, String) = { if (deletionVector != null) { if (deletionVector.storageType == DeletionVectorDescriptor.INLINE_DV_MARKER) { - (deletionVector, Hashing.sha256().hashString(deletionVector.uniqueId, UTF_8).toString) + val responseId = + Hashing.sha256().hashString(deletionVector.legacyUniqueId, UTF_8).toString + (deletionVector, responseId) } else { val dvPath = deletionVector.absolutePath(new Path("not-used")) ( @@ -93,7 +95,7 @@ trait DeltaSharingDataSourceDeltaTestUtils extends SharedSparkSession { SparkPath.fromPathString(dvPath.getName).urlEncoded), storageType = DeletionVectorDescriptor.PATH_DV_MARKER ), - Hashing.sha256().hashString(deletionVector.uniqueId, UTF_8).toString + Hashing.sha256().hashString(deletionVector.legacyUniqueId, UTF_8).toString ) } } else { diff --git a/spark-unified/src/main/scala-shims/spark-4.0-4.1/org/apache/spark/sql/delta/catalog/ChangelogSupport.scala b/spark-unified/src/main/scala-shims/spark-4.0-4.1/org/apache/spark/sql/delta/catalog/ChangelogSupport.scala index 83e574bc409..05f17d209e5 100644 --- a/spark-unified/src/main/scala-shims/spark-4.0-4.1/org/apache/spark/sql/delta/catalog/ChangelogSupport.scala +++ b/spark-unified/src/main/scala-shims/spark-4.0-4.1/org/apache/spark/sql/delta/catalog/ChangelogSupport.scala @@ -27,7 +27,7 @@ import org.apache.spark.sql.connector.catalog.TableCatalog * when building against Spark 4.2 (see `scala-shims/spark-4.2/...ChangelogSupport.scala`). * *

In 4.0/4.1 builds, mixing this empty trait into `DeltaCatalog` is a no-op: there is no - * `loadChangelog` to override, and downstream read-time CDF classes (`DeltaChangelog`, etc.) - * live in version-specific `java-shims/spark-4.2/` dirs and are not present here either. + * `loadChangelog` to override, and the downstream read-time CDF implementation lives in + * version-specific `java-shims/spark-4.2/` dirs and is not present here either. */ trait ChangelogSupport extends TableCatalog diff --git a/spark-unified/src/main/scala-shims/spark-4.2/org/apache/spark/sql/delta/catalog/ChangelogSupport.scala b/spark-unified/src/main/scala-shims/spark-4.2/org/apache/spark/sql/delta/catalog/ChangelogSupport.scala index 0110be0f964..ac260e5a677 100644 --- a/spark-unified/src/main/scala-shims/spark-4.2/org/apache/spark/sql/delta/catalog/ChangelogSupport.scala +++ b/spark-unified/src/main/scala-shims/spark-4.2/org/apache/spark/sql/delta/catalog/ChangelogSupport.scala @@ -17,7 +17,7 @@ package org.apache.spark.sql.delta.catalog import io.delta.spark.internal.v2.catalog.DeltaV2Table -import io.delta.spark.internal.v2.read.changelog.DeltaChangelog +import io.delta.spark.internal.v2.read.changelog.DeltaV2Changelog import org.apache.spark.sql.SparkSession import org.apache.spark.sql.connector.catalog.{Changelog, ChangelogContext, Identifier, TableCatalog} @@ -38,9 +38,9 @@ import org.apache.spark.sql.util.CaseInsensitiveStringMap * `loadTable`. Read-time CDF only flows through the V2 connector, so in `AUTO`/`STRICT` mode (see * [[DeltaV2Mode.shouldRouteChangelogToV2]]) the table is re-resolved to a [[DeltaV2Table]] for the * CHANGES read; in `NONE` mode it is rejected. It then resolves the requested [[ChangelogRange]] - * against the table's snapshot manager, and wraps everything into a [[DeltaChangelog]]. + * against the table's snapshot manager, and wraps everything into a [[DeltaV2Changelog]]. * All connector-level work (loading snapshots, validating row tracking, inspecting metadata - * actions) is deferred to the read path inside [[DeltaChangelog]]. + * actions) is deferred to the read path inside [[DeltaV2Changelog]]. * *

The whole entry point is gated by [[DeltaSQLConf.DELTA_CHANGELOG_V2_ENABLED]] (default * `false`). When the flag is off the trait follows the parent `loadChangelog` contract and throws @@ -62,7 +62,7 @@ trait ChangelogSupport extends TableCatalog { s"${name()} does not support Change Data Capture (CDC)") } val routeChangelogToV2 = new DeltaV2Mode(spark.sessionState.conf).shouldRouteChangelogToV2() - val sparkTable = loadTable(ident) match { + val deltaV2Table = loadTable(ident) match { case st: DeltaV2Table => st // Read-time CDF is V2-only. Re-resolve to V2. case v1: DeltaTableV2 if routeChangelogToV2 => @@ -70,8 +70,8 @@ trait ChangelogSupport extends TableCatalog { case other => DeltaErrors.throwChangelogRequiresV2Table(ident.toString, other.getClass.getName) } - val (startVersion, endVersion) = resolveRange(sparkTable, context.range()) - new DeltaChangelog(ident.name(), sparkTable, startVersion, endVersion) + val (startVersion, endVersion) = resolveRange(deltaV2Table, context.range()) + new DeltaV2Changelog(ident.name(), deltaV2Table, startVersion, endVersion) } /** @@ -96,10 +96,10 @@ trait ChangelogSupport extends TableCatalog { * subtracts 1) and are validated. `UnboundedRange` is rejected on batch reads. */ private def resolveRange( - sparkTable: DeltaV2Table, + deltaV2Table: DeltaV2Table, range: org.apache.spark.sql.connector.catalog.ChangelogRange): (Long, Long) = { - val snapshotManager = sparkTable.getSnapshotManager - val latestVersion = snapshotManager.loadLatestSnapshot().getVersion + val snapshotManager = deltaV2Table.getSnapshotManager + val latestVersion = snapshotManager.loadLatestSnapshot().version range match { case vr: VersionRange => val rawStart = vr.startingVersion().toLong diff --git a/spark-unified/src/main/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2OptimisticTransaction.scala b/spark-unified/src/main/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2OptimisticTransaction.scala index 107d0f822f2..71fecbe2669 100644 --- a/spark-unified/src/main/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2OptimisticTransaction.scala +++ b/spark-unified/src/main/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2OptimisticTransaction.scala @@ -16,78 +16,361 @@ package org.apache.spark.sql.delta.v2.interop -import org.apache.spark.sql.delta.{ - CurrentTransactionInfo, - DeltaLog, - LogSegment, - OptimisticTransaction, - VersionChecksum -} +// scalastyle:off import.ordering.noEmptyLine +import java.nio.file.FileAlreadyExistsException +import java.util.Optional +import java.util.concurrent.TimeUnit + +import scala.collection.JavaConverters._ +import scala.collection.mutable.ArrayBuffer +import scala.collection.mutable.HashMap +import scala.jdk.OptionConverters._ + +import org.apache.spark.sql.delta.{CurrentTransactionInfo, DeltaLog, LogSegment, OptimisticTransaction, RowId, Snapshot, VersionChecksum} +import org.apache.spark.sql.delta.WinningCommitSummary +import org.apache.spark.sql.delta.actions.{Action, AddFile, Checkpoint, CommitInfo, Protocol} +import org.apache.spark.sql.delta.amt.AMTCheckpointProvider +import org.apache.spark.sql.delta.hooks.{CheckpointHook, ChecksumHook, HudiConverterHook, IcebergConverterHook, PostCommitHook} +import org.apache.spark.sql.delta.util.{DeltaFileOperations, FileNames} +import org.apache.spark.sql.delta.v2.kernel.KernelActionUtils +import io.delta.spark.internal.v2.snapshot.SnapshotManagerFactory import io.delta.storage.commit.Commit import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.fs.Path +import org.apache.hadoop.fs.{FileStatus, Path} +import io.delta.kernel.{CommitActions => KernelCommitActions, DataWriteContext => KernelDataWriteContext, Operation => KernelOperation, PartitionKeyType => KernelPartitionKeyType, Snapshot => KernelSnapshot, Table => KernelTable, Transaction => KernelTransaction} +import io.delta.kernel.data.{Row => KernelRow} +import io.delta.kernel.engine.{Engine => KernelEngine} +import io.delta.kernel.expressions.{Literal => KernelLiteral} +import io.delta.kernel.internal.{SnapshotImpl => KernelSnapshotImpl} +import io.delta.kernel.internal.DeltaLogActionUtils.{DeltaAction => KernelDeltaAction} +import io.delta.kernel.internal.commitrange.{CommitRangeImpl => KernelCommitRangeImpl} +import io.delta.kernel.internal.util.{ColumnMapping => KernelColumnMapping, PartitionUtils => KernelPartitionUtils, Utils => KernelUtils} +import io.delta.kernel.statistics.{DataFileStatistics => KernelDataFileStatistics} +import io.delta.kernel.types.{StringType => KernelStringType} +import io.delta.kernel.utils.{CloseableIterable => KernelCloseableIterable, DataFileStatus => KernelDataFileStatus} +// scalastyle:on import.ordering.noEmptyLine import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.util.{Clock, SystemClock} /** - * OptimisticTransaction backed by Delta Kernel. - * - * Extends V1 [[org.apache.spark.sql.delta.OptimisticTransaction]] directly. - * Constructed with `deltaLog = null`: the transaction must not depend on the V1 DeltaLog, - * so any missed V1 reach surfaces loudly instead of silently using V1 state. - * - * This is a skeleton: it wires up only the construction path so the transaction can be built and - * its read state (`readVersion`, `metadata`, `protocol`, `snapshot`) resolved from the Kernel - * snapshot. The commit lifecycle ([[writeCommitFile]] and the rest of the Kernel commit path) is - * intentionally left unimplemented and fails loudly; it is a follow-up. + * OptimisticTransaction backed by Delta Kernel, extending + * [[org.apache.spark.sql.delta.OptimisticTransaction]] with `deltaLog = null` (a + * guardrail): every DeltaLog reach is either overridden by a + * Kernel-backed seam or neutralized by skipping registration (e.g. the checkpoint and checksum + * post-commit hooks), so any missed dependency surfaces loudly instead of silently reading + * DeltaLog-derived state. */ private[v2] class DeltaV2OptimisticTransaction( catalogTable: Option[CatalogTable], - val deltaV2Snapshot: DeltaV2Snapshot) + // `deltaV2Snapshot` and `kernelEngine` are intentionally private, which is internal + // implementation details we do not expose in the public API. + private val deltaV2Snapshot: DeltaV2Snapshot, + private val kernelEngine: KernelEngine) extends OptimisticTransaction( null.asInstanceOf[DeltaLog], catalogTable, deltaV2Snapshot) { - // Opt in to the base null-deltaLog guardrail: this transaction legitimately has no V1 DeltaLog + private lazy val deltaV2SnapshotManager: DeltaV2SnapshotManager = + SnapshotManagerFactory.create( + dataPath.toString, + kernelEngine, + catalogTable.toJava + ) + + /** + * Opt in to the base null-deltaLog guardrail: this transaction legitimately has no V1 DeltaLog. + */ override protected def allowNullDeltaLog: Boolean = true - // Kernel-sourced path / conf. + /** Kernel-sourced path / conf. */ override def dataPath: Path = deltaV2Snapshot.dataPath override def logPath: Path = deltaV2Snapshot.logPath - // No V1 deltaLog to source a Hadoop conf from, so use the session Hadoop conf. + /** No V1 deltaLog to source a Hadoop conf from, so use the session Hadoop conf. */ // scalastyle:off deltahadoopconfiguration override def newDeltaHadoopConf(): Configuration = spark.sessionState.newHadoopConf() // scalastyle:on deltahadoopconfiguration - // A Kernel-backed transaction maintains no V1 incremental-commit CRC state currently. + /** A Kernel-backed transaction maintains no V1 incremental-commit CRC state currently. */ override protected def computeIncrementalCommitEnabled: Boolean = false override protected def computeShouldVerifyIncrementalCommit: Boolean = false - // A Kernel-backed transaction has no V1 LogSegment (its snapshot's `logSegment` is null), so - // seed the pre-commit segment as null + /** + * A Kernel-backed transaction has no V1 LogSegment (its snapshot's `logSegment` is null), so + * seed the pre-commit segment as null. + */ override protected def initialPreCommitLogSegment: LogSegment = null - // Kernel snapshots have no V1 segment or commits to backfill. - // TODO: Before supporting coordinated commits or FsToCC, add Kernel recovery for interrupted - // CC->FS downgrades to prevent gaps in the backfilled commit sequence. + /** + * Kernel snapshots have no V1 segment or commits to backfill. + * + * Before supporting coordinated commits or FsToCC, add Kernel recovery for interrupted + * CC->FS downgrades to prevent gaps in the backfilled commit sequence. + */ override protected def maybeBackfillOnConstruction(): Unit = () - // A Kernel-backed transaction has no V1 DeltaLog to source a clock from; use a system clock + /** A Kernel-backed transaction has no V1 DeltaLog to source a clock from; use a system clock. */ override def clock: Clock = new SystemClock + /** No log store implemented, emit empty. */ + override protected def commitLogStoreClassName: String = "" + override protected[delta] def commitLogStoreClassNameForTag: String = "" + + + override def registerPostCommitHook(hook: PostCommitHook): Unit = { + val isUnsupported = + hook == ChecksumHook || + hook == CheckpointHook || + hook == IcebergConverterHook || + hook == HudiConverterHook + if (!isUnsupported) { + super.registerPostCommitHook(hook) + } + } + + /** + * The per-table commit lock serializes concurrent same-table commits within one driver JVM; V1 + * borrows it from the DeltaLog `snapshotLock`, for which a Kernel-backed transaction has no + * equivalent yet. Left disabled until the connector decides where to surface it. + */ + override private[delta] def isCommitLockEnabled: Boolean = false + + + /** + * V1 checkpointing is disabled for a Kernel-backed transaction because there is no equivalent + * Kernel implementation yet. + */ + override protected def isCheckpointNeeded( + committedVersion: Long, postCommitSnapshot: Snapshot): Boolean = false + + /** + * Return None because a [[DeltaV2Snapshot]]'s checkpoint provider is unimplemented. + */ + override protected def amtCheckpointProviderOpt: Option[AMTCheckpointProvider] = None + + /** + * Commit-stats telemetry: the table id is sourced from the Kernel snapshot. + */ + override protected def commitTableId: String = snapshot.metadata.id + /** - * Commit-IO seam. Not implemented in this skeleton yet. + * Kernel validated the table's protocol when it loaded the snapshot; protocol-CHANGING commits + * are an unsupported operation and must fail loudly. + */ + override protected def validateProtocolWrite(protocol: Protocol): Unit = { + if (protocol != snapshot.protocol) { + throw new UnsupportedOperationException( + "DeltaV2 unsupported operation: cannot validate protocol changes") + } + } + + /** + * Resolves the post-commit snapshot after a successful commit-file write. + */ + override protected def resolvePostCommitSnapshot( + committedVersion: Long, + commitOpt: Option[Commit], + newChecksumOpt: Option[VersionChecksum], + catalogTableOpt: Option[CatalogTable], + amtCheckpointWrittenInCommitOpt: Option[Checkpoint], + isIdempotentRetry: Boolean): Snapshot = { + // TODO: Use Kernel's incremental snapshot load API to build the post-commit snapshot from the + // pre-commit snapshot plus this commit, instead of a full reload that replays the log. It is + // supported in Rust but not yet exposed on the Java wrapper. + val kernelPostCommitSnapshot = KernelTable + .forPath(kernelEngine, dataPath.toString) + .getLatestSnapshot(kernelEngine) + .asInstanceOf[KernelSnapshotImpl] + require( + kernelPostCommitSnapshot.getVersion >= committedVersion, + s"Kernel reload returned version ${kernelPostCommitSnapshot.getVersion}, older than the " + + s"just-committed version $committedVersion") + new DeltaV2Snapshot(kernelPostCommitSnapshot) + } + + /** + * Reads the table commits in the inclusive version range `[startVersion, endVersion]`, applying + * `mapper` to each commit. In long term, this functionality should be owned by + * [[DeltaV2SnapshotManager]]. + */ + private def readCommitActionsInRange[T]( + startVersion: Long, + endVersion: Long)(mapper: KernelCommitActions => T): Seq[T] = { + val commitRange = deltaV2SnapshotManager + .getTableChanges( + kernelEngine, startVersion, Optional.of(java.lang.Long.valueOf(endVersion))) + .asInstanceOf[KernelCommitRangeImpl] + val actionSet = java.util.EnumSet.allOf(classOf[KernelDeltaAction]) + val commitActionsIter = commitRange.getCommitActions(kernelEngine, actionSet) + try { + val results = Seq.newBuilder[T] + while (commitActionsIter.hasNext) { + val commitActions = commitActionsIter.next() + try { + results += mapper(commitActions) + } finally { + commitActions.close() + } + } + results.result() + } finally { + commitActionsIter.close() + } + } + + /** + * Gets the conflicting versions through Kernel, from the previous attempt version to the latest. + */ + override protected def getConflictingVersions(previousAttemptVersion: Long): Seq[FileStatus] = { + val latestVersion = deltaV2SnapshotManager.loadLatestSnapshot().version + if (previousAttemptVersion > latestVersion) { + return Seq.empty + } + readCommitActionsInRange(previousAttemptVersion, latestVersion) { commitActions => + new FileStatus( + /* length = */ 0L, + /* isdir = */ false, + /* block_replication = */ 0, + /* blocksize = */ 0L, + /* modification_time = */ commitActions.getTimestamp(), + FileNames.unsafeDeltaFile(logPath, commitActions.getVersion())) + } + } + + /** + * Reads the commit's actions through Kernel and wraps them as a [[WinningCommitSummary]] for + * the ConflictChecker. + */ + override protected def readWinningCommitSummary( + fileStatus: FileStatus): WinningCommitSummary = { + val (actions, readTimeMs) = readCommitActions(fileStatus) + new WinningCommitSummary(actions, fileStatus, readTimeMs) + } + + /** + * Reads the winning commit's actions through Kernel and decodes them into [[Action]]s. + */ + private def readCommitActions(fileStatus: FileStatus): (Seq[Action], Long) = { + val startTimeNs = System.nanoTime() + val version = FileNames.deltaVersion(fileStatus) + val actions = readCommitActionsInRange(version, version)(KernelActionUtils.readActions).flatten + val readTimeMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTimeNs) + (actions, readTimeMs) + } + + /** + * Commit-IO seam: write the commit through Kernel. + * + * Translates staged [[Action]]s into Kernel action rows and commits through `Transaction.commit`, + * returning a [[Commit]] at the version Kernel reports. */ override protected[interop] def writeCommitFile( attemptVersion: Long, jsonActions: Iterator[String], currentTransactionInfo: CurrentTransactionInfo) - : (Option[VersionChecksum], Commit, CurrentTransactionInfo) = - throw new UnsupportedOperationException( - "DeltaV2OptimisticTransaction commit is not implemented yet") + : (Option[VersionChecksum], Commit, CurrentTransactionInfo) = { + val actions = currentTransactionInfo.finalActionsToCommit + val addFiles = new ArrayBuffer[AddFile]() + var rowTrackingHighWaterMark: Option[Long] = None + actions.foreach { + case a: AddFile => addFiles += a + case _: CommitInfo => // Kernel generates its own; V1 operation provenance is an JNR gap. + case RowId.RowTrackingMetadataDomain(domain) => + rowTrackingHighWaterMark = Some(domain.rowIdHighWaterMark) + case other => + throw new UnsupportedOperationException( + "DeltaV2 unsupported operation: cannot commit action " + + s"${other.getClass.getSimpleName}") + } + + val kernelSnapshotForCommit = KernelTable + .forPath(kernelEngine, dataPath.toString) + .getLatestSnapshot(kernelEngine) + if (kernelSnapshotForCommit.getVersion != attemptVersion - 1) { + throw new FileAlreadyExistsException( + s"Cannot commit version $attemptVersion through Kernel: the log is already at version " + + s"${kernelSnapshotForCommit.getVersion}; deferring to V1 conflict resolution") + } + + val kernelTxn = kernelSnapshotForCommit + .buildUpdateTableTransaction("DeltaV2OptimisticTransaction", KernelOperation.WRITE) + .build(kernelEngine) + try { + rowTrackingHighWaterMark.foreach { highWaterMark => + val domainMetadata = RowId.RowTrackingMetadataDomain(highWaterMark).toDomainMetadata + kernelTxn.addDomainMetadata(domainMetadata.domain, domainMetadata.configuration) + } + val kernelTxnState = kernelTxn.getTransactionState(kernelEngine) + val kernelCommitResult = kernelTxn.commit( + kernelEngine, + kernelAppendActionsIterable( + addFiles, kernelTxnState, kernelSnapshotForCommit, dataPath.toString)) + val committedVersion = kernelCommitResult.getVersion + val deltaFile = FileNames.unsafeDeltaFile(logPath, committedVersion) + val fs = deltaFile.getFileSystem(newDeltaHadoopConf()) + val fileStatus = fs.getFileStatus(deltaFile) + (None, new Commit(committedVersion, fileStatus, fileStatus.getModificationTime), + currentTransactionInfo) + } finally { + } + } + + /** + * Translates staged [[AddFile]]s into iterable Kernel append-action rows. + */ + private def kernelAppendActionsIterable( + addFiles: Iterable[AddFile], + kernelTxnState: KernelRow, + kernelSnapshot: KernelSnapshot, + dataPathStr: String): KernelCloseableIterable[KernelRow] = { + val kernelSchema = kernelSnapshot.getSchema + val partitionColNames = kernelSnapshot.getPartitionColumnNames.asScala + val partitionColumnDataTypesByName = kernelSchema.fields().asScala + .filter(f => partitionColNames.exists(_.equalsIgnoreCase(f.getName))) + .map(f => KernelColumnMapping.getPhysicalName(f) -> f.getDataType).toMap + + def generateKernelWriteContext( + partitionValues: Map[String, String]): KernelDataWriteContext = { + val kernelLiteralPartitionValues: java.util.Map[String, KernelLiteral] = + partitionValues.map { case (name, value) => + val kernelDataType = + partitionColumnDataTypesByName.getOrElse(name, KernelStringType.STRING) + name -> KernelPartitionUtils.literalForPartitionValue(kernelDataType, value) + }.asJava + // partition keys are physical column names, specify PHYSICAL in getWriteContext. + KernelTransaction.getWriteContext( + kernelEngine, kernelTxnState, kernelLiteralPartitionValues, KernelPartitionKeyType.PHYSICAL) + } + + def kernelDataFileStatusFor(addFile: AddFile): KernelDataFileStatus = + new KernelDataFileStatus( + DeltaFileOperations.absolutePath(dataPathStr, addFile.path).toString, + addFile.size, + addFile.modificationTime, + Option(addFile.stats).map(KernelDataFileStatistics.deserializeFromJson(_, kernelSchema)) + .getOrElse(Optional.empty[KernelDataFileStatistics]())) + + new KernelCloseableIterable[KernelRow] { + override def iterator() = { + val writeContextByPartition = + new HashMap[Map[String, String], KernelDataWriteContext]() + KernelUtils.toCloseableIterator(addFiles.iterator.asJava).flatMap { + (addFile: AddFile) => + val kernelWriteContext = writeContextByPartition.getOrElseUpdate( + addFile.partitionValues, generateKernelWriteContext(addFile.partitionValues)) + KernelTransaction.generateAppendActions( + kernelEngine, + kernelTxnState, + KernelUtils.singletonCloseableIterator(kernelDataFileStatusFor(addFile)), + kernelWriteContext) + } + } + + override def close(): Unit = {} + } + } } diff --git a/spark-unified/src/main/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2SnapshotConversionsUtils.scala b/spark-unified/src/main/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2SnapshotConversionsUtils.scala deleted file mode 100644 index 6f02d37779e..00000000000 --- a/spark-unified/src/main/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2SnapshotConversionsUtils.scala +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (2026) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.delta.v2.interop - -import scala.collection.JavaConverters._ - -import org.apache.spark.sql.delta.actions.{Format, Metadata, Protocol} -import io.delta.kernel.internal.actions.{Metadata => KernelMetadata} -import io.delta.kernel.internal.actions.{Protocol => KernelProtocol} -import io.delta.kernel.internal.util.VectorUtils - -/** - * Bridges Kernel's actions ([[KernelMetadata]] / [[KernelProtocol]]) to V1 Delta actions - * ([[Metadata]] / [[Protocol]]). - */ -private[delta] object DeltaV2SnapshotConversionsUtils { - - /** - * Converts a Kernel [[KernelMetadata]] into a V1 [[Metadata]], mapping id, name, description, - * format (provider + options), schema string, partition columns, configuration, and created - * time. Absent Kernel optionals (name, description, created time) map to `null` / `None` to - * match the V1 contract. - */ - def metadataFromKernel(metadata: KernelMetadata): Metadata = { - Metadata( - id = metadata.getId, - name = metadata.getName.orElse(null), - description = metadata.getDescription.orElse(null), - format = Format( - provider = metadata.getFormat.getProvider, - options = metadata.getFormat.getOptions.asScala.toMap), - schemaString = metadata.getSchemaString, - // getPartitionColumns preserves the declared order and original case of partition - // columns; getPartitionColNames would return an unordered set of lowercased names. - partitionColumns = - VectorUtils.toJavaList[String](metadata.getPartitionColumns).asScala.toSeq, - configuration = metadata.getConfiguration.asScala.toMap, - createdTime = - if (metadata.getCreatedTime.isPresent) Some(metadata.getCreatedTime.get.longValue()) - else None) - } - - /** - * Converts a Kernel [[KernelProtocol]] into a V1 [[Protocol]], mapping the reader/writer - * versions and the reader/writer table features. - */ - def protocolFromKernel(protocol: KernelProtocol): Protocol = { - val readerFeatures = - Option(protocol.getReaderFeatures).map(_.asScala.toSet).getOrElse(Set.empty) - val writerFeatures = - Option(protocol.getWriterFeatures).map(_.asScala.toSet).getOrElse(Set.empty) - - Protocol(protocol.getMinReaderVersion, protocol.getMinWriterVersion) - .withWriterFeatures(writerFeatures) - .withReaderFeatures(readerFeatures) - } -} diff --git a/spark-unified/src/test/scala-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/CDCPartitionReaderTest.java b/spark-unified/src/test/scala-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/CDCPartitionReaderTest.java index 572ad1e0701..a91ab5b43c2 100644 --- a/spark-unified/src/test/scala-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/CDCPartitionReaderTest.java +++ b/spark-unified/src/test/scala-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/CDCPartitionReaderTest.java @@ -40,7 +40,7 @@ import scala.collection.immutable.Map$; /** - * Unit test for {@link DeltaChangelogBatch.CDCPartitionReader}, the executor-side reader that + * Unit test for {@link DeltaV2ChangelogBatch.CDCPartitionReader}, the executor-side reader that * iterates every {@link PartitionedFile} bundled into one bin-packed {@link FilePartition}, opens a * delegate reader per file, and joins that file's CDC tail ({@code _change_type}, * {@code _commit_version}, {@code _commit_timestamp}) onto each of its rows. @@ -68,7 +68,7 @@ private static InternalRow dataRow(long id) { /** * Build a {@link PartitionedFile} whose constant-metadata map holds only the CDC tail, mirroring - * what {@code DeltaChangelogBatch.buildPartition} packs for the reader to recover per file. + * what {@code DeltaV2ChangelogBatch.buildPartition} packs for the reader to recover per file. */ private static PartitionedFile cdcFile( int index, String changeType, long commitVersion, long commitTimestampMicros) { @@ -178,7 +178,7 @@ public String toString() { } } - private static List drain(DeltaChangelogBatch.CDCPartitionReader reader) + private static List drain(DeltaV2ChangelogBatch.CDCPartitionReader reader) throws IOException { List out = new ArrayList<>(); while (reader.next()) { @@ -206,8 +206,8 @@ public void multipleFilesEmitAllRowsWithPerFileTail() throws IOException { List.of(dataRow(20L))); FakeFactory factory = new FakeFactory(rowsPerFile); - DeltaChangelogBatch.CDCPartitionReader reader = - new DeltaChangelogBatch.CDCPartitionReader(factory, partition, OUTPUT_SCHEMA); + DeltaV2ChangelogBatch.CDCPartitionReader reader = + new DeltaV2ChangelogBatch.CDCPartitionReader(factory, partition, OUTPUT_SCHEMA); List emitted = drain(reader); reader.close(); @@ -242,8 +242,8 @@ public void emptyMiddleFileIsSkipped() throws IOException { List.of(List.of(dataRow(1L)), List.of(), List.of(dataRow(3L))); FakeFactory factory = new FakeFactory(rowsPerFile); - DeltaChangelogBatch.CDCPartitionReader reader = - new DeltaChangelogBatch.CDCPartitionReader(factory, partition, OUTPUT_SCHEMA); + DeltaV2ChangelogBatch.CDCPartitionReader reader = + new DeltaV2ChangelogBatch.CDCPartitionReader(factory, partition, OUTPUT_SCHEMA); List emitted = drain(reader); reader.close(); @@ -262,8 +262,8 @@ public void emptyPartitionEmitsNothing() throws IOException { FilePartition partition = new FilePartition(0, new PartitionedFile[0]); FakeFactory factory = new FakeFactory(List.of()); - DeltaChangelogBatch.CDCPartitionReader reader = - new DeltaChangelogBatch.CDCPartitionReader(factory, partition, OUTPUT_SCHEMA); + DeltaV2ChangelogBatch.CDCPartitionReader reader = + new DeltaV2ChangelogBatch.CDCPartitionReader(factory, partition, OUTPUT_SCHEMA); assertFalse(reader.next(), "no files means no rows"); reader.close(); assertEquals(0, factory.created.size()); diff --git a/spark-unified/src/test/scala-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaChangelogCatalogIntegrationTest.java b/spark-unified/src/test/scala-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaV2ChangelogCatalogIntegrationTest.java similarity index 99% rename from spark-unified/src/test/scala-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaChangelogCatalogIntegrationTest.java rename to spark-unified/src/test/scala-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaV2ChangelogCatalogIntegrationTest.java index f6493efb37d..6668babc765 100644 --- a/spark-unified/src/test/scala-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaChangelogCatalogIntegrationTest.java +++ b/spark-unified/src/test/scala-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaV2ChangelogCatalogIntegrationTest.java @@ -20,7 +20,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import io.delta.spark.internal.v2.snapshot.DeltaSnapshotManager; +import org.apache.spark.sql.delta.v2.interop.DeltaV2SnapshotManager; import io.delta.spark.internal.v2.snapshot.SnapshotManagerFactory; import java.util.Arrays; import java.util.List; @@ -33,10 +33,10 @@ /** * Integration tests for catalog-routed CDC entrypoint (TableCatalog.loadChangelog). * - *

These tests intentionally exercise SQL/DataFrame paths (not direct DeltaChangelog + *

These tests intentionally exercise SQL/DataFrame paths (not direct DeltaV2Changelog * construction) so they validate analyzer -> catalog -> changelog wiring. */ -public class DeltaChangelogCatalogIntegrationTest extends DeltaChangelogTestBase { +class DeltaV2ChangelogCatalogIntegrationTest extends DeltaV2ChangelogTestBase { // =========================================================================================== // Fixtures and helpers @@ -106,9 +106,9 @@ private interface ThrowingConsumer { private java.sql.Timestamp commitTimestamp(String tableLocation, long version) { // Reads through the kernel snapshot manager, so it works inside the STRICT block the test body // runs in. tableLocation is resolved by withHistoryTable before STRICT is set. - DeltaSnapshotManager snapshotManager = + DeltaV2SnapshotManager snapshotManager = SnapshotManagerFactory.create(tableLocation, defaultEngine, Optional.empty()); - long millis = snapshotManager.loadSnapshotAt(version).getTimestamp(defaultEngine); + long millis = snapshotManager.loadSnapshotAt(version).timestamp(); return new java.sql.Timestamp(millis); } @@ -944,7 +944,7 @@ public void testUnboundedBatchChangesIsRejectedForNow() throws Exception { /** * A CHANGES read on a table that does not have row tracking enabled at the end version must be * rejected with the {@code DELTA_CHANGELOG_REQUIRES_ROW_TRACKING} error class. The check happens - * eagerly in {@code DeltaChangelogScanBuilder.build} against the end-version snapshot. + * eagerly in {@code DeltaV2ChangelogScanBuilder.build} against the end-version snapshot. */ @Test public void testChangelogRejectsTableWithoutRowTracking() throws Exception { @@ -984,7 +984,8 @@ public void testChangelogRejectsTableWithoutRowTracking() throws Exception { /** * Range ends in an RT-disabled state. The eager end-snapshot check in {@code - * DeltaChangelogScanBuilder.build} must reject with {@code DELTA_CHANGELOG_REQUIRES_ROW_TRACKING} + * DeltaV2ChangelogScanBuilder.build} must reject with {@code + * DELTA_CHANGELOG_REQUIRES_ROW_TRACKING} * before the per-commit loop runs. */ @Test @@ -1034,7 +1035,7 @@ public void testChangelogRejectsRowTrackingDisabledAtEnd() throws Exception { /** * Range starts and ends with row tracking enabled, but a mid-range commit carries a Metadata * action that disables row tracking. The per-commit Metadata loop in {@code - * DeltaChangelogBatch.planInputPartitions} must reject with {@code + * DeltaV2ChangelogBatch.planInputPartitions} must reject with {@code * DELTA_CHANGELOG_ROW_TRACKING_DISABLED_IN_RANGE}, because the eager boundary checks see only * RT-enabled endpoints. */ diff --git a/spark-unified/src/test/scala-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaChangelogDirectBatchExecutionTest.java b/spark-unified/src/test/scala-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaV2ChangelogDirectBatchExecutionTest.java similarity index 93% rename from spark-unified/src/test/scala-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaChangelogDirectBatchExecutionTest.java rename to spark-unified/src/test/scala-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaV2ChangelogDirectBatchExecutionTest.java index 6601117b3b8..547959b7d7a 100644 --- a/spark-unified/src/test/scala-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaChangelogDirectBatchExecutionTest.java +++ b/spark-unified/src/test/scala-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaV2ChangelogDirectBatchExecutionTest.java @@ -20,7 +20,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import io.delta.spark.internal.v2.catalog.DeltaV2Table; -import io.delta.spark.internal.v2.snapshot.DeltaSnapshotManager; +import org.apache.spark.sql.delta.v2.interop.DeltaV2SnapshotManager; import io.delta.spark.internal.v2.snapshot.SnapshotManagerFactory; import java.sql.Timestamp; import java.util.ArrayList; @@ -53,7 +53,7 @@ *

This is intentionally strict with explicit expected rows to validate end-to-end behavior * through ScanBuilder -> Scan -> Batch -> PartitionReader. */ -public class DeltaChangelogDirectBatchExecutionTest extends DeltaChangelogTestBase { +class DeltaV2ChangelogDirectBatchExecutionTest extends DeltaV2ChangelogTestBase { @Disabled("DV-off changelog read over-counts (returns 5 rows instead of 3). " + "Pre-existing, independent of the path-encoding fix.") @@ -76,20 +76,20 @@ public void testDirectBatchExecutionWithExplicitExpectedRows(boolean enableDvs) spark.sql(String.format("INSERT INTO %s VALUES (1, 'Alice'), (2, 'Bob')", tableName)); spark.sql(String.format("DELETE FROM %s WHERE id = 1", tableName)); - DeltaSnapshotManager snapshotManager = + DeltaV2SnapshotManager snapshotManager = SnapshotManagerFactory.create(tablePath, defaultEngine, Optional.empty()); StructType dataSchema = spark.table(tableName).schema(); - long latestVersion = snapshotManager.loadLatestSnapshot().getVersion(); + long latestVersion = snapshotManager.loadLatestSnapshot().version(); Map commitTimestampsMicros = loadCommitTimestampsMicros(tableName); - DeltaChangelog changelog = - new DeltaChangelog( + DeltaV2Changelog changeLog = + new DeltaV2Changelog( tableName, new DeltaV2Table(Identifier.of(new String[0], tableName), tablePath), 0L, latestVersion); ScanBuilder scanBuilder = - changelog.newScanBuilder(new CaseInsensitiveStringMap(Collections.emptyMap())); + changeLog.newScanBuilder(new CaseInsensitiveStringMap(Collections.emptyMap())); Scan scan = scanBuilder.build(); StructType schema = scan.readSchema(); Batch batch = scan.toBatch(); @@ -126,10 +126,10 @@ public void testDirectBatchExecutionWithExplicitExpectedRows(boolean enableDvs) } /** - * Asserts that {@link DeltaChangelog#rowId()} and {@link DeltaChangelog#rowVersion()} expose the - * per-row tracking metadata fields (not the per-commit columns). Spark's batch CDC post-processor - * reads these references to perform carry-over removal and update detection. If {@code - * rowVersion()} ever pointed at {@code _commit_version} instead of {@code + * Asserts that {@link DeltaV2Changelog#rowId()} and {@link DeltaV2Changelog#rowVersion()} expose + * the per-row tracking metadata fields (not the per-commit columns). Spark's batch CDC + * post-processor reads these references to perform carry-over removal and update detection. If + * {@code rowVersion()} ever pointed at {@code _commit_version} instead of {@code * _metadata.row_commit_version}, real updates whose DELETE/INSERT halves share the same commit * version would be silently dropped as carry-overs. */ @@ -148,19 +148,19 @@ public void testRowTrackingMetadataReferences() throws Exception { tableName, tablePath)); spark.sql(String.format("INSERT INTO %s VALUES (1)", tableName)); - DeltaSnapshotManager snapshotManager = + DeltaV2SnapshotManager snapshotManager = SnapshotManagerFactory.create(tablePath, defaultEngine, Optional.empty()); StructType dataSchema = spark.table(tableName).schema(); - long latestVersion = snapshotManager.loadLatestSnapshot().getVersion(); + long latestVersion = snapshotManager.loadLatestSnapshot().version(); - DeltaChangelog changelog = - new DeltaChangelog( + DeltaV2Changelog changeLog = + new DeltaV2Changelog( tableName, new DeltaV2Table(Identifier.of(new String[0], tableName), tablePath), 0L, latestVersion); - NamedReference[] rowIdRefs = changelog.rowId(); + NamedReference[] rowIdRefs = changeLog.rowId(); assertEquals(1, rowIdRefs.length, "Expected a single rowId field reference"); assertArrayEquals( new String[] {"_metadata", "row_id"}, @@ -168,7 +168,7 @@ public void testRowTrackingMetadataReferences() throws Exception { "rowId() must point at _metadata.row_id"); assertArrayEquals( new String[] {"_metadata", "row_commit_version"}, - changelog.rowVersion().fieldNames(), + changeLog.rowVersion().fieldNames(), "rowVersion() must point at _metadata.row_commit_version (per-row), " + "not _commit_version (per-commit)"); }); @@ -199,20 +199,20 @@ public void testUpdateProducesPairedDeleteAndInsert(boolean enableDvs) throws Ex spark.sql(String.format("INSERT INTO %s VALUES (1, 'Alice')", tableName)); spark.sql(String.format("UPDATE %s SET name = 'AliceX' WHERE id = 1", tableName)); - DeltaSnapshotManager snapshotManager = + DeltaV2SnapshotManager snapshotManager = SnapshotManagerFactory.create(tablePath, defaultEngine, Optional.empty()); StructType dataSchema = spark.table(tableName).schema(); - long latestVersion = snapshotManager.loadLatestSnapshot().getVersion(); + long latestVersion = snapshotManager.loadLatestSnapshot().version(); Map commitTimestampsMicros = loadCommitTimestampsMicros(tableName); - DeltaChangelog changelog = - new DeltaChangelog( + DeltaV2Changelog changeLog = + new DeltaV2Changelog( tableName, new DeltaV2Table(Identifier.of(new String[0], tableName), tablePath), 0L, latestVersion); Scan scan = - changelog + changeLog .newScanBuilder(new CaseInsensitiveStringMap(Collections.emptyMap())) .build(); Batch batch = scan.toBatch(); @@ -272,20 +272,20 @@ public void testRangeSlicingNonZeroStart() throws Exception { spark.sql(String.format("INSERT INTO %s VALUES (2, 'Bob')", tableName)); spark.sql(String.format("INSERT INTO %s VALUES (3, 'Charlie')", tableName)); - DeltaSnapshotManager snapshotManager = + DeltaV2SnapshotManager snapshotManager = SnapshotManagerFactory.create(tablePath, defaultEngine, Optional.empty()); StructType dataSchema = spark.table(tableName).schema(); Map commitTimestampsMicros = loadCommitTimestampsMicros(tableName); // Range = [v2, v3]. v0 (CREATE) and v1 (Alice) must be excluded from the output. - DeltaChangelog changelog = - new DeltaChangelog( + DeltaV2Changelog changeLog = + new DeltaV2Changelog( tableName, new DeltaV2Table(Identifier.of(new String[0], tableName), tablePath), 2L, 3L); Scan scan = - changelog + changeLog .newScanBuilder(new CaseInsensitiveStringMap(Collections.emptyMap())) .build(); Batch batch = scan.toBatch(); @@ -341,13 +341,13 @@ public void testChangelogWithReservedCharsInPath() throws Exception { tableName, tablePath)); spark.sql(String.format("INSERT INTO %s VALUES (1, 'Alice'), (2, 'Bob')", tableName)); - DeltaSnapshotManager snapshotManager = + DeltaV2SnapshotManager snapshotManager = SnapshotManagerFactory.create(tablePath, defaultEngine, Optional.empty()); - long latestVersion = snapshotManager.loadLatestSnapshot().getVersion(); + long latestVersion = snapshotManager.loadLatestSnapshot().version(); Map commitTimestampsMicros = loadCommitTimestampsMicros(tableName); - DeltaChangelog changelog = - new DeltaChangelog( + DeltaV2Changelog changelog = + new DeltaV2Changelog( tableName, new DeltaV2Table(Identifier.of(new String[0], tableName), tablePath), 0L, @@ -406,13 +406,13 @@ public void testChangelogWithReservedCharsInDataFileNames() throws Exception { tableName, tablePath)); spark.sql(String.format("INSERT INTO %s VALUES (1, 'Alice'), (2, 'Bob')", tableName)); - DeltaSnapshotManager snapshotManager = + DeltaV2SnapshotManager snapshotManager = SnapshotManagerFactory.create(tablePath, defaultEngine, Optional.empty()); - long latestVersion = snapshotManager.loadLatestSnapshot().getVersion(); + long latestVersion = snapshotManager.loadLatestSnapshot().version(); Map commitTimestampsMicros = loadCommitTimestampsMicros(tableName); - DeltaChangelog changelog = - new DeltaChangelog( + DeltaV2Changelog changelog = + new DeltaV2Changelog( tableName, new DeltaV2Table(Identifier.of(new String[0], tableName), tablePath), 0L, diff --git a/spark-unified/src/test/scala-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaChangelogDvTest.java b/spark-unified/src/test/scala-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaV2ChangelogDvTest.java similarity index 97% rename from spark-unified/src/test/scala-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaChangelogDvTest.java rename to spark-unified/src/test/scala-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaV2ChangelogDvTest.java index 102cecb6ecf..2836cb44e6c 100644 --- a/spark-unified/src/test/scala-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaChangelogDvTest.java +++ b/spark-unified/src/test/scala-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaV2ChangelogDvTest.java @@ -25,14 +25,14 @@ /** * Integration tests for read-time CDF / DSv2 changelog with Deletion Vectors (DVs) that are NOT * covered by parameterized DV-on/off variants of - * {@link DeltaChangelogDirectBatchExecutionTest}. The parameterized tests already cover the + * {@link DeltaV2ChangelogDirectBatchExecutionTest}. The parameterized tests already cover the * basic single-file DELETE and UPDATE; this file holds scenarios that need DV-specific table * shape (multi-file, multi-commit). * *

Correctness relies on Catalyst post-processing (Phase 1 carry-over removal via * {@code row_commit_version} comparison, Phase 2 net-changes matrix). */ -public class DeltaChangelogDvTest extends DeltaChangelogTestBase { +class DeltaV2ChangelogDvTest extends DeltaV2ChangelogTestBase { // =========================================================================================== // Fixtures diff --git a/spark-unified/src/test/scala-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaChangelogTestBase.java b/spark-unified/src/test/scala-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaV2ChangelogTestBase.java similarity index 97% rename from spark-unified/src/test/scala-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaChangelogTestBase.java rename to spark-unified/src/test/scala-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaV2ChangelogTestBase.java index 5ddf5400422..d7e43880ded 100644 --- a/spark-unified/src/test/scala-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaChangelogTestBase.java +++ b/spark-unified/src/test/scala-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaV2ChangelogTestBase.java @@ -38,7 +38,7 @@ * continue to exercise the V1-only catalog and the gated-off catalog default that production users * hit. */ -public abstract class DeltaChangelogTestBase extends DeltaV2TestBase { +abstract class DeltaV2ChangelogTestBase extends DeltaV2TestBase { @BeforeAll public static void setUpChangelogSparkAndEngine() { diff --git a/spark-unified/src/test/scala/io/delta/internal/ApplyV2ReadOptionsSuite.scala b/spark-unified/src/test/scala/io/delta/internal/ApplyV2ReadOptionsSuite.scala index 8b0cb674af1..23c76a69f8a 100644 --- a/spark-unified/src/test/scala/io/delta/internal/ApplyV2ReadOptionsSuite.scala +++ b/spark-unified/src/test/scala/io/delta/internal/ApplyV2ReadOptionsSuite.scala @@ -21,14 +21,9 @@ import java.util.{HashMap => JHashMap} import scala.jdk.CollectionConverters._ -import io.delta.kernel.internal.SnapshotImpl -import io.delta.spark.internal.v2.catalog.DeltaV2Table -import io.delta.spark.internal.v2.snapshot.PathBasedSnapshotManager -import io.delta.storage.commit.uccommitcoordinator.UCCommitCoordinatorClient import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable, CatalogTableType} import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan -import org.apache.spark.sql.catalyst.catalog.CatalogTable -import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTableType} import org.apache.spark.sql.catalyst.streaming.StreamingRelationV2 import org.apache.spark.sql.catalyst.types.DataTypeUtils.toAttributes import org.apache.spark.sql.delta.{DeltaLog, DeltaOptions} @@ -37,10 +32,14 @@ import org.apache.spark.sql.delta.test.shims.StreamingRelationV2Shim import org.apache.spark.sql.delta.commands.cdc.CDCReader import org.apache.spark.sql.delta.sources.{DeltaSourceMetadataTrackingLog, DeltaSourceUtils, DeltaSQLConf, PersistedMetadata} import org.apache.spark.sql.delta.test.DeltaSQLCommandTest -import org.apache.spark.sql.execution.datasources.DataSource +import org.apache.spark.sql.util.CaseInsensitiveStringMap +import io.delta.spark.internal.v2.catalog.DeltaV2Table +import io.delta.spark.internal.v2.snapshot.PathBasedSnapshotManager +import io.delta.storage.commit.uccommitcoordinator.UCCommitCoordinatorClient + import org.apache.spark.sql.connector.catalog.Identifier +import org.apache.spark.sql.execution.datasources.DataSource import org.apache.spark.sql.types.{StringType, StructType} -import org.apache.spark.sql.util.CaseInsensitiveStringMap class ApplyV2ReadOptionsSuite extends DeltaSQLCommandTest { @@ -272,8 +271,7 @@ class ApplyV2ReadOptionsSuite extends DeltaSQLCommandTest { val deltaLog = DeltaLog.forTable(spark, tablePath) val snapshotManager = new PathBasedSnapshotManager(tablePath, deltaLog.newDeltaHadoopConf()) - val tableId = - snapshotManager.loadLatestSnapshot.asInstanceOf[SnapshotImpl].getMetadata.getId + val tableId = snapshotManager.loadLatestSnapshot.metadata.getId val trackingLog = DeltaSourceMetadataTrackingLog.create( spark, schemaLogPath, tableId, tablePath, parameters = Map.empty[String, String]) val customSchemaJson = diff --git a/spark-unified/src/test/scala/io/delta/spark/internal/v2/read/DeltaV2BatchScanAssertions.scala b/spark-unified/src/test/scala/io/delta/spark/internal/v2/read/DeltaV2BatchScanAssertions.scala new file mode 100644 index 00000000000..d3c740d94f2 --- /dev/null +++ b/spark-unified/src/test/scala/io/delta/spark/internal/v2/read/DeltaV2BatchScanAssertions.scala @@ -0,0 +1,90 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.spark.internal.v2.read + +import org.apache.spark.sql.DataFrame +import org.apache.spark.sql.execution.{FileSourceScanLike, SparkPlan} +import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper +import org.apache.spark.sql.execution.datasources.v2.BatchScanExec + +private[read] trait DeltaV2BatchScanAssertions { + self: AdaptiveSparkPlanHelper => + + private def deltaV2BatchScans(plan: SparkPlan): Seq[DeltaV2Scan] = collect(plan) { + case scan: BatchScanExec if scan.scan.isInstanceOf[DeltaV2Scan] => + scan.scan.asInstanceOf[DeltaV2Scan] + } + + protected final def assertDeltaV2BatchScan(df: DataFrame, context: String): Unit = { + val plan = df.queryExecution.executedPlan + assert( + deltaV2BatchScans(plan).nonEmpty, + s"expected a Delta V2 BatchScan for $context, got:\n$plan") + assert( + collect(plan) { case _: FileSourceScanLike => true }.isEmpty, + s"expected $context to stay off file-source execution, got:\n$plan") + } + + protected final def assertDeltaV2BatchScanCount( + df: DataFrame, + expected: Int, + context: String): Unit = { + val plan = df.queryExecution.executedPlan + val actual = deltaV2BatchScans(plan).size + assert( + actual == expected, + s"expected $expected Delta V2 BatchScans for $context, got $actual:\n$plan") + assert( + collect(plan) { case _: FileSourceScanLike => true }.isEmpty, + s"expected $context to stay off file-source execution, got:\n$plan") + } + + protected final def deltaV2BatchScanReadFieldNames( + df: DataFrame, + expectedScanCount: Int, + context: String): Seq[Seq[String]] = { + assertDeltaV2BatchScanCount(df, expectedScanCount, context) + deltaV2BatchScans(df.queryExecution.executedPlan) + .map(_.getReadDataSchema.fieldNames.toSeq) + } + + protected final def assertDeltaV2BatchScanPushedLimit( + df: DataFrame, + expected: Int, + context: String): Unit = { + assertDeltaV2BatchScan(df, context) + val plan = df.queryExecution.executedPlan + val scans = deltaV2BatchScans(plan) + val marker = s"PushedLimit: $expected" + assert( + scans.nonEmpty && scans.forall(_.description().contains(marker)), + s"expected every Delta V2 BatchScan for $context to have $marker, " + + s"got ${scans.map(_.description()).mkString("[", ", ", "]")}:\n$plan") + } + + protected final def assertDeltaV2BatchScanHasNoPushedLimit( + df: DataFrame, + context: String): Unit = { + assertDeltaV2BatchScan(df, context) + val plan = df.queryExecution.executedPlan + val scans = deltaV2BatchScans(plan) + assert( + scans.nonEmpty && scans.forall(scan => !scan.description().contains("PushedLimit")), + s"expected no Delta V2 BatchScan for $context to have PushedLimit, " + + s"got ${scans.map(_.description()).mkString("[", ", ", "]")}:\n$plan") + } +} diff --git a/spark-unified/src/test/scala/io/delta/spark/internal/v2/read/DeltaV2ColumnPruningE2ESuite.scala b/spark-unified/src/test/scala/io/delta/spark/internal/v2/read/DeltaV2ColumnPruningE2ESuite.scala new file mode 100644 index 00000000000..f0057c3ab58 --- /dev/null +++ b/spark-unified/src/test/scala/io/delta/spark/internal/v2/read/DeltaV2ColumnPruningE2ESuite.scala @@ -0,0 +1,165 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.spark.internal.v2.read + +import org.apache.spark.sql.Row + +private[read] trait DeltaV2ColumnPruningE2ETests { + self: DeltaV2ScanE2ETestUtils => + + // V2ScanColumnPruningIntegrationTest.testSelectSubsetPrunesDataColumns + test("selecting a subset prunes unused data columns") { + val table = "v2_scan_e2e_pruning_subset" + withTable(table) { + withV1Mode { + sql(s"CREATE TABLE $table (a INT, b INT, c INT) USING $tableProvider") + sql(s"INSERT INTO $table VALUES (1, 2, 3), (4, 5, 6)") + } + + val df = sql(s"SELECT a FROM $table") + checkAnswer(df, Seq(Row(1), Row(4))) + val requiredSchemas = + expectedScanRequiredFieldNames(df, 1, "subset column-pruning read") + assert( + requiredSchemas == Seq(Seq("a")), + s"expected the scan to read only a, got $requiredSchemas") + } + } + + // V2ScanColumnPruningIntegrationTest.testFilterPushdownLeavesOnlyProjectedColumns + test("a residual filter retains its column and prunes unused columns") { + val table = "v2_scan_e2e_pruning_filter" + withTable(table) { + withV1Mode { + sql(s"CREATE TABLE $table (a INT, b INT, c INT) USING $tableProvider") + sql(s"INSERT INTO $table VALUES (1, 2, 3), (4, 5, 6)") + } + + val df = sql(s"SELECT a FROM $table WHERE b > 3") + checkAnswer(df, Seq(Row(4))) + val requiredSchemas = + expectedScanRequiredFieldNames(df, 1, "residual-filter column-pruning read") + assert( + requiredSchemas == Seq(Seq("a", "b")), + s"expected the scan to read a and b but not c, got $requiredSchemas") + } + } + + // V2ScanColumnPruningIntegrationTest.testAllColumnsUsedMeansNoPruning + test("using every data column leaves the read schema unchanged") { + val table = "v2_scan_e2e_pruning_all_columns" + withTable(table) { + withV1Mode { + sql(s"CREATE TABLE $table (a INT, b INT) USING $tableProvider") + sql(s"INSERT INTO $table VALUES (1, 2), (3, 4)") + } + + val df = sql(s"SELECT a, b FROM $table") + checkAnswer(df, Seq(Row(1, 2), Row(3, 4))) + val requiredSchemas = + expectedScanRequiredFieldNames(df, 1, "all-columns pruning read") + assert( + requiredSchemas == Seq(Seq("a", "b")), + s"expected the scan to read a and b, got $requiredSchemas") + } + } + + // V2ScanColumnPruningIntegrationTest.testPartitionOnlyQueryHasEmptyReadDataSchema + test("a partition-only query has an empty data read schema") { + val table = "v2_scan_e2e_pruning_partition_only" + withTable(table) { + withV1Mode { + sql( + s"CREATE TABLE $table (a INT, p STRING) USING $tableProvider " + + "PARTITIONED BY (p)") + sql(s"INSERT INTO $table VALUES (1, 'x'), (2, 'y')") + } + + val df = sql(s"SELECT p FROM $table") + checkAnswer(df, Seq(Row("x"), Row("y"))) + val requiredSchemas = + expectedScanRequiredFieldNames(df, 1, "partition-only column-pruning read") + assert( + requiredSchemas == Seq(Seq.empty[String]), + s"expected an empty data requiredSchema, got $requiredSchemas") + } + } + + // V2ScanColumnPruningIntegrationTest.testSubqueryDecorrelation_prunesUnusedColumns + test("subquery decorrelation prunes unused columns") { + val orders = "v2_scan_e2e_pruning_orders" + val vips = "v2_scan_e2e_pruning_vips" + withTable(orders, vips) { + withV1Mode { + sql( + s"CREATE TABLE $orders (" + + s"order_id INT, customer_id INT, amount INT, status STRING) " + + s"USING $tableProvider") + sql( + s"INSERT INTO $orders VALUES " + + "(1, 10, 100, 'open'), (2, 20, 200, 'closed')") + sql(s"CREATE TABLE $vips (customer_id INT) USING $tableProvider") + sql(s"INSERT INTO $vips VALUES (10), (30)") + } + + val df = sql( + s"SELECT order_id, amount FROM $orders " + + s"WHERE customer_id IN (SELECT customer_id FROM $vips)") + checkAnswer(df, Seq(Row(1, 100))) + val requiredSchemas = + expectedScanRequiredFieldNames(df, 2, "decorrelated subquery column-pruning read") + val expectedSchemas = Set( + Seq("order_id", "customer_id", "amount"), + Seq("customer_id")) + assert( + requiredSchemas.toSet == expectedSchemas, + s"expected orders and VIP required schemas $expectedSchemas, got $requiredSchemas") + } + } + + // V2ScanColumnPruningIntegrationTest.testJoinPrunesUnreferencedColumns + test("a join prunes unreferenced columns from both sides") { + val left = "v2_scan_e2e_pruning_join_left" + val right = "v2_scan_e2e_pruning_join_right" + withTable(left, right) { + withV1Mode { + sql(s"CREATE TABLE $left (a INT, b INT, c INT) USING $tableProvider") + sql(s"INSERT INTO $left VALUES (1, 2, 3), (4, 5, 6)") + sql(s"CREATE TABLE $right (x INT, y INT, z INT) USING $tableProvider") + sql(s"INSERT INTO $right VALUES (2, 20, 200), (5, 50, 500)") + } + + val df = + sql(s"SELECT a, y FROM $left t1 JOIN $right t2 ON t1.b = t2.x") + checkAnswer(df, Seq(Row(1, 20), Row(4, 50))) + val requiredSchemas = + expectedScanRequiredFieldNames(df, 2, "join column-pruning read") + val expectedSchemas = Set(Seq("a", "b"), Seq("x", "y")) + assert( + requiredSchemas.toSet == expectedSchemas, + s"expected left and right required schemas $expectedSchemas, got $requiredSchemas") + } + } +} + +/** Covers required-schema and column-pruning contracts shared across Delta V2 scan paths. + */ +class DeltaV2ColumnPruningE2ESuite + extends DeltaV2ScanE2ETestUtils + with DeltaV2ColumnPruningE2ETests +{ +} diff --git a/spark-unified/src/test/scala/io/delta/spark/internal/v2/read/DeltaV2LimitPushdownE2ESuite.scala b/spark-unified/src/test/scala/io/delta/spark/internal/v2/read/DeltaV2LimitPushdownE2ESuite.scala new file mode 100644 index 00000000000..faeb619a4fe --- /dev/null +++ b/spark-unified/src/test/scala/io/delta/spark/internal/v2/read/DeltaV2LimitPushdownE2ESuite.scala @@ -0,0 +1,354 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.spark.internal.v2.read + +import org.apache.spark.sql.delta.sources.DeltaSQLConf + +import org.apache.spark.sql.Row + +private[read] trait DeltaV2LimitPushdownE2ETests { + self: DeltaV2ScanE2ETestUtils => + + // V2LimitPushdownTest.testLimitBasic. + test("a basic LIMIT returns the requested rows and reaches the scan") { + val table = "v2_scan_e2e_limit_basic" + withTable(table) { + withV1Mode { + sql(s"CREATE TABLE $table (id INT, name STRING) USING $tableProvider") + sql( + s"INSERT INTO $table VALUES " + + "(1, 'a'), (2, 'b'), (3, 'c'), (4, 'd'), (5, 'e')") + } + + val df = sql(s"SELECT * FROM $table LIMIT 3") + assert(df.count() == 3L, "LIMIT 3 should return exactly 3 rows") + assertExpectedPushedLimit(df, 3, "basic LIMIT read") + } + } + + // V2LimitPushdownTest.testLimitLargerThanTable. + test("a LIMIT larger than the table returns every row and reaches the scan") { + val table = "v2_scan_e2e_limit_larger_than_table" + withTable(table) { + withV1Mode { + sql(s"CREATE TABLE $table (id INT) USING $tableProvider") + sql(s"INSERT INTO $table VALUES (1), (2), (3)") + } + + val df = sql(s"SELECT * FROM $table LIMIT 100") + checkAnswer(df, Seq(Row(1), Row(2), Row(3))) + assertExpectedPushedLimit(df, 100, "LIMIT larger than table") + } + } + + // V2LimitPushdownTest.testLimit1. + test("LIMIT one returns one row and reaches the scan") { + val table = "v2_scan_e2e_limit_one" + withTable(table) { + withV1Mode { + sql(s"CREATE TABLE $table (id INT) USING $tableProvider") + sql(s"INSERT INTO $table VALUES (1), (2), (3)") + } + + val df = sql(s"SELECT * FROM $table LIMIT 1") + assert(df.count() == 1L, "LIMIT 1 should return exactly 1 row") + assertExpectedPushedLimit(df, 1, "LIMIT one read") + } + } + + // V2LimitPushdownTest.testLimitEmptyTable. + test("LIMIT on an empty table returns no rows and reaches the scan") { + val table = "v2_scan_e2e_limit_empty" + withTable(table) { + withV1Mode { + sql(s"CREATE TABLE $table (id INT) USING $tableProvider") + } + + val df = sql(s"SELECT * FROM $table LIMIT 10") + checkAnswer(df, Seq.empty) + assertExpectedPushedLimit(df, 10, "LIMIT on empty table") + } + } + + // V2LimitPushdownTest.testLimitWithDeletionVectors. + test("LIMIT with deletion vectors returns live rows and reaches the scan") { + val table = "v2_scan_e2e_limit_dv" + withTable(table) { + withV1Mode { + sql( + s"CREATE TABLE $table (id LONG, value STRING) USING $tableProvider " + + "TBLPROPERTIES ('delta.enableDeletionVectors' = 'true')") + spark + .range(1000) + .selectExpr("id", "cast(id as string) as value") + .write + .mode("append") + .insertInto(table) + sql(s"DELETE FROM $table WHERE id % 2 = 0") + } + + val query = s"SELECT * FROM $table LIMIT 50" + assertAnySelectedFileHasDeletionVector( + sql(query), + "LIMIT read with deletion vectors") + val df = sql(query) + val rows = df.collect().toSeq + assert(rows.length == 50, "LIMIT 50 with DVs should return exactly 50 rows") + assert( + rows.forall(_.getLong(0) % 2 == 1), + s"LIMIT 50 with DVs returned a deleted even id: ${rows.mkString(", ")}") + assertExpectedPushedLimit(df, 50, "LIMIT read with deletion vectors") + assertExpectedScan(df, "LIMIT read with deletion vectors") + } + } + + // V2LimitPushdownTest.testLimitWithHeavyDVs. + test("LIMIT with heavy deletion vectors spans enough live rows") { + val table = "v2_scan_e2e_limit_heavy_dv" + withTable(table) { + withV1Mode { + sql( + s"CREATE TABLE $table (id LONG, value STRING) USING $tableProvider " + + "TBLPROPERTIES ('delta.enableDeletionVectors' = 'true')") + spark + .range(0, 100) + .selectExpr("id", "cast(id as string) as value") + .coalesce(1) + .write + .mode("append") + .insertInto(table) + spark + .range(100, 200) + .selectExpr("id", "cast(id as string) as value") + .coalesce(1) + .write + .mode("append") + .insertInto(table) + sql( + s"DELETE FROM $table WHERE " + + "(id >= 40 AND id < 100) OR (id >= 140 AND id < 200)") + } + + val numFiles = fileCount(table) + assert(numFiles == 2L) + val query = s"SELECT * FROM $table LIMIT 50" + assertAllSelectedFilesHaveDeletionVectors( + sql(query), + "LIMIT read with heavy deletion vectors") + val df = sql(query) + val rows = df.collect().toSeq + assert(rows.length == 50, "LIMIT 50 with heavy DVs should return exactly 50 rows") + assert( + rows.forall { row => + val id = row.getLong(0) + (id >= 0 && id < 40) || (id >= 100 && id < 140) + }, + s"LIMIT 50 with heavy DVs returned a deleted id: ${rows.mkString(", ")}") + assertExpectedPushedLimit(df, 50, "LIMIT read with heavy deletion vectors") + assertExpectedScan(df, "LIMIT read with heavy deletion vectors") + assertRouteSpecificInputFileCount( + df, + expected = 2, + context = "LIMIT 50 must select both heavy-DV files") + } + } + + // V2LimitPushdownTest.testLimitWithPartitionFilter. + test("a partition filter with LIMIT returns matching rows") { + val table = "v2_scan_e2e_limit_partition_filter" + withTable(table) { + withV1Mode { + sql( + s"CREATE TABLE $table (id INT, part STRING) USING $tableProvider " + + "PARTITIONED BY (part)") + sql( + s"INSERT INTO $table VALUES " + + "(1, 'a'), (2, 'a'), (3, 'a'), (4, 'b'), (5, 'b')") + } + + val df = sql(s"SELECT * FROM $table WHERE part = 'a' LIMIT 2") + val rows = df.collect().toSeq + assert(rows.length == 2, "Partition filter + LIMIT should return 2 rows") + assert( + rows.forall(_.getString(1) == "a"), + s"Partition filter + LIMIT returned a non-matching row: ${rows.mkString(", ")}") + } + } + + // V2LimitPushdownTest.testLimitWithDataFilter. + test("a data filter with LIMIT returns matching rows without pushdown") { + val table = "v2_scan_e2e_limit_data_filter" + withTable(table) { + withV1Mode { + sql(s"CREATE TABLE $table (id INT, name STRING) USING $tableProvider") + sql( + s"INSERT INTO $table VALUES " + + "(1, 'a'), (2, 'b'), (3, 'c'), (4, 'd'), (5, 'e')") + } + + val df = sql(s"SELECT * FROM $table WHERE id > 2 LIMIT 2") + val rows = df.collect().toSeq + assert(rows.length == 2, "Data filter + LIMIT should return 2 rows") + assert( + rows.forall(_.getInt(0) > 2), + s"Data filter + LIMIT returned an id <= 2: ${rows.mkString(", ")}") + assertExpectedNoPushedLimit(df, "data filter with LIMIT") + } + } + + // V2LimitPushdownTest.testLimitWithColumnProjection. + test("a projected LIMIT returns only the requested column and reaches the scan") { + val table = "v2_scan_e2e_limit_projection" + withTable(table) { + withV1Mode { + sql( + s"CREATE TABLE $table (id INT, name STRING, value DOUBLE) USING $tableProvider") + sql( + s"INSERT INTO $table VALUES " + + "(1, 'a', 1.0), (2, 'b', 2.0), (3, 'c', 3.0)") + } + + val df = sql(s"SELECT name FROM $table LIMIT 2") + val rows = df.collect().toSeq + assert(rows.length == 2, "Column projection + LIMIT should return 2 rows") + assert(df.columns.toSeq == Seq("name"), "projected result should contain only name") + val expectedNames = Set("a", "b", "c") + assert( + rows.forall(row => !row.isNullAt(0) && expectedNames.contains(row.getString(0))), + s"Column projection + LIMIT returned an unexpected value: ${rows.mkString(", ")}") + assertExpectedPushedLimit(df, 2, "projected LIMIT read") + } + } + + // V2LimitPushdownTest.testLimitWithMultipleFiles. + test("LIMIT over multiple files returns the requested rows and reaches the scan") { + val table = "v2_scan_e2e_limit_multiple_files" + withTable(table) { + withV1Mode { + sql(s"CREATE TABLE $table (id INT) USING $tableProvider") + // Preserve the source's deterministic one-row-per-file layout. + (0 until 20).foreach { id => + sql(s"INSERT INTO $table VALUES ($id)") + } + } + + val numFiles = fileCount(table) + assert(numFiles == 20L, s"expected 20 source files, got $numFiles") + + val df = sql(s"SELECT * FROM $table LIMIT 3") + assert(df.count() == 3L, "LIMIT 3 over 20 single-row files should return 3 rows") + assertExpectedPushedLimit(df, 3, "LIMIT read over multiple files") + assertRouteSpecificInputFileCount( + df, + expected = 3, + context = "LIMIT 3 route-specific input-file count") + } + } + + // DeltaV2ScanTest.testLimitPushdown_missingNumRecordsPlansAllFiles. + test("LIMIT pushdown retains files without numRecords") { + val table = "v2_scan_e2e_limit_missing_num_records" + withTable(table) { + withV1Mode { + withSQLConf(DeltaSQLConf.DELTA_COLLECT_STATS.key -> "false") { + sql( + s"CREATE TABLE $table (id INT) USING $tableProvider " + + "TBLPROPERTIES ('delta.enableRowTracking' = 'false')") + sql(s"INSERT INTO $table VALUES (1)") + sql(s"INSERT INTO $table VALUES (2)") + } + } + + val numFiles = fileCount(table) + assert(numFiles == 2L, s"expected two source files, got $numFiles") + + val df = sql(s"SELECT * FROM $table LIMIT 1") + val rows = df.collect().toSeq + assert(rows.length == 1, s"expected one row, got ${rows.length}") + assert(Set(Row(1), Row(2)).contains(rows.head), s"unexpected source row ${rows.head}") + assertExpectedPushedLimit(df, 1, "LIMIT read with missing numRecords") + assertExpectedScan(df, "LIMIT read with missing numRecords") + assertRouteSpecificInputFileCount( + df, + expected = 2, + // With file statistics disabled, neither file counts toward the pushed limit. + context = "LIMIT 1 must retain files without numRecords") + } + } + + // V2LimitPushdownTest.testLimitWithNonDeterministicFilterIsNotPushed. + test("a non-deterministic filter prevents LIMIT pushdown") { + val table = "v2_scan_e2e_limit_nondeterministic_filter" + withTable(table) { + withV1Mode { + sql(s"CREATE TABLE $table (id INT) USING $tableProvider") + spark + .range(100) + .selectExpr("cast(id as int) as id") + .write + .mode("append") + .insertInto(table) + } + + val df = sql(s"SELECT * FROM $table WHERE rand(0) > 0.5 LIMIT 5") + assert(df.count() == 5L) + assertExpectedNoPushedLimit(df, "non-deterministic filter with LIMIT") + } + } + + // V2LimitPushdownTest.testLimitWithOffsetPushesCombinedRowRequirement. + test("LIMIT with OFFSET pushes the combined row requirement") { + val table = "v2_scan_e2e_limit_offset" + withTable(table) { + withV1Mode { + sql(s"CREATE TABLE $table (id INT) USING $tableProvider") + sql(s"INSERT INTO $table VALUES (1), (2), (3), (4), (5)") + } + + val df = sql(s"SELECT * FROM $table LIMIT 2 OFFSET 1") + assert(df.count() == 2L) + assertExpectedPushedLimit(df, 3, "LIMIT with OFFSET read") + } + } + + // V2LimitPushdownTest.testLimitOrderBy. + test("ORDER BY with LIMIT returns ordered rows without pushdown") { + val table = "v2_scan_e2e_limit_order_by" + withTable(table) { + withV1Mode { + sql(s"CREATE TABLE $table (id INT, name STRING) USING $tableProvider") + sql( + s"INSERT INTO $table VALUES " + + "(3, 'c'), (1, 'a'), (5, 'e'), (2, 'b'), (4, 'd')") + } + + val df = sql(s"SELECT * FROM $table ORDER BY id LIMIT 3") + checkAnswer(df, Seq(Row(1, "a"), Row(2, "b"), Row(3, "c"))) + assertExpectedNoPushedLimit(df, "ORDER BY with LIMIT") + } + } +} + + +/** Covers LIMIT correctness and pushdown semantics shared across Delta V2 scan paths. + * Path-specific DPP and column-mapping gaps are kept in their corresponding test traits. + */ +class DeltaV2LimitPushdownE2ESuite + extends DeltaV2ScanE2ETestUtils + with DeltaV2LimitPushdownE2ETests +{ +} diff --git a/spark-unified/src/test/scala/io/delta/spark/internal/v2/read/DeltaV2MetadataReadE2ESuite.scala b/spark-unified/src/test/scala/io/delta/spark/internal/v2/read/DeltaV2MetadataReadE2ESuite.scala new file mode 100644 index 00000000000..cfe64608077 --- /dev/null +++ b/spark-unified/src/test/scala/io/delta/spark/internal/v2/read/DeltaV2MetadataReadE2ESuite.scala @@ -0,0 +1,195 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.spark.internal.v2.read + +import java.net.URI +import java.sql.Timestamp + +import org.apache.spark.sql.{DataFrame, Row} +import org.apache.spark.sql.execution.datasources.FileFormat +import org.apache.spark.sql.types.StructType + +private[read] trait DeltaV2MetadataReadE2ETests { + self: DeltaV2ScanE2ETestUtils => + + private val baseMetadataFieldNames = + FileFormat.BASE_METADATA_FIELDS.map(_.name) + + private def withMetadataTable( + table: String, + rowTrackingEnabled: Boolean)(body: String => Unit): Unit = { + withTable(table) { + withV1Mode { + sql( + s"CREATE TABLE $table (id LONG, name STRING) USING $tableProvider " + + s"TBLPROPERTIES ('delta.enableRowTracking' = '$rowTrackingEnabled')") + withSQLConf("spark.sql.leafNodeDefaultParallelism" -> "1") { + sql(s"INSERT INTO $table VALUES (1, 'Alice'), (2, 'Bob'), (3, 'Charlie')") + } + } + val tableLocation = withV1Mode { + sql(s"DESCRIBE DETAIL $table").select("location").head().getString(0) + } + body(tableLocation) + } + } + + private def assertFileBelongsToTable(filePath: String, tableLocation: String): Unit = { + val normalizedFileUri = new URI(filePath).normalize() + val normalizedTableUri = new URI(tableLocation).normalize() + assert( + Option(normalizedFileUri.getScheme) == Option(normalizedTableUri.getScheme), + s"expected file URI scheme to match table location $tableLocation, got $filePath") + assert( + Option(normalizedFileUri.getAuthority) == Option(normalizedTableUri.getAuthority), + s"expected file URI authority to match table location $tableLocation, got $filePath") + val normalizedTablePath = normalizedTableUri.getPath.stripSuffix("/") + assert( + normalizedFileUri.getPath.startsWith(s"$normalizedTablePath/"), + s"expected file path under table location $tableLocation, got $filePath") + } + + private def assertMetadataValueShape( + fieldName: String, + value: Any, + tableLocation: String): Unit = { + assert(value != null, s"expected _metadata.$fieldName to be populated") + fieldName match { + case "file_path" => + val filePath = value.asInstanceOf[String] + assert(filePath.startsWith("file:"), s"expected file URI, got $filePath") + assert(filePath.endsWith(".parquet"), s"expected Parquet path, got $filePath") + assertFileBelongsToTable(filePath, tableLocation) + case "file_name" => + val fileName = value.asInstanceOf[String] + assert(fileName.endsWith(".parquet"), s"expected Parquet file name, got $fileName") + assert(!fileName.contains("/"), s"expected a leaf file name, got $fileName") + case "file_size" => + assert(value.asInstanceOf[Long] > 0L, s"expected positive file size, got $value") + case "file_block_start" => + assert(value.asInstanceOf[Long] >= 0L, s"expected non-negative block start, got $value") + case "file_block_length" => + assert(value.asInstanceOf[Long] > 0L, s"expected positive block length, got $value") + case "file_modification_time" => + val modificationTime = value.asInstanceOf[Timestamp] + assert( + modificationTime.getTime > 0L, + s"expected positive file modification epoch, got $modificationTime") + } + } + + private def assertSingleFileMetadataRows( + df: DataFrame, + valueOrdinal: Int, + fieldName: String, + tableLocation: String): Unit = { + val rows = df.collect().toSeq + assert(rows.length == 3, s"expected three rows, got ${rows.length}") + rows.foreach { row => + assertMetadataValueShape(fieldName, row.get(valueOrdinal), tableLocation) + } + val values = rows.map(_.get(valueOrdinal)) + assert( + values.distinct.size == 1, + s"expected one source file for _metadata.$fieldName, got ${values.distinct}") + } + + private def assertMetadataStruct(metadata: Row, tableLocation: String): Unit = { + assert(metadata != null, "expected _metadata to be populated") + baseMetadataFieldNames.zipWithIndex.foreach { case (fieldName, ordinal) => + assertMetadataValueShape(fieldName, metadata.get(ordinal), tableLocation) + } + } + + Seq(true, false).foreach { rowTrackingEnabled => + test(s"file_path is available when row tracking is $rowTrackingEnabled") { + val table = s"v2_scan_e2e_metadata_path_$rowTrackingEnabled" + withMetadataTable(table, rowTrackingEnabled) { tableLocation => + val df = + sql(s"SELECT id, _metadata.file_path FROM $table ORDER BY id") + assert(df.schema.fieldNames.toSeq == Seq("id", "file_path")) + assert(df.collect().map(_.getLong(0)).toSeq == Seq(1L, 2L, 3L)) + assertSingleFileMetadataRows( + df, + valueOrdinal = 1, + fieldName = "file_path", + tableLocation = tableLocation) + assertExpectedScan(df, s"file_path read with row tracking $rowTrackingEnabled") + } + } + } + + baseMetadataFieldNames.foreach { fieldName => + test(s"base metadata field $fieldName has the expected shape") { + val table = s"v2_scan_e2e_metadata_${fieldName}_field" + withMetadataTable(table, rowTrackingEnabled = false) { tableLocation => + val query = + s"SELECT id, _metadata.$fieldName AS metadata_value FROM $table ORDER BY id" + val df = sql(query) + assert(df.schema.fieldNames.toSeq == Seq("id", "metadata_value")) + assertSingleFileMetadataRows( + df, + valueOrdinal = 1, + fieldName = fieldName, + tableLocation = tableLocation) + if (fieldName == "file_modification_time") { + val v1Rows = withV1Mode { + sql(query).collect().toSeq + } + checkAnswer(df, v1Rows) + } + assertExpectedScan(df, s"$fieldName metadata read") + } + } + } + + test("the whole metadata struct contains every base field") { + val table = "v2_scan_e2e_metadata_whole" + withMetadataTable(table, rowTrackingEnabled = false) { tableLocation => + val df = sql(s"SELECT _metadata FROM $table") + assert(df.schema.fieldNames.toSeq == Seq("_metadata")) + val metadataType = df.schema("_metadata").dataType.asInstanceOf[StructType] + assert(metadataType.fieldNames.toSeq == baseMetadataFieldNames) + val rows = df.collect().toSeq + assert(rows.length == 3, s"expected three rows, got ${rows.length}") + rows.foreach(row => assertMetadataStruct(row.getStruct(0), tableLocation)) + assertExpectedScan(df, "whole metadata struct read") + } + } + + test("metadata alongside star preserves metadata and data columns") { + val table = "v2_scan_e2e_metadata_star" + withMetadataTable(table, rowTrackingEnabled = false) { tableLocation => + val df = sql(s"SELECT _metadata, * FROM $table ORDER BY id") + assert(df.schema.fieldNames.toSeq == Seq("_metadata", "id", "name")) + val rows = df.collect().toSeq + assert(rows.map(_.getLong(1)) == Seq(1L, 2L, 3L)) + assert(rows.map(_.getString(2)) == Seq("Alice", "Bob", "Charlie")) + rows.foreach(row => assertMetadataStruct(row.getStruct(0), tableLocation)) + assertExpectedScan(df, "metadata alongside star read") + } + } +} + +/** Covers Spark base `_metadata` field values and schemas shared across Delta V2 scan paths. + * Row-tracking-only metadata fields stay outside this suite. + */ +class DeltaV2MetadataReadE2ESuite + extends DeltaV2ScanE2ETestUtils + with DeltaV2MetadataReadE2ETests +{ +} diff --git a/spark-unified/src/test/scala/io/delta/spark/internal/v2/read/DeltaV2ReadE2ESuite.scala b/spark-unified/src/test/scala/io/delta/spark/internal/v2/read/DeltaV2ReadE2ESuite.scala new file mode 100644 index 00000000000..e9f0d45461c --- /dev/null +++ b/spark-unified/src/test/scala/io/delta/spark/internal/v2/read/DeltaV2ReadE2ESuite.scala @@ -0,0 +1,312 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.spark.internal.v2.read + +import org.apache.spark.sql.Row + +private[read] trait DeltaV2ReadE2ETests { + self: DeltaV2ScanE2ETestUtils => + + test("plain catalog read preserves the requested schema and rows") { + withDeltaTable("v2_scan_e2e_plain") { + checkRead("plain catalog read")(sql("SELECT id, value FROM v2_scan_e2e_plain")) { df => + assert(df.schema.fieldNames.toSeq == Seq("id", "value")) + checkAnswer(df, Seq(Row(1L, "a"), Row(2L, "b"), Row(3L, "c"))) + } + } + } + + test("partition-filtered read returns matching partition rows") { + withTable("v2_scan_e2e_partitioned") { + withV1Mode { + sql( + s"CREATE TABLE v2_scan_e2e_partitioned (id LONG, part STRING) " + + s"USING $tableProvider PARTITIONED BY (part)") + sql("INSERT INTO v2_scan_e2e_partitioned VALUES (1, 'x'), (2, 'y'), (3, 'x')") + } + checkRead("partition-filtered read") { + sql("SELECT id, part FROM v2_scan_e2e_partitioned WHERE part = 'x'") + } { df => + checkAnswer(df, Seq(Row(1L, "x"), Row(3L, "x"))) + } + } + } + + test("projection with a residual data filter returns matching rows") { + withDeltaTable("v2_scan_e2e_projection") { + checkRead("projected residual-filtered read") { + sql("SELECT value FROM v2_scan_e2e_projection WHERE id > 1") + } { df => + checkAnswer(df, Seq(Row("b"), Row("c"))) + } + } + } + + test("a deletion-vector-capable table without selected vectors preserves live rows") { + withDeltaTable( + "v2_scan_e2e_dv_unused", + "TBLPROPERTIES ('delta.enableDeletionVectors' = 'true')") { + checkRead("deletion-vector-capable read without selected vectors") { + sql("SELECT id, value FROM v2_scan_e2e_dv_unused") + } { df => + checkAnswer(df, Seq(Row(1L, "a"), Row(2L, "b"), Row(3L, "c"))) + } + } + } + + test("a selected deletion vector preserves only live rows") { + val table = "v2_scan_e2e_dv" + withDeltaTable(table, "TBLPROPERTIES ('delta.enableDeletionVectors' = 'true')") { + withV1Mode { + sql(s"DELETE FROM $table WHERE id = 2") + } + val query = s"SELECT id, value FROM $table" + assertAnySelectedFileHasDeletionVector( + sql(query), + "selected deletion-vector read") + val df = sql(query) + checkAnswer(df, Seq(Row(1L, "a"), Row(3L, "c"))) + assertExpectedScan(df, "selected deletion-vector read") + } + } + + test("mixed deletion-vector and plain files preserve only live rows") { + val table = "v2_scan_e2e_dv_mixed" + withDeltaTable(table, "TBLPROPERTIES ('delta.enableDeletionVectors' = 'true')") { + withV1Mode { + sql(s"INSERT INTO $table VALUES (4, 'd'), (5, 'e')") + sql(s"DELETE FROM $table WHERE id = 2") + } + val query = s"SELECT id, value FROM $table" + assertMixedSelectedFileDeletionVectors( + sql(query), + "mixed deletion-vector and plain read") + val df = sql(query) + checkAnswer(df, Seq(Row(1L, "a"), Row(3L, "c"), Row(4L, "d"), Row(5L, "e"))) + assertExpectedScan(df, "mixed deletion-vector and plain read") + } + } + + test("partitioned deletion vectors preserve live rows and partition pruning") { + val table = "v2_scan_e2e_dv_partitioned" + withTable(table) { + withV1Mode { + sql( + s"CREATE TABLE $table (id LONG, part STRING) USING $tableProvider " + + "PARTITIONED BY (part) " + + "TBLPROPERTIES ('delta.enableDeletionVectors' = 'true')") + sql(s"INSERT INTO $table VALUES (1, 'x'), (2, 'x'), (3, 'y'), (4, 'y')") + sql(s"DELETE FROM $table WHERE id IN (2, 3)") + } + val numFiles = fileCount(table) + assert(numFiles == 2L) + val query = s"SELECT id, part FROM $table" + assertAllSelectedFilesHaveDeletionVectors( + sql(query), + "partitioned deletion-vector read") + val df = sql(query) + checkAnswer(df, Seq(Row(1L, "x"), Row(4L, "y"))) + assertExpectedScan(df, "partitioned deletion-vector read") + + val pruned = sql(s"SELECT id, part FROM $table WHERE part = 'y'") + checkAnswer(pruned, Seq(Row(4L, "y"))) + assertExpectedScan(pruned, "partitioned deletion-vector read with partition filter") + } + } + + test("a second delete on the same file reads through the rewritten vector") { + val table = "v2_scan_e2e_dv_rewritten" + withDeltaTable(table, "TBLPROPERTIES ('delta.enableDeletionVectors' = 'true')") { + withV1Mode { + sql(s"DELETE FROM $table WHERE id = 1") + sql(s"DELETE FROM $table WHERE id = 2") + } + val query = s"SELECT id, value FROM $table" + assertAnySelectedFileHasDeletionVector( + sql(query), + "rewritten deletion-vector read") + val df = sql(query) + checkAnswer(df, Seq(Row(3L, "c"))) + assertExpectedScan(df, "rewritten deletion-vector read") + } + } + + test("row-tracking protocol and deletion vectors preserve ordinary-column rows") { + val table = "v2_scan_e2e_dv_row_tracking" + withDeltaTable( + table, + "TBLPROPERTIES ('delta.enableDeletionVectors' = 'true', " + + "'delta.enableRowTracking' = 'true')") { + withV1Mode { + sql(s"DELETE FROM $table WHERE id = 2") + } + val query = s"SELECT id, value FROM $table" + assertAnySelectedFileHasDeletionVector( + sql(query), + "row-tracking protocol DV read") + val df = sql(query) + checkAnswer(df, Seq(Row(1L, "a"), Row(3L, "c"))) + assertExpectedScan(df, "row-tracking protocol DV read") + } + } + + test("a row-tracking table preserves ordinary-column rows") { + withDeltaTable( + "v2_scan_e2e_row_tracking", + "TBLPROPERTIES ('delta.enableRowTracking' = 'true')") { + checkRead("row-tracking table ordinary-column read") { + sql("SELECT id, value FROM v2_scan_e2e_row_tracking") + } { df => + checkAnswer(df, Seq(Row(1L, "a"), Row(2L, "b"), Row(3L, "c"))) + } + } + } + + test("partition column in the middle preserves DDL schema order") { + withPartitionColumnInMiddleTable("v2_scan_e2e_partition_middle") { + checkRead("partition column in the middle") { + sql("SELECT * FROM v2_scan_e2e_partition_middle ORDER BY id") + } { df => + assert(df.schema.fieldNames.toSeq == Seq("id", "part", "col3")) + checkAnswer( + df, + Seq(Row(1L, 10L, 100), Row(2L, 20L, 200), Row(3L, 30L, 300))) + } + } + } + + // V2ReadTest.testBatchReadPartitionColumnInMiddleWithPruning. + test("partition column in the middle preserves pruned projections") { + val table = "v2_scan_e2e_partition_middle_pruning" + withPartitionColumnInMiddleTable(table) { + checkRead("reordered partition and data projection") { + sql(s"SELECT part, id FROM $table ORDER BY id") + } { df => + assert(df.schema.fieldNames.toSeq == Seq("part", "id")) + checkAnswer(df, Seq(Row(10L, 1L), Row(20L, 2L), Row(30L, 3L))) + } + + checkRead("partition-only projection") { + sql(s"SELECT part FROM $table ORDER BY part") + } { df => + assert(df.schema.fieldNames.toSeq == Seq("part")) + checkAnswer(df, Seq(Row(10L), Row(20L), Row(30L))) + } + + checkRead("data-only projection") { + sql(s"SELECT col3 FROM $table ORDER BY col3") + } { df => + assert(df.schema.fieldNames.toSeq == Seq("col3")) + checkAnswer(df, Seq(Row(100), Row(200), Row(300))) + } + } + } + + // V2ReadTest.testBatchReadPartitionColumnAtEnd. + test("partition column at the end preserves DDL schema order") { + val table = "v2_scan_e2e_partition_end" + withTable(table) { + withV1Mode { + sql( + s"CREATE TABLE $table (id LONG, col3 INT, part LONG) USING $tableProvider " + + "PARTITIONED BY (part)") + sql( + s"INSERT INTO $table VALUES " + + "(1, 100, 10), (2, 200, 20), (3, 300, 30)") + } + checkRead("partition column at the end") { + sql(s"SELECT * FROM $table ORDER BY id") + } { df => + assert(df.schema.fieldNames.toSeq == Seq("id", "col3", "part")) + checkAnswer( + df, + Seq(Row(1L, 100, 10L), Row(2L, 200, 20L), Row(3L, 300, 30L))) + } + } + } + + // V2ReadTest.testBatchReadMultiplePartitionColumns. + test("multiple interleaved partition columns preserve DDL schema order") { + val table = "v2_scan_e2e_multiple_partitions" + withTable(table) { + withV1Mode { + sql( + s"CREATE TABLE $table (a LONG, p1 STRING, b INT, p2 STRING, c DOUBLE) " + + s"USING $tableProvider PARTITIONED BY (p2, p1)") + sql( + s"INSERT INTO $table VALUES " + + "(1, 'x', 10, 'y', 1.5), " + + "(2, 'x', 20, 'z', 2.5), " + + "(3, 'w', 30, 'y', 3.5)") + } + checkRead("multiple interleaved partition columns") { + sql(s"SELECT * FROM $table ORDER BY a") + } { df => + assert(df.schema.fieldNames.toSeq == Seq("a", "p1", "b", "p2", "c")) + checkAnswer( + df, + Seq( + Row(1L, "x", 10, "y", 1.5), + Row(2L, "x", 20, "z", 2.5), + Row(3L, "w", 30, "y", 3.5))) + } + } + } + + // V2ReadTest.testBatchReadWithDeletionVectorAndPartitionColumnInMiddle. + test("deletion vectors with a middle partition column preserve rows and DDL order") { + val table = "v2_scan_e2e_dv_partition_middle" + withTable(table) { + withV1Mode { + sql( + s"CREATE TABLE $table (id LONG, part LONG, val INT) USING $tableProvider " + + "PARTITIONED BY (part) " + + "TBLPROPERTIES ('delta.enableDeletionVectors' = 'true')") + spark + .range(2000) + .selectExpr("id", "id % 2 AS part", "cast(id * 10 AS INT) AS val") + .write + .mode("append") + .insertInto(table) + sql(s"DELETE FROM $table WHERE id < 100") + } + + val query = s"SELECT * FROM $table ORDER BY id" + assertAnySelectedFileHasDeletionVector( + sql(query), + "deletion-vector read with a middle partition column") + val df = sql(query) + assert(df.schema.fieldNames.toSeq == Seq("id", "part", "val")) + val rows = df.collect() + assert(rows.length == 1900, "expected 1900 surviving rows after delete") + assert(rows.head == Row(100L, 0L, 1000)) + assertExpectedScan( + df, + "deletion-vector read with a middle partition column") + } + } +} + + +/** Covers ordinary read, schema, and deletion-vector semantics shared across Delta V2 scan paths. + * Path-specific fallback and gap cases are kept in their corresponding test traits. + */ +class DeltaV2ReadE2ESuite + extends DeltaV2ScanE2ETestUtils + with DeltaV2ReadE2ETests +{ +} diff --git a/spark-unified/src/test/scala/io/delta/spark/internal/v2/read/DeltaV2ScanE2ETestUtils.scala b/spark-unified/src/test/scala/io/delta/spark/internal/v2/read/DeltaV2ScanE2ETestUtils.scala new file mode 100644 index 00000000000..e0c6b1b6bf0 --- /dev/null +++ b/spark-unified/src/test/scala/io/delta/spark/internal/v2/read/DeltaV2ScanE2ETestUtils.scala @@ -0,0 +1,173 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.spark.internal.v2.read + +import org.apache.spark.sql.delta.DeltaTableProvider +import org.apache.spark.sql.delta.sources.DeltaSQLConf +import org.apache.spark.sql.delta.test.{DeltaSQLCommandTest, DeltaSQLTestUtils} + +import org.apache.spark.SparkConf +import org.apache.spark.sql.{DataFrame, QueryTest} +import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper +import org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanRelation + +/** Provides shared table fixtures and path-neutral plan hooks for Delta V2 E2E suites. + * Concrete suites combine these utilities with reusable tests and path-specific overrides. + */ +private[read] trait DeltaV2ScanE2ETestUtils + extends QueryTest + with DeltaSQLCommandTest + with DeltaSQLTestUtils + with DeltaTableProvider + with AdaptiveSparkPlanHelper + with DeltaV2BatchScanAssertions { + + protected def configureE2ESparkConf(conf: SparkConf): SparkConf = conf + + abstract override protected def sparkConf: SparkConf = + configureE2ESparkConf( + super.sparkConf.set(DeltaSQLConf.V2_ENABLE_MODE.key, "STRICT")) + + protected def withV1Mode[T](body: => T): T = + withSQLConf(DeltaSQLConf.V2_ENABLE_MODE.key -> "NONE")(body) + + protected def withDeltaTable( + name: String, + properties: String = "")(body: => Unit): Unit = { + withTable(name) { + withV1Mode { + sql(s"CREATE TABLE $name (id LONG, value STRING) USING $tableProvider $properties") + sql(s"INSERT INTO $name VALUES (1, 'a'), (2, 'b'), (3, 'c')") + } + body + } + } + + protected def withPartitionColumnInMiddleTable( + name: String, + properties: String = "")(body: => Unit): Unit = { + withTable(name) { + withV1Mode { + sql( + s"CREATE TABLE $name (id LONG, part LONG, col3 INT) USING $tableProvider " + + s"PARTITIONED BY (part) $properties") + sql( + s"INSERT INTO $name VALUES " + + "(1, 10, 100), (2, 20, 200), (3, 30, 300)") + } + body + } + } + + protected def fileCount(table: String): Long = { + withV1Mode { + sql(s"DESCRIBE DETAIL $table") + .selectExpr("CAST(numFiles AS BIGINT)") + .head() + .getLong(0) + } + } + + protected def withDeltaV2ScanForSelectedFiles[T]( + query: => DataFrame, + context: String)(body: DeltaV2Scan => T): T = { + val df = query + val plan = df.queryExecution.optimizedPlan + val scans = plan.collect { + case relation: DataSourceV2ScanRelation + if relation.scan.isInstanceOf[DeltaV2Scan] => + relation.scan.asInstanceOf[DeltaV2Scan] + } + assert( + scans.length == 1, + s"expected one optimized DeltaV2Scan for $context, got " + + s"${scans.length}:\n$plan") + body(scans.head) + } + + private def selectedFileDeletionVectorFlags( + query: => DataFrame, + context: String): Seq[Boolean] = { + withDeltaV2ScanForSelectedFiles(query, context) { scan => + scan.preparedScan().files.map(_.deletionVector != null) + } + } + + protected def assertAnySelectedFileHasDeletionVector( + query: => DataFrame, + context: String): Unit = { + val hasDeletionVectors = selectedFileDeletionVectorFlags(query, context) + assert( + hasDeletionVectors.contains(true), + s"expected a selected file with a deletion vector for $context") + } + + protected def assertMixedSelectedFileDeletionVectors( + query: => DataFrame, + context: String): Unit = { + val hasDeletionVectors = selectedFileDeletionVectorFlags(query, context) + assert( + hasDeletionVectors.contains(true) && hasDeletionVectors.contains(false), + s"expected selected files with and without deletion vectors for $context") + } + + protected def assertAllSelectedFilesHaveDeletionVectors( + query: => DataFrame, + context: String): Unit = { + val hasDeletionVectors = selectedFileDeletionVectorFlags(query, context) + assert( + hasDeletionVectors.nonEmpty && hasDeletionVectors.forall(identity), + s"expected all selected files to have deletion vectors for $context") + } + + protected def assertExpectedScan(df: DataFrame, context: String): Unit = + assertDeltaV2BatchScan(df, context) + + protected def assertExpectedScanCount( + df: DataFrame, + expected: Int, + context: String): Unit = + assertDeltaV2BatchScanCount(df, expected, context) + + protected def expectedScanRequiredFieldNames( + df: DataFrame, + expectedScanCount: Int, + context: String): Seq[Seq[String]] = + deltaV2BatchScanReadFieldNames(df, expectedScanCount, context) + + protected def assertExpectedPushedLimit( + df: DataFrame, + expected: Int, + context: String): Unit = + assertDeltaV2BatchScanPushedLimit(df, expected, context) + + protected def assertExpectedNoPushedLimit(df: DataFrame, context: String): Unit = + assertDeltaV2BatchScanHasNoPushedLimit(df, context) + + protected def assertRouteSpecificInputFileCount( + df: DataFrame, + expected: Int, + context: String): Unit = {} + + protected def checkRead(context: String)( + query: => DataFrame)( + assertions: DataFrame => Unit): Unit = { + val df = query + assertions(df) + assertExpectedScan(df, context) + } +} diff --git a/spark-unified/src/test/scala/io/delta/spark/internal/v2/tablemanager/DeltaV2TableManagerCacheKeySuite.scala b/spark-unified/src/test/scala/io/delta/spark/internal/v2/tablemanager/DeltaV2TableManagerCacheKeySuite.scala new file mode 100644 index 00000000000..9740a0cd095 --- /dev/null +++ b/spark-unified/src/test/scala/io/delta/spark/internal/v2/tablemanager/DeltaV2TableManagerCacheKeySuite.scala @@ -0,0 +1,212 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.spark.internal.v2.tablemanager + +import java.io.File + +import scala.jdk.CollectionConverters._ + +import io.delta.spark.internal.v2.tablemanager.DeltaV2TableManagerCache.CacheKey + +import org.apache.spark.sql.delta.sources.DeltaSQLConf + +import org.apache.hadoop.fs.{FileSystem, Path} +import org.apache.spark.sql.QueryTest +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable, CatalogTableType} +import org.apache.spark.sql.test.SharedSparkSession +import org.apache.spark.sql.types.StructType + +class DeltaV2TableManagerCacheKeySuite + extends QueryTest + with SharedSparkSession { + + test("produces a _delta_log path from the data path") { + withTempDir { dataPath => + val key = CacheKey.from(spark, dataPath.getCanonicalPath, Map.empty[String, String].asJava) + + assert(key.path.isAbsolute) + assert(key.path.toString.endsWith("_delta_log")) + val parentUri = key.path.getParent.toUri + val expectedUri = new Path(dataPath.getCanonicalPath).toUri.normalize() + assert(parentUri.normalize().getPath === expectedUri.getPath, + "qualified parent path component must match data path") + } + } + + test("same inputs produce equal keys") { + withTempDir { dataPath => + val options = Map("fs.test.option" -> "value", "reader.option" -> "ignored") + val first = CacheKey.from(spark, dataPath.getCanonicalPath, options.asJava) + val second = CacheKey.from(spark, dataPath.getCanonicalPath, options.asJava) + assert(first === second) + } + } + + test("preserves path derivation for paths containing colons") { + withTempDir { parent => + val dataPath = new File(parent, "table:with:colons") + assert(dataPath.mkdirs()) + + withSQLConf(DeltaSQLConf.DELTA_WORK_AROUND_COLONS_IN_HADOOP_PATHS.key -> "true") { + val key = CacheKey.from(spark, dataPath.getCanonicalPath, Map.empty[String, String].asJava) + assert(key.path.isAbsolute) + assert(key.path.toString.endsWith("_delta_log")) + } + } + } + + test("filters non-filesystem options and isolates filesystem credentials") { + withTempDir { dataPath => + val path = dataPath.getCanonicalPath + val firstOptions = Map( + "fs.test.secret" -> "first", + "dfs.test.endpoint" -> "endpoint", + "reader.option" -> "ignored") + val secondOptions = firstOptions.updated("fs.test.secret", "second") + + val first = CacheKey.from(spark, path, firstOptions.asJava) + val second = CacheKey.from(spark, path, secondOptions.asJava) + + assert(first.sessionInvariantFsOptions === firstOptions.filter { case (key, _) => + key.startsWith("fs.") || key.startsWith("dfs.") + }) + assert(!first.sessionInvariantFsOptions.contains("reader.option")) + assert(first !== second) + } + } + + test("redacts filesystem option names and values from rendering") { + withTempDir { dataPath => + val optionName = "fs.test.secret" + val optionValue = "credential-value" + val key = + CacheKey.from(spark, dataPath.getCanonicalPath, Map(optionName -> optionValue).asJava) + val rendered = key.toString + + assert(!rendered.contains(optionName)) + assert(!rendered.contains(optionValue)) + assert(rendered.contains("fsOptions=")) + } + } + + // --- Local-filesystem path normalization ------------------------- + + test("trailing slash in data path converges to same key") { + withTempDir { dataPath => + val path = dataPath.getCanonicalPath + val empty = Map.empty[String, String].asJava + val withSlash = CacheKey.from(spark, path + "/", empty) + val withoutSlash = CacheKey.from(spark, path, empty) + assert(withSlash === withoutSlash) + } + } + + test("explicit file scheme alias converges to same key") { + withTempDir { dataPath => + val plain = dataPath.getCanonicalPath + val withScheme = "file://" + plain + val empty = Map.empty[String, String].asJava + val keyPlain = CacheKey.from(spark, plain, empty) + val keyScheme = CacheKey.from(spark, withScheme, empty) + assert(keyPlain === keyScheme) + } + } + + test("different fs options produce distinct keys for same path") { + withTempDir { dataPath => + val path = dataPath.getCanonicalPath + val keyA = CacheKey.from(spark, path, Map("fs.test" -> "a").asJava) + val keyB = CacheKey.from(spark, path, Map("fs.test" -> "b").asJava) + assert(keyA !== keyB) + } + } + + // --- LOAD_FILE_SYSTEM_CONFIGS_FROM_DATAFRAME_OPTIONS off --------- + + test("DataFrame fs options ignored when config is off") { + withTempDir { dataPath => + val path = dataPath.getCanonicalPath + val confKey = DeltaSQLConf.LOAD_FILE_SYSTEM_CONFIGS_FROM_DATAFRAME_OPTIONS.key + withSQLConf(confKey -> "false") { + val withFsOpts = CacheKey.from( + spark, path, Map("fs.test" -> "v", "dfs.test" -> "x").asJava) + val bare = CacheKey.from(spark, path, Map.empty[String, String].asJava) + assert(withFsOpts === bare, "fs/dfs DataFrame options should be ignored") + assert(withFsOpts.sessionInvariantFsOptions.isEmpty) + } + } + } + + test("catalog storage properties authoritative when config off") { + withTempDir { dataPath => + val path = dataPath.getCanonicalPath + val confKey = DeltaSQLConf.LOAD_FILE_SYSTEM_CONFIGS_FROM_DATAFRAME_OPTIONS.key + val catalogTable = CatalogTable( + identifier = TableIdentifier("test_table"), + tableType = CatalogTableType.EXTERNAL, + storage = CatalogStorageFormat.empty.copy( + properties = Map("fs.catalog.key" -> "catalog-value")), + schema = new StructType()) + withSQLConf(confKey -> "false") { + val withCatalog = CacheKey.from( + spark, path, Map("fs.ignored" -> "from-options").asJava, + Some(catalogTable)) + val bare = CacheKey.from(spark, path, Map.empty[String, String].asJava) + assert(withCatalog !== bare, "catalog fs options should differentiate keys") + assert(withCatalog.sessionInvariantFsOptions.contains("fs.catalog.key")) + assert(!withCatalog.sessionInvariantFsOptions.contains("fs.ignored")) + } + } + } + + // --- Qualification and non-retention invariants ------------------ + + test("absolute schemeless path is qualified with a URI scheme") { + withTempDir { dataPath => + val schemelessAbsolute = dataPath.getCanonicalPath + assert(!schemelessAbsolute.contains("://"), "precondition: input must not carry a URI scheme") + val key = CacheKey.from(spark, schemelessAbsolute, Map.empty[String, String].asJava) + + // Independently compute the expected qualified path using the same Hadoop configuration and + // FileSystem that the production code would use. + val rawLogPath = new Path(new Path(schemelessAbsolute), "_delta_log") + // scalastyle:off deltahadoopconfiguration + val hadoopConf = spark.sessionState.newHadoopConfWithOptions(Map.empty) + // scalastyle:on deltahadoopconfiguration + val expectedPath = rawLogPath.getFileSystem(hadoopConf).makeQualified(rawLogPath) + + assert(key.path === expectedPath, "key path must equal the FileSystem-qualified path") + assert(key.path.toUri.getScheme != null, "qualified path must carry a URI scheme") + assert( + key.path.toUri.getAuthority != null || + key.path.toUri.getScheme == "file", + "qualified path must carry authority or be local") + } + } + + test("cache key does not retain a Hadoop FileSystem instance") { + // Deterministic structural assertion: none of the case-class fields have a type assignable to + // Hadoop FileSystem. This pins the non-retention invariant without relying on GC timing or + // weak references, which are invalid under Hadoop's global FileSystem cache. + val fsClass = classOf[FileSystem] + val fields = classOf[CacheKey].getDeclaredFields + val fsFields = fields.filter(field => fsClass.isAssignableFrom(field.getType)) + assert(fsFields.isEmpty, + "CacheKey must not hold a FileSystem; found: " + + fsFields.map(_.getName).mkString(", ")) + } +} diff --git a/spark-unified/src/test/scala/io/delta/spark/internal/v2/tablemanager/DeltaV2TableManagerCacheSuite.scala b/spark-unified/src/test/scala/io/delta/spark/internal/v2/tablemanager/DeltaV2TableManagerCacheSuite.scala new file mode 100644 index 00000000000..bdfe3e14957 --- /dev/null +++ b/spark-unified/src/test/scala/io/delta/spark/internal/v2/tablemanager/DeltaV2TableManagerCacheSuite.scala @@ -0,0 +1,434 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.spark.internal.v2.tablemanager + +import java.util.Collections +import java.util.concurrent.{CountDownLatch, Executors, TimeUnit} +import java.util.concurrent.atomic.AtomicInteger + +import io.delta.spark.internal.v2.kernel.KernelContext +import io.delta.spark.internal.v2.tablemanager.DeltaV2TableManagerCache.CacheKey + +import org.apache.spark.sql.delta.sources.DeltaSQLConf +import org.apache.spark.sql.delta.storage.LogStore +import org.apache.spark.sql.delta.v2.interop.DeltaV2SnapshotManager + +import com.google.common.base.Ticker +import org.apache.spark.sql.QueryTest +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable, CatalogTableType} +import org.apache.spark.sql.test.SharedSparkSession +import org.apache.spark.sql.types.StructType + +class DeltaV2TableManagerCacheSuite + extends QueryTest + with SharedSparkSession { + + /** Creates a cache key from a data directory using the public factory. */ + private def makeKey(dataPath: String): CacheKey = + CacheKey.from(spark, dataPath, Collections.emptyMap()) + + // Process-global companion tests use unsetCache for isolation. + override def beforeEach(): Unit = { + super.beforeEach() + DeltaV2TableManagerCache.unsetCache() + } + + override def afterEach(): Unit = { + DeltaV2TableManagerCache.unsetCache() + super.afterEach() + } + + // === Per-instance tests (class methods, custom factories) ======= + + test("per-instance: cache hit returns same instance") { + val cache = new DeltaV2TableManagerCache(maxSize = 1000, ttlMinutes = 60) + withTempDir { dir => + val key = makeKey(dir.getCanonicalPath) + val first = cache.getOrCreate(key) + val second = cache.getOrCreate(key) + assert(first eq second) + assert(cache.size() == 1) + } + } + + test("per-instance: different keys produce different entries") { + val cache = new DeltaV2TableManagerCache(maxSize = 1000, ttlMinutes = 60) + withTempDir { dirA => + withTempDir { dirB => + val keyA = makeKey(dirA.getCanonicalPath) + val keyB = makeKey(dirB.getCanonicalPath) + cache.getOrCreate(keyA) + cache.getOrCreate(keyB) + assert(cache.size() == 2) + } + } + } + + test("per-instance: cache hit preserves initialCatalogTableOpt from first load") { + val cache = new DeltaV2TableManagerCache(maxSize = 1000, ttlMinutes = 60) + withTempDir { dir => + val key = makeKey(dir.getCanonicalPath) + val catalogA = CatalogTable( + identifier = TableIdentifier("tableA"), + tableType = CatalogTableType.EXTERNAL, + storage = CatalogStorageFormat.empty, + schema = new StructType()) + val catalogB = CatalogTable( + identifier = TableIdentifier("tableB"), + tableType = CatalogTableType.EXTERNAL, + storage = CatalogStorageFormat.empty, + schema = new StructType()) + + val first = cache.getOrCreate(key, Some(catalogA)) + val second = cache.getOrCreate(key, Some(catalogB)) + assert(first eq second) + val impl = first.asInstanceOf[DeltaV2TableManagerImpl] + assert(impl.initialCatalogTableOpt === Some(catalogA), + "initial catalog should be from first load") + } + } + + test("per-instance: removal listener retires manager on invalidation") { + val stub = new StubTableManager("a") + val cache = new DeltaV2TableManagerCache( + maxSize = 1000, ttlMinutes = 60, + managerFactory = (_, _) => stub) + withTempDir { dir => + val key = makeKey(dir.getCanonicalPath) + cache.getOrCreate(key) + assert(!stub.retired) + cache.invalidate(key) + assert(stub.retired, "retire() should be called on eviction") + } + } + + test("per-instance: maxSize=1 LRU eviction retires old manager") { + val stubA = new StubTableManager("a") + val stubB = new StubTableManager("b") + val stubs = Iterator(stubA, stubB) + val cache = new DeltaV2TableManagerCache( + maxSize = 1, ttlMinutes = 60, + managerFactory = (_, _) => stubs.next()) + withTempDir { dirA => + withTempDir { dirB => + val keyA = makeKey(dirA.getCanonicalPath) + val keyB = makeKey(dirB.getCanonicalPath) + cache.getOrCreate(keyA) + cache.getOrCreate(keyB) + cache.cleanUp() + assert(cache.size() == 1, "effective maxSize should remain 1") + assert(stubA.retired, "LRU entry should be retired on capacity eviction") + assert(!stubB.retired, "newest entry should remain cached") + assert(cache.contains(keyB), "newest entry should still be present") + } + } + } + + test("per-instance: deterministic TTL eviction retires expired entry") { + val ticker = new TestTicker() + val ttlMinutes = 10 + val stub = new StubTableManager("ttl") + val cache = new DeltaV2TableManagerCache( + maxSize = 1000, ttlMinutes = ttlMinutes, + ticker = ticker, + managerFactory = (_, _) => stub) + withTempDir { dir => + val key = makeKey(dir.getCanonicalPath) + cache.getOrCreate(key) + assert(cache.contains(key)) + + ticker.advance((ttlMinutes + 1) * 60L) + cache.cleanUp() + + assert(stub.retired, "expired entry should be retired") + assert(!cache.contains(key)) + } + } + + // --- Guava exception unwrapping via injected factory ------------- + + test("per-instance: unwraps checked exception from ExecutionException") { + val cause = new java.io.IOException("checked-cause") + val cache = new DeltaV2TableManagerCache( + maxSize = 1000, ttlMinutes = 60, + managerFactory = (_, _) => throw cause) + withTempDir { dir => + val key = makeKey(dir.getCanonicalPath) + val caught = intercept[java.io.IOException] { + cache.getOrCreate(key) + } + assert(caught eq cause, "original cause identity must be preserved") + } + } + + test("per-instance: unwraps runtime exception from UncheckedExecutionException") { + val cause = new IllegalStateException("runtime-cause") + val cache = new DeltaV2TableManagerCache( + maxSize = 1000, ttlMinutes = 60, + managerFactory = (_, _) => throw cause) + withTempDir { dir => + val key = makeKey(dir.getCanonicalPath) + val caught = intercept[IllegalStateException] { + cache.getOrCreate(key) + } + assert(caught eq cause, "original cause identity must be preserved") + } + } + + test("per-instance: unwraps Error from ExecutionError") { + val cause = new StackOverflowError("error-cause") + val cache = new DeltaV2TableManagerCache( + maxSize = 1000, ttlMinutes = 60, + managerFactory = (_, _) => throw cause) + withTempDir { dir => + val key = makeKey(dir.getCanonicalPath) + val caught = intercept[StackOverflowError] { + cache.getOrCreate(key) + } + assert(caught eq cause, "original cause identity must be preserved") + } + } + + // --- Per-instance size / getIfPresent / invalidation ------------- + + test("per-instance: size and getIfPresent reflect cache state") { + val cache = new DeltaV2TableManagerCache(maxSize = 1000, ttlMinutes = 60) + withTempDir { dir => + val key = makeKey(dir.getCanonicalPath) + assert(cache.size() == 0) + assert(cache.getIfPresent(key).isEmpty) + cache.getOrCreate(key) + assert(cache.size() == 1) + assert(cache.getIfPresent(key).isDefined) + } + } + + test("per-instance: invalidate removes a single entry") { + val cache = new DeltaV2TableManagerCache(maxSize = 1000, ttlMinutes = 60) + withTempDir { dir => + val key = makeKey(dir.getCanonicalPath) + cache.getOrCreate(key) + assert(cache.contains(key)) + cache.invalidate(key) + assert(!cache.contains(key)) + } + } + + test("per-instance: invalidateByLogPath removes all entries with matching path") { + // Two distinct keys share the same qualified log path but differ in sessionInvariantFsOptions, + // proving removeIf matches every entry whose path equals the target -- not just one key. + val stubA = new StubTableManager("pathA") + val stubB = new StubTableManager("pathB") + val stubs = Iterator(stubA, stubB) + withTempDir { dir => + val sharedLogPath = makeKey(dir.getCanonicalPath).path + val keyA = CacheKey(sharedLogPath, Map("fs.s3a.access.key" -> "AAA")) + val keyB = CacheKey(sharedLogPath, Map("fs.s3a.access.key" -> "BBB")) + val cache = new DeltaV2TableManagerCache( + maxSize = 1000, ttlMinutes = 60, + managerFactory = (_, _) => stubs.next()) + cache.getOrCreate(keyA) + cache.getOrCreate(keyB) + assert(cache.size() == 2) + + cache.invalidateByLogPath(sharedLogPath) + assert(cache.size() == 0, "both entries with the same path must be removed") + assert(stubA.retired, "first manager must be retired") + assert(stubB.retired, "second manager must be retired") + } + } + + test("per-instance: invalidateAll empties the entire cache") { + val cache = new DeltaV2TableManagerCache(maxSize = 1000, ttlMinutes = 60) + withTempDir { dir => + val key = makeKey(dir.getCanonicalPath) + cache.getOrCreate(key) + assert(cache.size() > 0) + cache.invalidateAll() + assert(cache.size() == 0) + } + } + + // === Process-global companion object tests ====================== + + test("enabled by default") { + assert(DeltaV2TableManagerCache.isEnabled(spark.sessionState.conf)) + } + + test("default cache size is positive") { + val size = spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_LOG_CACHE_SIZE) + assert(size > 0, s"Expected positive cache size, got $size") + } + + test("disabled when size=0") { + withSQLConf(DeltaSQLConf.DELTA_LOG_CACHE_SIZE.key -> "0") { + assert(!DeltaV2TableManagerCache.isEnabled(spark.sessionState.conf)) + } + } + + test("getOrCreate bypasses cache when size=0") { + withSQLConf(DeltaSQLConf.DELTA_LOG_CACHE_SIZE.key -> "0") { + withTempDir { dir => + val key = makeKey(dir.getCanonicalPath) + val first = DeltaV2TableManagerCache.getOrCreate(spark.sessionState.conf, key) + val second = DeltaV2TableManagerCache.getOrCreate(spark.sessionState.conf, key) + assert(first ne second) + } + } + } + + test("getOrCreate caches when enabled") { + withSQLConf(DeltaSQLConf.DELTA_LOG_CACHE_SIZE.key -> "1000") { + withTempDir { dir => + val key = makeKey(dir.getCanonicalPath) + val first = DeltaV2TableManagerCache.getOrCreate(spark.sessionState.conf, key) + val second = DeltaV2TableManagerCache.getOrCreate(spark.sessionState.conf, key) + assert(first eq second) + } + } + } + + test("process-global: different sessions share same instance") { + withSQLConf(DeltaSQLConf.DELTA_LOG_CACHE_SIZE.key -> "1000") { + withTempDir { dir => + val sessionA = spark.newSession() + val sessionB = spark.newSession() + sessionB.conf.set(DeltaSQLConf.DELTA_LOG_CACHE_SIZE.key, "500") + + val keyA = CacheKey.from(sessionA, dir.getCanonicalPath, Collections.emptyMap()) + val keyB = CacheKey.from(sessionB, dir.getCanonicalPath, Collections.emptyMap()) + + val fromA = DeltaV2TableManagerCache.getOrCreate(sessionA.sessionState.conf, keyA) + val fromB = DeltaV2TableManagerCache.getOrCreate(sessionB.sessionState.conf, keyB) + + assert(keyA == keyB, "Same path must produce the same cache key across sessions") + assert(fromA eq fromB, "Same key from different sessions must share one cached instance") + } + } + } + + test("per-instance: Guava per-key single-flight loader invokes factory once") { + // Holds the first loader mid-flight while a second caller requests the same key, proving + // Guava's per-key single-flight guarantees exactly one factory invocation and identical result + // for both callers. + val invocations = new AtomicInteger(0) + val loaderEntered = new CountDownLatch(1) + val loaderRelease = new CountDownLatch(1) + val stub = new StubTableManager("single-flight") + + val cache = new DeltaV2TableManagerCache( + maxSize = 1000, ttlMinutes = 60, + managerFactory = (_, _) => { + invocations.incrementAndGet() + loaderEntered.countDown() + loaderRelease.await() + stub + }) + + withTempDir { dir => + val key = makeKey(dir.getCanonicalPath) + // scalastyle:off sparkThreadPools + val executor = Executors.newFixedThreadPool(2) + // scalastyle:on sparkThreadPools + try { + val futureA = executor.submit(() => cache.getOrCreate(key)) + assert(loaderEntered.await(10, TimeUnit.SECONDS), "loader must be entered") + val futureB = executor.submit(() => cache.getOrCreate(key)) + loaderRelease.countDown() + + val resultA = futureA.get(10, TimeUnit.SECONDS) + val resultB = futureB.get(10, TimeUnit.SECONDS) + assert(invocations.get() == 1, "factory must be invoked exactly once") + assert(resultA eq resultB, "both callers must receive the same instance") + } finally { + executor.shutdownNow() + } + } + } + + test("process-global: first caller's size config remains effective") { + // Session A initializes the singleton with maxSize=1. + val sessionA = spark.newSession() + sessionA.conf.set(DeltaSQLConf.DELTA_LOG_CACHE_SIZE.key, "1") + + // Session B would prefer a larger cache, but the singleton is already built. + val sessionB = spark.newSession() + sessionB.conf.set(DeltaSQLConf.DELTA_LOG_CACHE_SIZE.key, "1000") + + withTempDir { dirA => + withTempDir { dirB => + val keyA = CacheKey.from(sessionA, dirA.getCanonicalPath, Collections.emptyMap()) + val keyB = CacheKey.from(sessionB, dirB.getCanonicalPath, Collections.emptyMap()) + + // Session A loads key A -- initializes singleton (size 1). + val originalA = DeltaV2TableManagerCache.getOrCreate(sessionA.sessionState.conf, keyA) + + // Session B loads key B -- evicts key A under size-1. + DeltaV2TableManagerCache.getOrCreate(sessionB.sessionState.conf, keyB) + + // Re-lookup key A: must create a new instance because size-1 evicted the original, proving + // first caller's config governs, not the second caller's (size 1000). + val reloadedA = DeltaV2TableManagerCache.getOrCreate(sessionA.sessionState.conf, keyA) + assert(reloadedA ne originalA, + "Key A should have been evicted under the first caller's maxSize=1 and " + + "reloaded as a new instance") + } + } + } + + // === Facade and manager-field assertions ======================== + + test("forTable produces a manager with expected qualified path and options") { + withTempDir { dir => + val manager = DeltaV2TableManagerCache.forTable( + spark, dir.getCanonicalPath, Collections.emptyMap()) + val impl = manager.asInstanceOf[DeltaV2TableManagerImpl] + assert(impl.qualifiedTableDataPath.isAbsolute) + assert(impl.qualifiedTableDataPath.toUri.getPath.contains(dir.getName)) + assert(impl.sessionInvariantFsOptions.isEmpty) + assert(impl.initialCatalogTableOpt.isEmpty) + val tableStore = impl.logStore + val tableKernelContext = impl.kernelContext + assert(tableKernelContext.logStore eq tableStore) + assert(impl.kernelContext eq tableKernelContext) + assert(impl.logStore eq tableStore) + } + } +} + +// === Test helpers (package-private, same-file) ==================== + +private[tablemanager] class TestTicker extends Ticker { + private var now: Long = System.nanoTime() + override def read(): Long = now + def advance(seconds: Long): Long = { + now += TimeUnit.NANOSECONDS.convert(seconds, TimeUnit.SECONDS) + now + } +} + +private[tablemanager] class StubTableManager(val id: String) extends DeltaV2TableManager { + @volatile var retired: Boolean = false + override private[v2] def kernelContext: KernelContext = + throw new UnsupportedOperationException("stub") + override private[v2] def logStore: LogStore = + throw new UnsupportedOperationException("stub") + override private[v2] def snapshotManager( + catalogTableOpt: Option[CatalogTable]): DeltaV2SnapshotManager = + throw new UnsupportedOperationException("stub") + override def retire(): Unit = { retired = true } +} diff --git a/spark-unified/src/test/scala/org/apache/spark/sql/delta/coordinatedcommits/CatalogManagedStreamingV2Suite.scala b/spark-unified/src/test/scala/org/apache/spark/sql/delta/coordinatedcommits/CatalogManagedStreamingV2Suite.scala new file mode 100644 index 00000000000..69be968a515 --- /dev/null +++ b/spark-unified/src/test/scala/org/apache/spark/sql/delta/coordinatedcommits/CatalogManagedStreamingV2Suite.scala @@ -0,0 +1,48 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.coordinatedcommits + +import org.apache.spark.sql.delta.test.V2ForceTest + +/** Runs the catalog-managed streaming tests through the strict Delta V2 connector. */ +class CatalogManagedStreamingV2Suite + extends CatalogManagedStreamingSuiteBase + with V2ForceTest { + + override protected def assertNoV1Fallback: Boolean = true + + override protected def withV1Mode(f: => Unit): Unit = inV1Mode(f) + + override protected def shouldPassTests: Set[String] = + CatalogManagedStreamingV2Suite.PassingTests + + override protected def shouldFailTests: Set[String] = + CatalogManagedStreamingV2Suite.FailingTests +} + +object CatalogManagedStreamingV2Suite { + val PassingTests: Set[String] = Set( + "stream from delta source" + ) + + val FailingTests: Set[String] = Set( + // TODO(#7140): Delta V2 supports catalog-managed reads but not catalog-managed + // streaming commits. + "stream to delta sink", + "stream from delta source to delta sink with shared commit coordinator" + ) +} diff --git a/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2CDCStreamSuite.scala b/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2CDCStreamSuite.scala index 6ed8d0a83d9..74264a09d62 100644 --- a/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2CDCStreamSuite.scala +++ b/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2CDCStreamSuite.scala @@ -81,7 +81,10 @@ class DeltaV2CDCStreamSuite extends DeltaCDCStreamSuite with V2ForceTest { // ========== CDC combination coverage ========== "CDC stream on partitioned table strips partition and CDC columns correctly", - "CDC stream on column-mapped table passes through correctly" + "CDC stream on column-mapped table passes through correctly", + + // ========== filterFiles END-marker offset-roll ========== + "CDF stream rolls the offset past a change-data commit (keeps END marker)" ) override protected def shouldFailTests: Set[String] = Set( diff --git a/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SinkSuite.scala b/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SinkSuite.scala index 38dd33ba96f..37b32f8c8f5 100644 --- a/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SinkSuite.scala +++ b/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SinkSuite.scala @@ -53,7 +53,8 @@ object DeltaV2SinkSuite { "append mode", "work with aggregation + watermark", "do not trust user nullability, so that parquet files aren't corrupted", - "can't write out with all columns being partition columns" + "can't write out with all columns being partition columns", + "streaming write shreds variant columns only when the table enables it" ) val FailingTests: Set[String] = Set( @@ -71,6 +72,7 @@ object DeltaV2SinkSuite { // No partition support: the V2 write rejects a partitioned target, surfaced as an async // StreamingQueryException, not the outcome these tests expect. "partitioned writing and batch reading", + "partitioned writing into a column-mapped table", "SPARK-21167: encode and decode path correctly", "throw exception when users are trying to write in batch with different partitioning", // No NullType support, creating a table with void column fails. diff --git a/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceColumnMappingSuite.scala b/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceColumnMappingSuite.scala index 32dd8e96ee5..14ea021a7d8 100644 --- a/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceColumnMappingSuite.scala +++ b/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceColumnMappingSuite.scala @@ -16,10 +16,12 @@ package org.apache.spark.sql.delta.test -import org.apache.spark.sql.delta.ColumnMappingStreamingBlockedWorkflowSuiteBase -import org.apache.spark.sql.delta.DeltaSourceIdColumnMappingSuite -import org.apache.spark.sql.delta.DeltaSourceNameColumnMappingSuite -import org.apache.spark.sql.delta.DeltaSourceSuite +import org.apache.spark.sql.delta.{ + ColumnMappingStreamingBlockedWorkflowSuiteBase, + DeltaSourceIdColumnMappingSuite, + DeltaSourceNameColumnMappingSuite, + DeltaSourceSuite +} /** * Test suite that runs DeltaSourceColumnMappingSuite using the V2 connector diff --git a/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceRowTrackingSuite.scala b/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceRowTrackingSuite.scala index 32a3b8cf565..0d50f698e8a 100644 --- a/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceRowTrackingSuite.scala +++ b/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceRowTrackingSuite.scala @@ -28,16 +28,16 @@ class DeltaV2SourceRowTrackingSuite override protected def useDsv2: Boolean = true + override protected def assertNoV1Fallback: Boolean = true + override protected def shouldPassTests: Set[String] = Set( "CDC stream on row-tracking table works when _metadata not selected", "CDC stream on row-tracking column-mapped table rejects _metadata.row_id" ) override protected def shouldFailTests: Set[String] = Set( - // TODO: Requires Spark PR apache/spark#56133, which adds pruneColumns() to - // MicroBatchExecution for SupportsPushDownRequiredColumns sources. Without it - // _metadata is not added to requiredDataSchema, causing AIOOBE at executor - // runtime when codegen reads position N from an N-column batch. + // This inherited test expects _metadata.row_id to be unavailable, but the V2 connector + // exposes row-tracking metadata in streaming. Keep the V1-specific assertion ignored in V2. "_metadata.row_id is not available in streaming" ) } diff --git a/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceSchemaEvolutionSuite.scala b/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceSchemaEvolutionSuite.scala index efea9fc213a..680f89cd5ca 100644 --- a/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceSchemaEvolutionSuite.scala +++ b/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceSchemaEvolutionSuite.scala @@ -26,12 +26,8 @@ import org.apache.spark.sql.delta.{ } /** - * Test suite that runs DeltaSourceSchemaEvolutionSuite using the V2 connector - * (V2_ENABLE_MODE=STRICT). - */ - -/** - * Base trait for V2 schema evolution streaming tests. + * Base trait for V2 schema evolution streaming tests, running + * DeltaSourceSchemaEvolutionSuite using the V2 connector (V2_ENABLE_MODE=STRICT). * Provides common overrides shared by all V2 schema evolution suites. */ trait DeltaV2SourceSchemaEvolutionSuiteBase extends V2ForceTest { diff --git a/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceSuite.scala b/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceSuite.scala index 159f3c17962..4dd525a8028 100644 --- a/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceSuite.scala +++ b/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceSuite.scala @@ -153,6 +153,7 @@ object DeltaV2SourceSuite { "can delete old files of a snapshot without update", "Delta source advances with non-data inserts and generates empty dataframe for " + "non-data operations", + "make sure that the delta sources works fine", "reading from table with multiple partition columns succeeds during restart", "streaming read returns correct data from table with partition column in middle", "streaming read with column pruning and partition column in middle", @@ -175,6 +176,9 @@ object DeltaV2SourceSuite { // schema without rebuilding the DataFrame. "relax nullability: restarting with stale DataFrame should recover", "type widening: restarting with stale DataFrame should recover", + // V2 pins the schema when the DataFrame is loaded, so restarting the same DataFrame after the + // nullability change fails instead of adopting the current table schema. + "handling nullability schema changes", // === Data Loss Detection === // V2 only tolerates missing start versions with failOnDataLoss=false; mid-log gaps still @@ -194,6 +198,12 @@ object DeltaV2SourceSuite { "fail on missing trailing commit - empty batch from startIndex >= endIndex is not a" + " false positive readChangeFeed=false", + // === Commit-level dataChange is invisible to the V2 read path === + // Kernel's CommitInfo schema has no dataChange field, so the V2 path keeps deciding from the + // file actions (see StreamingHelper.getAddFileWithDataChange). The test makes the two disagree + // and expects the commit-level value to win, which it cannot here. + "a commit recording dataChange = false does not break an append-only stream", + // === Misc === // TODO(#5900): fix exception mismatch "no schema should throw an exception", @@ -202,9 +212,7 @@ object DeltaV2SourceSuite { // === Tests that bypass V2 by not using loadStreamWithOptions === "disallow user specified schema", // Uses .schema() directly - "make sure that the delta sources works fine", // Uses .delta() directly "self union a Delta table should pass the catalog table assert", // Uses .table() directly - "handling nullability schema changes", // Uses .table() directly "allow user specified schema if consistent: v1 source", // Uses DataSource directly // Calls deltaSource.createSource() directly "createSource should create source with empty or matching table schema provided" diff --git a/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceTableAPISuite.scala b/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceTableAPISuite.scala new file mode 100644 index 00000000000..feb0997d57c --- /dev/null +++ b/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceTableAPISuite.scala @@ -0,0 +1,53 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.test + +import org.apache.spark.sql.delta.DeltaSourceTableAPISuite + +/** Runs the streaming source tests from [[DeltaSourceTableAPISuite]] through the V2 connector. */ +class DeltaV2SourceTableAPISuite extends DeltaSourceTableAPISuite with V2ForceTest { + + override protected def assertNoV1Fallback: Boolean = true + + override protected def withV1Mode(f: => Unit): Unit = inV1Mode(f) + + override protected def shouldPassTests: Set[String] = + DeltaV2SourceTableAPISuite.PassingTests + + override protected def shouldFailTests: Set[String] = + DeltaV2SourceTableAPISuite.NonSourceTests +} + +object DeltaV2SourceTableAPISuite { + val PassingTests: Set[String] = Set( + "table API", + "table API with database" + ) + + // These inherited cases test writeStream.toTable. They do not cover the streaming source. + val NonSourceTests: Set[String] = Set( + "writeStream.table - create new external table", + "writeStream.table - create new managed table", + "writeStream.table - create new managed table with database", + "writeStream.table - create table from existing output", + "writeStream.table - fail writing into a view", + "writeStream.table - fail due to different schema than existing Delta table", + "writeStream.table - fail due to different partitioning on existing Delta table", + "writeStream.table - fail writing into an external nonDelta table", + "writeStream.table - fail writing into an external nonDelta path" + ) +} diff --git a/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/V1V2SourceSchemaLogCompatibilitySuite.scala b/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/V1V2SourceSchemaLogCompatibilitySuite.scala index 3c37894bd52..7720330101c 100644 --- a/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/V1V2SourceSchemaLogCompatibilitySuite.scala +++ b/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/V1V2SourceSchemaLogCompatibilitySuite.scala @@ -46,7 +46,7 @@ trait V1V2SourceSchemaLogCompatibilitySuiteBase override protected def isCdcTest: Boolean = false override protected def runOnlyTests: Seq[String] = Seq( - // Scenario tests — multi-step flows where each connector picks up what the other left. + // Scenario tests - multi-step flows where each connector picks up what the other left. "alternating connectors with no schema evolution leaves the schema log untouched", "V1-initialized schema log can be read by V2", "V2-initialized schema log can be read by V1", @@ -59,7 +59,7 @@ trait V1V2SourceSchemaLogCompatibilitySuiteBase "alternating connectors (V2-V1-V2-V1) across additive then non-additive evolution (rename)", "alternating connectors (V2-V1-V2-V1) across additive then non-additive evolution (drop)", - // Equivalence tests — V1 and V2 produce/consume byte-identical log files for the same input. + // Equivalence tests - V1 and V2 produce/consume byte-identical log files for the same input. "V1 and V2 write equivalent schema tracking log entries (rename)", "V1 and V2 write equivalent schema tracking log entries (drop)", "V1 and V2 mergers produce equivalent merged schema log entries", diff --git a/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/V2ForceTest.scala b/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/V2ForceTest.scala index b5a1cd2c4e3..4e95144ad9a 100644 --- a/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/V2ForceTest.scala +++ b/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/V2ForceTest.scala @@ -126,8 +126,4 @@ trait V2ForceTest extends DeltaSQLCommandTest with AdaptiveSparkPlanHelper { /** Run a SQL statement through the V1 connector. */ protected def executeInV1Mode(sqlText: String): Unit = inV1Mode(sql(sqlText)) - - override def afterAll(): Unit = { - super.afterAll() - } } diff --git a/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/typewidening/TypeWideningStreamingV2SourceSuite.scala b/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/typewidening/TypeWideningStreamingV2SourceSuite.scala index 9e97ffa75d9..8c7c56d65fc 100644 --- a/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/typewidening/TypeWideningStreamingV2SourceSuite.scala +++ b/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/typewidening/TypeWideningStreamingV2SourceSuite.scala @@ -28,7 +28,7 @@ import org.apache.spark.sql.delta.typewidening.{ * * The base lists every test from `TypeWideningStreamingSourceTests` as passing: V2 supports * type-widening streaming reads. Subclasses move specific tests back to `shouldFailTests` when - * there is a concrete V2 limitation (e.g., partition-column schema bug, missing event logging). + * there is a concrete V2 limitation, such as missing event logging. */ trait TypeWideningStreamingV2SourceSuiteBase extends V2ForceTest { self: TypeWideningStreamingSourceTestMixin => diff --git a/spark-unified/src/test/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2OptimisticTransactionSuite.scala b/spark-unified/src/test/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2OptimisticTransactionSuite.scala index a6811d84947..ff3063090ce 100644 --- a/spark-unified/src/test/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2OptimisticTransactionSuite.scala +++ b/spark-unified/src/test/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2OptimisticTransactionSuite.scala @@ -17,40 +17,103 @@ package org.apache.spark.sql.delta.v2.interop import java.io.File +import java.nio.file.{Files, StandardCopyOption} -import org.apache.spark.sql.delta.DeltaLog -import org.apache.spark.sql.delta.test.DeltaSQLCommandTest -import io.delta.kernel.Table -import io.delta.kernel.defaults.engine.DefaultEngine -import io.delta.kernel.internal.SnapshotImpl +import org.apache.spark.sql.delta.{DeltaColumnMapping, DeltaOperations, OptimisticTransaction, OptimisticTransactionSuite} +import org.apache.spark.sql.delta.actions.{AddFile, Metadata, Protocol, SetTransaction} +import org.apache.spark.sql.delta.test.V2ForceTest +import io.delta.spark.internal.v2.kernel.KernelEngineFactory +import org.apache.hadoop.fs.Path +import io.delta.kernel.{Table => KernelTable} +import io.delta.kernel.internal.{SnapshotImpl => KernelSnapshotImpl} -import org.apache.spark.sql.QueryTest -import org.apache.spark.sql.test.SharedSparkSession +import org.apache.spark.sql.types.{LongType, StructType} import org.apache.spark.util.SystemClock /** - * Tests for the [[DeltaV2OptimisticTransaction]] skeleton. This suite exercises the construction - * path only: a transaction can be built over a Kernel snapshot (with a null V1 `deltaLog`) and its - * read state (`readVersion`, `metadata`, `protocol`, `snapshot`) resolves from the wrapped - * [[DeltaV2Snapshot]]. The commit lifecycle is not yet implemented and must fail loudly. + * Tests for [[DeltaV2OptimisticTransaction]]. Extends [[OptimisticTransactionSuite]] and forces the + * V2 connector via [[V2ForceTest]]; inherited cases not yet supported on the V2 path are listed in + * [[shouldFailTests]]. Also covers construction, AddFile, commits, stats, and retry. */ class DeltaV2OptimisticTransactionSuite - extends QueryTest - with SharedSparkSession - with DeltaSQLCommandTest { + extends OptimisticTransactionSuite + with V2ForceTest { + + import testImplicits._ + + override protected def withTempDir(f: File => Unit): Unit = withTempDir(prefix = "spark")(f) + + /** + * Creates the transaction under test as a Kernel-backed [[DeltaV2OptimisticTransaction]] for the + * inherited V1 conflict test cases. + */ + override protected def startTestTransaction(dataPath: Path): OptimisticTransaction = { + // scalastyle:off deltahadoopconfiguration + val kernelEngine = KernelEngineFactory.createDefaultEngine(spark.sessionState.newHadoopConf()) + // scalastyle:on deltahadoopconfiguration + val kernelSnap = KernelTable + .forPath(kernelEngine, dataPath.toString) + .getLatestSnapshot(kernelEngine) + .asInstanceOf[KernelSnapshotImpl] + val deltaV2Snapshot = new DeltaV2Snapshot(kernelSnap) + new DeltaV2OptimisticTransaction(catalogTable = None, deltaV2Snapshot, kernelEngine) + } + + override protected def testDefaultMetadata(): Metadata = + Metadata(schemaString = new StructType().add("id", LongType).json) + override protected def testDefaultProtocol(): Protocol = Protocol(1, 2) + + override protected def shouldFailTests: Set[String] = Set( + // Conflict-checking: DelatV2 unsupported operations gaps + "conflicting txns - should conflict", + "disjoint txns - should not conflict", + "delete / delete - should conflict", + "upgrade / upgrade - should conflict", + // V2 write-path gaps that STRICT reroutes through the connector (deletion vectors): + "Duplicate action - remove file twice - same DV", + "Duplicate action - remove file twice - DV vs. no DV", + "Duplicate action - remove file twice - different DVs", + "Duplicate action - add file twice - same DV", + "Duplicate action - add file twice - DV vs. no DV", + "Duplicate action - add file twice - different DVs", + "Duplicate action - remove and add file - same DV", + "DVs cannot be added to files without numRecords stat", + // V2 write-path gaps that STRICT reroutes through the connector (SQL/DML/DDL, CLONE): + "preCommitLogSegment is updated during conflict checking", + "CommitInfo does not contain fully qualified column names", + "partition column changes not thrown for sql error on new path table", + "partition column changes not thrown for sql overwrite on new path table", + "partition column changes not thrown for sql append on path", + "partition column changes allowed for CLONE operations" + , + "partition column changes allowed for RenameColumn when partition column renamed" + ) + + override protected def shouldFail(testName: String): Boolean = shouldFailTests.contains(testName) /** Builds a Kernel-backed transaction over the latest snapshot of the table at `dir`. */ private def startKernelTxn(dir: File): DeltaV2OptimisticTransaction = { // scalastyle:off deltahadoopconfiguration // No DeltaLog here (the snapshot is loaded via Kernel), so use the session Hadoop conf. - val engine = DefaultEngine.create(spark.sessionState.newHadoopConf()) + val kernelEngine = KernelEngineFactory.createDefaultEngine(spark.sessionState.newHadoopConf()) // scalastyle:on deltahadoopconfiguration - val kernelSnap = Table - .forPath(engine, dir.getCanonicalPath) - .getLatestSnapshot(engine) - .asInstanceOf[SnapshotImpl] - val deltaV2Snapshot = new DeltaV2Snapshot(kernelSnap, spark) - new DeltaV2OptimisticTransaction(catalogTable = None, deltaV2Snapshot) + val kernelSnap = KernelTable + .forPath(kernelEngine, dir.getCanonicalPath) + .getLatestSnapshot(kernelEngine) + .asInstanceOf[KernelSnapshotImpl] + val deltaV2Snapshot = new DeltaV2Snapshot(kernelSnap) + new DeltaV2OptimisticTransaction(catalogTable = None, deltaV2Snapshot, kernelEngine) + } + + private def latestKernelSnapshot(dir: File): DeltaV2Snapshot = { + // scalastyle:off deltahadoopconfiguration + val kernelEngine = KernelEngineFactory.createDefaultEngine(spark.sessionState.newHadoopConf()) + // scalastyle:on deltahadoopconfiguration + val kernelSnap = KernelTable + .forPath(kernelEngine, dir.getCanonicalPath) + .getLatestSnapshot(kernelEngine) + .asInstanceOf[KernelSnapshotImpl] + new DeltaV2Snapshot(kernelSnap) } /** Seeds a simple (unpartitioned) V1 Delta table at `dir`. */ @@ -70,21 +133,25 @@ class DeltaV2OptimisticTransactionSuite } } - test("metadata and protocol resolve from the Kernel snapshot and match V1") { + test("metadata and protocol resolve from the Kernel snapshot") { withTempDir { dir => seedTable(dir) val txn = startKernelTxn(dir) - val log = DeltaLog.forTable(spark, dir.getCanonicalPath) - val v1 = log.update() - assert(txn.metadata.id === v1.metadata.id) - assert(txn.metadata.schemaString === v1.metadata.schemaString) - assert(txn.metadata.partitionColumns === v1.metadata.partitionColumns) - assert(txn.protocol === v1.protocol) + // Read state reflects the seeded table: a single LONG `id` column, unpartitioned. + assert(txn.metadata.schema.fieldNames.toSeq === Seq("id")) + assert(txn.metadata.schema("id").dataType === LongType) + assert(txn.metadata.partitionColumns.isEmpty) + assert(txn.metadata.id.nonEmpty) + + // A basic table resolves a protocol with at least the minimum supported versions. + assert(txn.protocol.minReaderVersion >= 1) + assert(txn.protocol.minWriterVersion >= 2) - // The Kernel-sourced path overrides match the V1 DeltaLog's paths. - assert(txn.dataPath === log.dataPath) - assert(txn.logPath === log.logPath) + // Kernel-sourced path overrides point at the table directory and its `_delta_log` child. + assert(txn.dataPath.getName === dir.getName) + assert(txn.logPath.getName === "_delta_log") + assert(txn.logPath.getParent.getName === dir.getName) } } @@ -112,25 +179,296 @@ class DeltaV2OptimisticTransactionSuite .write.format("delta").partitionBy("p").save(dir.getCanonicalPath) val txn = startKernelTxn(dir) - assert(txn.snapshot eq txn.deltaV2Snapshot) - assert(txn.deltaV2Snapshot.isInstanceOf[DeltaV2Snapshot]) + assert(txn.snapshot.isInstanceOf[DeltaV2Snapshot]) assert(txn.metadata.partitionColumns === Seq("p")) } } - test("commit path is not implemented and fails loudly") { + test("blind append of a synthetic AddFile commits through Kernel") { withTempDir { dir => seedTable(dir) val txn = startKernelTxn(dir) - // Call the commit chokepoint directly: going through txn.commit(...) would first reach the - // V1 prepareCommit machinery, which NPEs on the null deltaLog before this seam. + assert(txn.readVersion === 0L) + + val add = AddFile( + path = "synthetic-file", + partitionValues = Map.empty, + size = 1L, + modificationTime = 1L, + dataChange = true) + txn.commit(add :: Nil, DeltaOperations.ManualUpdate) + + val post = latestKernelSnapshot(dir) + assert(post.version === 1L) + assert(post.allFiles.collect().map(_.path).contains("synthetic-file")) + } + } + + test("appended parquet file is readable end to end") { + withTempDir { dir => + seedTable(dir) + + // Stage a real parquet file with the table's schema in a sibling directory (writing + // format("parquet") under the table root trips DELTA_INVALID_FORMAT validation), then copy + // it into the table dir, mirroring how OptimisticTransactionSuite drives commits at the + // file-action level. + val staging = new File(dir.getParentFile, s"${dir.getName}-staging") + spark.range(5, 7).toDF("id").coalesce(1) + .write.parquet(staging.getCanonicalPath) + val parquetFile = staging.listFiles() + .filter(f => f.getName.endsWith(".parquet") && !f.getName.startsWith("_")) + .head + val targetName = s"part-kernel-poc-${parquetFile.getName}" + val target = new File(dir, targetName) + Files.copy(parquetFile.toPath, target.toPath, StandardCopyOption.REPLACE_EXISTING) + + val txn = startKernelTxn(dir) + val add = AddFile( + path = targetName, + partitionValues = Map.empty, + size = target.length(), + modificationTime = target.lastModified(), + dataChange = true) + txn.commit(add :: Nil, DeltaOperations.ManualUpdate) + + checkAnswer( + spark.read.format("delta").load(dir.getCanonicalPath), + Seq(0L, 1L, 2L, 3L, 4L, 5L, 6L).toDF("id")) + } + } + + test("consecutive kernel commits advance the version") { + withTempDir { dir => + seedTable(dir) + (1 to 3).foreach { i => + val txn = startKernelTxn(dir) + assert(txn.readVersion === (i - 1).toLong) + val add = AddFile(s"synthetic-$i", Map.empty, 1L, 1L, dataChange = true) + txn.commit(add :: Nil, DeltaOperations.ManualUpdate) + } + val post = latestKernelSnapshot(dir) + assert(post.version === 3L) + assert(post.allFiles.collect().map(_.path).count(_.startsWith("synthetic-")) === 3) + } + } + + test("unsupported actions fail loudly") { + withTempDir { dir => + seedTable(dir) + val txn = startKernelTxn(dir) + val setTxn = SetTransaction("test-app", 1L, Some(1L)) val e = intercept[UnsupportedOperationException] { - txn.writeCommitFile( - attemptVersion = txn.readVersion + 1L, - jsonActions = Iterator.empty, - currentTransactionInfo = null) + txn.commit(setTxn :: Nil, DeltaOperations.ManualUpdate) + } + assert(e.getMessage == + "DeltaV2 unsupported operation: cannot commit action SetTransaction") + } + } + + /** + * Commits a synthetic [[AddFile]] carrying `statsJson` through Kernel. Used by the stats tests + * below to drive `kernelStatistics` with a chosen stats payload. + */ + private def commitWithStats(dir: File, statsJson: String): Unit = { + val txn = startKernelTxn(dir) + val add = AddFile( + path = "synthetic-stats-file", + partitionValues = Map.empty, + size = 1L, + modificationTime = 1L, + dataChange = true, + stats = statsJson) + txn.commit(add :: Nil, DeltaOperations.ManualUpdate) + } + + test("full stats JSON round-trips into the kernel add action") { + withTempDir { dir => + seedTable(dir) + commitWithStats( + dir, + """{"numRecords":3,"minValues":{"id":1},"maxValues":{"id":9},"nullCount":{"id":0}}""") + + val post = latestKernelSnapshot(dir) + assert(post.version === 1L) + val committed = post.allFiles.collect().find(_.path == "synthetic-stats-file").get + // Only numRecords is asserted: the native (JNR) FFI add-file path (ActionRowConverter) + // intentionally carries numRecords only, so min/max/nullCount are not populated. + assert(committed.stats.contains("\"numRecords\":3")) + } + } + + test("stats-less AddFile commits with empty stats rather than failing") { + withTempDir { dir => + seedTable(dir) + // stats defaults to null on AddFile; absent stats are legitimate and must not error. + commitWithStats(dir, statsJson = null) + assert(latestKernelSnapshot(dir).version === 1L) + } + } + + test("numRecords-only stats JSON commits (no min/max to parse)") { + withTempDir { dir => + seedTable(dir) + commitWithStats(dir, """{"numRecords":7}""") + + val post = latestKernelSnapshot(dir) + val committed = post.allFiles.collect().find(_.path == "synthetic-stats-file").get + assert(committed.stats.contains("\"numRecords\":7")) + } + } + + test("malformed stats JSON fails the commit instead of silently dropping stats") { + withTempDir { dir => + seedTable(dir) + // Truncated JSON: the previous implementation swallowed this and committed + // numRecords-only (here, no stats at all), losing data-skipping stats silently. + val e = intercept[Exception] { + commitWithStats(dir, """{"numRecords":3,"minValues":{"id":""") } - assert(e.getMessage.contains("not implemented")) + assert( + e.getMessage != null && e.getMessage.contains("Failed to parse JSON"), + s"expected a JSON parse failure, got: ${e.getMessage}") + // The bad write did not land. + assert(latestKernelSnapshot(dir).version === 0L) + } + } + + test("stats JSON whose value type contradicts the schema fails the commit") { + withTempDir { dir => + seedTable(dir) + // `id` is a LONG; a string min value is a real stats/schema inconsistency. + val e = intercept[Exception] { + commitWithStats( + dir, + """{"numRecords":3,"minValues":{"id":"not-a-number"},"maxValues":{"id":9}}""") + } + assert(e.getMessage != null, "expected a typed-parse failure with a message") + assert(latestKernelSnapshot(dir).version === 0L) + } + } + + /** Checks the append with typed partition values. */ + private def checkTypedPartitionAppend( + partitionType: String, + partitionValueSql: String, + mappingMode: Option[String] = None): Unit = { + withTempDir { dir => + val path = dir.getCanonicalPath + val columnMappingProperty = mappingMode + .map(mode => s" TBLPROPERTIES ('delta.columnMapping.mode' = '$mode')") + .getOrElse("") + spark.sql( + s"""CREATE TABLE delta.`$path` (id LONG, p $partitionType) + |USING delta PARTITIONED BY (p)$columnMappingProperty""".stripMargin) + spark.sql(s"INSERT INTO delta.`$path` VALUES (1, $partitionValueSql)") + + val base = latestKernelSnapshot(dir) + val sample = base.allFiles.collect().head + val physicalPartitionName = DeltaColumnMapping.getPhysicalName(base.metadata.schema("p")) + assert(sample.partitionValues.keySet === Set(physicalPartitionName)) + if (mappingMode.isDefined) { + assert(physicalPartitionName.startsWith("col-")) + } + + val add = AddFile( + path = s"synthetic-${sample.path}", + partitionValues = sample.partitionValues, + size = 1L, + modificationTime = 1L, + dataChange = true) + startKernelTxn(dir).commit(add :: Nil, DeltaOperations.ManualUpdate) + + val post = latestKernelSnapshot(dir) + assert(post.version === base.version + 1) + val committedAdds = post.allFiles.collect().filter(_.path == add.path) + assert(committedAdds.length === 1) + assert(committedAdds.head.partitionValues === sample.partitionValues) + } + } + + test("append to int-partitioned table types the partition literal") { + checkTypedPartitionAppend("INT", "5") + } + + test("append to date-partitioned table types the partition literal") { + checkTypedPartitionAppend("DATE", "DATE'2024-03-11'") + } + + test("append to timestamp-partitioned table types the partition literal") { + checkTypedPartitionAppend("TIMESTAMP", "TIMESTAMP'2024-03-11 11:00:00.123456'") + } + + test("append to decimal-partitioned table types the partition literal") { + checkTypedPartitionAppend("DECIMAL(10,2)", "CAST(12.34 AS DECIMAL(10,2))") + } + + test("append to boolean-partitioned table types the partition literal") { + checkTypedPartitionAppend("BOOLEAN", "true") + } + + test("append with a null partition value types the null literal") { + checkTypedPartitionAppend("INT", "NULL") + } + + test("append to string-partitioned table still works") { + checkTypedPartitionAppend("STRING", "'foo'") + } + + Seq("name", "id").foreach { mappingMode => + test(s"append with column mapping(mode: $mappingMode) is enabled") { + checkTypedPartitionAppend("INT", "5", Some(mappingMode)) + } + } + + private def concurrentAppend( + dir: File, fileName: String, partitionValues: Map[String, String] = Map.empty): Unit = { + val txn = startKernelTxn(dir) + txn.commit( + AddFile(fileName, partitionValues, 1L, 1L, dataChange = true) :: Nil, + DeltaOperations.ManualUpdate) + } + + test("multiple AddFiles per partition all commit through Kernel") { + withTempDir { dir => + val path = dir.getCanonicalPath + spark.sql( + s"""CREATE TABLE delta.`$path` (id LONG, p INT) + |USING delta PARTITIONED BY (p)""".stripMargin) // version 0 + + concurrentAppend(dir, "p=0/seed", Map("p" -> "0")) // version 1 + concurrentAppend(dir, "p=1/seed", Map("p" -> "1")) // version 2 + + val base = latestKernelSnapshot(dir) + val baseVersion = base.version + // partitionValues keyed by partition -> one representative file per partition. + val samplesByPartition = + base.allFiles.collect().groupBy(_.partitionValues).map { + case (partitionValues, files) => partitionValues -> files.head + } + assert(samplesByPartition.size === 2, "expected one file per partition to reuse") + + // Stage two synthetic AddFiles per partition (four total) + val adds = samplesByPartition.toSeq.flatMap { case (partitionValues, sample) => + (1 to 2).map { i => + AddFile( + path = s"synthetic-p${partitionValues.values.head}-$i-${sample.path}", + partitionValues = partitionValues, + size = 1L, + modificationTime = 1L, + dataChange = true) + } + } + assert(adds.size === 4) + + val txn = startKernelTxn(dir) + txn.commit(adds.toList, DeltaOperations.ManualUpdate) + + val post = latestKernelSnapshot(dir) + assert(post.version === baseVersion + 1) + val committedPaths = post.allFiles.collect().map(_.path) + adds.foreach(add => assert(committedPaths.contains(add.path), s"missing ${add.path}")) + assert(committedPaths.count(_.startsWith("synthetic-p0")) === 2) + assert(committedPaths.count(_.startsWith("synthetic-p1")) === 2) } } } diff --git a/spark-unified/src/test/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2QueryContextSuite.scala b/spark-unified/src/test/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2QueryContextSuite.scala new file mode 100644 index 00000000000..4323555b368 --- /dev/null +++ b/spark-unified/src/test/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2QueryContextSuite.scala @@ -0,0 +1,49 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.v2.interop + +import java.util.Optional + +import org.apache.spark.SparkFunSuite +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable, CatalogTableType} +import org.apache.spark.sql.types.StructType + +class DeltaV2QueryContextSuite extends SparkFunSuite { + + test("Java catalog metadata is preserved by identity") { + val catalogTable = CatalogTable( + identifier = TableIdentifier("test_table"), + tableType = CatalogTableType.EXTERNAL, + storage = CatalogStorageFormat.empty, + schema = StructType(Nil)) + + val queryContext = DeltaV2QueryContext(Optional.of(catalogTable)) + + assert(queryContext.catalogTableOpt.exists(_ eq catalogTable)) + assert(DeltaV2QueryContext(Optional.empty[CatalogTable]()).catalogTableOpt.isEmpty) + } + + test("null context inputs are rejected") { + intercept[NullPointerException] { + DeltaV2QueryContext(null: Option[CatalogTable]) + } + intercept[NullPointerException] { + DeltaV2QueryContext(null: Optional[CatalogTable]) + } + } +} diff --git a/spark-unified/src/test/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2SnapshotConversionsUtilsSuite.scala b/spark-unified/src/test/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2SnapshotConversionsUtilsSuite.scala deleted file mode 100644 index 47af58f8963..00000000000 --- a/spark-unified/src/test/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2SnapshotConversionsUtilsSuite.scala +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Copyright (2026) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.delta.v2.interop - -import java.util.Optional - -import scala.collection.JavaConverters._ - -import io.delta.kernel.internal.actions.{Format => KernelFormat} -import io.delta.kernel.internal.actions.{Metadata => KernelMetadata} -import io.delta.kernel.internal.actions.{Protocol => KernelProtocol} -import io.delta.kernel.internal.util.VectorUtils -import io.delta.kernel.types.{StringType, StructType} - -import org.apache.spark.SparkFunSuite - -/** - * Unit tests for [[DeltaV2SnapshotConversionsUtils]], which bridges Kernel's actions to V1 Delta - * actions. The tests build real Kernel [[KernelMetadata]] / [[KernelProtocol]] instances and assert - * the converted V1 Metadata / Protocol preserves every field. - */ -class DeltaV2SnapshotConversionsUtilsSuite extends SparkFunSuite { - - private val emptySchemaString = """{"type":"struct","fields":[]}""" - - test("metadataFromKernel preserves all populated fields") { - val kernelMetadata = new KernelMetadata( - "table-id", - Optional.of("table-name"), - Optional.of("table-description"), - new KernelFormat("parquet", Map("compression" -> "snappy").asJava), - emptySchemaString, - new StructType(), - VectorUtils.buildArrayValue(Seq("Part2", "part1").asJava, StringType.STRING), - Optional.of(java.lang.Long.valueOf(12345L)), - VectorUtils.stringStringMapValue(Map("delta.appendOnly" -> "true").asJava)) - - val metadata = DeltaV2SnapshotConversionsUtils.metadataFromKernel(kernelMetadata) - - assert(metadata.id === "table-id") - assert(metadata.name === "table-name") - assert(metadata.description === "table-description") - assert(metadata.format.provider === "parquet") - assert(metadata.format.options === Map("compression" -> "snappy")) - assert(metadata.schemaString === emptySchemaString) - // Order and case of partition columns must round-trip unchanged. - assert(metadata.partitionColumns === Seq("Part2", "part1")) - assert(metadata.configuration === Map("delta.appendOnly" -> "true")) - assert(metadata.createdTime === Some(12345L)) - } - - test("metadataFromKernel maps absent optionals and empty collections") { - val kernelMetadata = new KernelMetadata( - "table-id", - Optional.empty[String](), - Optional.empty[String](), - new KernelFormat("parquet", Map.empty[String, String].asJava), - emptySchemaString, - new StructType(), - VectorUtils.buildArrayValue(Seq.empty[String].asJava, StringType.STRING), - Optional.empty[java.lang.Long](), - VectorUtils.stringStringMapValue(Map.empty[String, String].asJava)) - - val metadata = DeltaV2SnapshotConversionsUtils.metadataFromKernel(kernelMetadata) - - assert(metadata.name === null) - assert(metadata.description === null) - assert(metadata.format.provider === "parquet") - assert(metadata.format.options.isEmpty) - assert(metadata.partitionColumns.isEmpty) - assert(metadata.configuration.isEmpty) - assert(metadata.createdTime === None) - } - - test("protocolFromKernel converts a table-features protocol with reader/writer features") { - val kernelProtocol = new KernelProtocol( - 3, - 7, - Set("v2Checkpoint").asJava, - Set("appendOnly", "invariants").asJava) - - val protocol = DeltaV2SnapshotConversionsUtils.protocolFromKernel(kernelProtocol) - - assert(protocol.minReaderVersion === 3) - assert(protocol.minWriterVersion === 7) - assert(protocol.readerFeatures === Some(Set("v2Checkpoint"))) - assert(protocol.writerFeatures === Some(Set("appendOnly", "invariants"))) - } - - test("protocolFromKernel maps a legacy (1, 2) protocol with no table features") { - // A legacy protocol does not support table features (reader < 3, writer < 7), so the - // converted V1 Protocol must carry no feature sets at all. - val kernelProtocol = new KernelProtocol(1, 2) - - val protocol = DeltaV2SnapshotConversionsUtils.protocolFromKernel(kernelProtocol) - - assert(protocol.minReaderVersion === 1) - assert(protocol.minWriterVersion === 2) - assert(protocol.readerFeatures === None) - assert(protocol.writerFeatures === None) - } - - test("protocolFromKernel maps a writer-only (1, 7) table-features protocol") { - // Writer version 7 supports writer features while the legacy reader version 1 does not, so - // only writerFeatures is populated and readerFeatures stays None. - val kernelProtocol = new KernelProtocol( - 1, - 7, - Set.empty[String].asJava, - Set("appendOnly", "invariants").asJava) - - val protocol = DeltaV2SnapshotConversionsUtils.protocolFromKernel(kernelProtocol) - - assert(protocol.minReaderVersion === 1) - assert(protocol.minWriterVersion === 7) - assert(protocol.readerFeatures === None) - assert(protocol.writerFeatures === Some(Set("appendOnly", "invariants"))) - } -} diff --git a/spark-unified/src/test/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2SnapshotManagerSuite.scala b/spark-unified/src/test/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2SnapshotManagerSuite.scala new file mode 100644 index 00000000000..1e5c88d9cc4 --- /dev/null +++ b/spark-unified/src/test/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2SnapshotManagerSuite.scala @@ -0,0 +1,118 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.v2.interop + +import java.lang.{Long => JLong} +import java.util.Optional + +import org.apache.spark.sql.delta.Snapshot +import io.delta.kernel.{CommitRange => KernelCommitRange} +import io.delta.kernel.engine.{Engine => KernelEngine} +import io.delta.kernel.internal.{ + DeltaHistoryManager => KernelDeltaHistoryManager +} + +import org.apache.spark.SparkFunSuite + +class DeltaV2SnapshotManagerSuite extends SparkFunSuite { + + test("query-context methods forward to context-free implementations by default") { + var forwardedCalls = 0 + + val manager = new DeltaV2SnapshotManager { + override def loadLatestSnapshot(): Snapshot = { + forwardedCalls += 1 + null + } + + override def loadSnapshotAt(version: Long): Snapshot = { + assert(version == 17) + forwardedCalls += 1 + null + } + + override def getActiveCommitAtTime( + timestampMillis: Long, + canReturnLastCommit: Boolean, + mustBeRecreatable: Boolean, + canReturnEarliestCommit: Boolean): KernelDeltaHistoryManager.Commit = { + assert(timestampMillis == 23) + assert(canReturnLastCommit) + assert(mustBeRecreatable) + assert(!canReturnEarliestCommit) + forwardedCalls += 1 + null + } + + override def checkVersionExists( + version: Long, + mustBeRecreatable: Boolean, + allowOutOfRange: Boolean): Unit = { + assert(version == 29) + assert(mustBeRecreatable) + assert(!allowOutOfRange) + forwardedCalls += 1 + } + + override def getTableChanges( + kernelEngine: KernelEngine, + startVersion: Long, + endVersion: Optional[JLong]): KernelCommitRange = { + assert(kernelEngine == null) + assert(startVersion == 31) + assert(endVersion == Optional.of[JLong](37L)) + forwardedCalls += 1 + null + } + } + + val queryContext = DeltaV2QueryContext.empty + + assert(manager.loadLatestSnapshot(queryContext) == null) + assert(manager.loadSnapshotAt(17, queryContext) == null) + assert( + manager.getActiveCommitAtTime( + 23, + canReturnLastCommit = true, + mustBeRecreatable = true, + canReturnEarliestCommit = false, + queryContext = queryContext) == null) + manager.checkVersionExists( + 29, + mustBeRecreatable = true, + allowOutOfRange = false, + queryContext = queryContext) + assert( + manager.getTableChanges( + null, + 31, + Optional.of[JLong](37L), + queryContext) == null) + + def assertNullQueryContext(body: => Any): Unit = { + val error = intercept[NullPointerException](body) + assert(error.getMessage == "queryContext is null") + } + + assertNullQueryContext(manager.loadLatestSnapshot(null)) + assertNullQueryContext(manager.loadSnapshotAt(41, null)) + assertNullQueryContext(manager.getActiveCommitAtTime(43, true, false, true, null)) + assertNullQueryContext(manager.checkVersionExists(47, false, true, null)) + assertNullQueryContext(manager.getTableChanges(null, 53, Optional.empty(), null)) + assert(forwardedCalls == 5) + } +} diff --git a/spark-unified/src/test/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2SnapshotSuite.scala b/spark-unified/src/test/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2SnapshotSuite.scala index 659629eb36d..72039e14338 100644 --- a/spark-unified/src/test/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2SnapshotSuite.scala +++ b/spark-unified/src/test/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2SnapshotSuite.scala @@ -16,14 +16,20 @@ package org.apache.spark.sql.delta.v2.interop -import org.apache.spark.sql.delta.{DeltaLog, Snapshot} +import org.apache.spark.sql.delta.{DeltaLog, DeltaOperations, NoMapping, Snapshot} +import org.apache.spark.sql.delta.actions.DomainMetadata +import org.apache.spark.sql.delta.sources.DeltaSQLConf +import org.apache.spark.sql.delta.stats.FileSizeHistogramUtils import org.apache.spark.sql.delta.test.DeltaSQLCommandTest +import io.delta.spark.internal.v2.kernel.KernelEngineFactory import org.apache.hadoop.fs.Path import io.delta.kernel.TableManager -import io.delta.kernel.defaults.engine.DefaultEngine import io.delta.kernel.engine.Engine import io.delta.kernel.internal.{SnapshotImpl => KernelSnapshot} +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable, CatalogTableType} + /** * Tests for [[DeltaV2Snapshot]] focusing on the construction surface that is wired to the * underlying Kernel snapshot: `path` / `version`, the data members read from Kernel @@ -34,7 +40,8 @@ class DeltaV2SnapshotSuite extends DeltaSQLCommandTest { // scalastyle:off deltahadoopconfiguration // No DeltaLog in this test (the snapshot is loaded via Kernel), so use the session Hadoop conf. - private def engine: Engine = DefaultEngine.create(spark.sessionState.newHadoopConf()) + private def engine: Engine = + KernelEngineFactory.createDefaultEngine(spark.sessionState.newHadoopConf()) // scalastyle:on deltahadoopconfiguration /** Load the latest snapshot of the Delta table at `path` via Kernel. */ @@ -47,7 +54,7 @@ class DeltaV2SnapshotSuite extends DeltaSQLCommandTest { spark.range(5).write.format("delta").save(path) // version 0 val kernelSnapshot0 = loadKernelSnapshot(path) - val snapshot0 = new DeltaV2Snapshot(kernelSnapshot0, spark) + val snapshot0 = new DeltaV2Snapshot(kernelSnapshot0) assert(snapshot0.version === kernelSnapshot0.getVersion) assert(snapshot0.version === 0L) assert(snapshot0.path === new Path(kernelSnapshot0.getDataPath.toString)) @@ -57,19 +64,36 @@ class DeltaV2SnapshotSuite extends DeltaSQLCommandTest { spark.range(1).write.format("delta").mode("append").save(path) // version 2 val kernelSnapshot2 = loadKernelSnapshot(path) - val snapshot2 = new DeltaV2Snapshot(kernelSnapshot2, spark) + val snapshot2 = new DeltaV2Snapshot(kernelSnapshot2) assert(snapshot2.version === 2L) assert(snapshot2.version === kernelSnapshot2.getVersion) } } + test("catalog table is preserved when provided") { + withTempDir { dir => + val path = dir.getCanonicalPath + spark.range(1).write.format("delta").save(path) + val kernelSnapshot = loadKernelSnapshot(path) + val catalogTable = CatalogTable( + identifier = TableIdentifier("test_table"), + tableType = CatalogTableType.EXTERNAL, + storage = CatalogStorageFormat.empty.copy(locationUri = Some(dir.toURI)), + schema = spark.range(0).schema) + + assert(new DeltaV2Snapshot(kernelSnapshot).catalogTable.isEmpty) + assert(new DeltaV2Snapshot(kernelSnapshot, Some(catalogTable)).catalogTable.contains( + catalogTable)) + } + } + /** * Differential check: every member the DeltaV2Snapshot sources from Kernel must match what the V1 * `DeltaLog.snapshot` reports for the same physical table. */ private def assertMatchesV1(path: String): Unit = { val v1 = DeltaLog.forTable(spark, path).update() - val v2 = new DeltaV2Snapshot(loadKernelSnapshot(path), spark) + val v2 = new DeltaV2Snapshot(loadKernelSnapshot(path)) assert(v2.version === v1.version) assert(v2.metadata.id === v1.metadata.id) @@ -83,12 +107,18 @@ class DeltaV2SnapshotSuite extends DeltaSQLCommandTest { assert(v2.dataPath === v1.dataPath) // With ICT disabled, both read the commit-file modification time for the same commit. assert(v2.timestamp === v1.timestamp) + // A Kernel-backed snapshot has no V1 DeltaLog from which to derive usage-log tags. + assert(v2.getCommonTags.isEmpty) // allFiles is read through Kernel but must describe the same physical files as V1: same paths, // sizes, and partition values (compared as a set -- ordering is not guaranteed). def fileKeys(s: Snapshot) = s.allFiles.collect().map(f => (f.path, f.size, f.partitionValues)).toSet assert(fileKeys(v2) === fileKeys(v1)) + + // Both bin the same file sizes with the same default bins, so the histograms agree. + assert(v2.sizeInBytes === v1.sizeInBytes) + assert(v2.fileSizeHistogram === v1.fileSizeHistogram) } /** @@ -150,14 +180,171 @@ class DeltaV2SnapshotSuite extends DeltaSQLCommandTest { val path = dir.getCanonicalPath spark.range(1).write.format("delta").save(path) - val snapshot = new DeltaV2Snapshot(loadKernelSnapshot(path), spark) + val snapshot = new DeltaV2Snapshot(loadKernelSnapshot(path)) // These still fail loudly so an unmigrated caller cannot silently read empty V1 state. intercept[UnsupportedOperationException](snapshot.stateDF) intercept[UnsupportedOperationException](snapshot.stateDS) intercept[UnsupportedOperationException](snapshot.tombstones) intercept[UnsupportedOperationException](snapshot.computeChecksum) + intercept[UnsupportedOperationException](snapshot.deltaFileIndexOpt) + intercept[UnsupportedOperationException](snapshot.checkpointProvider) + + assert(snapshot.deltaFileSizeInBytes() === -1L) + assert(snapshot.checkpointSizeInBytes() === -1L) + } + } + + test("domainMetadata is served correctly") { + withTempDir { dir => + val path = dir.getCanonicalPath + spark.sql( + s"CREATE TABLE delta.`$path` (id LONG) USING delta " + + "TBLPROPERTIES ('delta.feature.domainMetadata' = 'supported')") + + // Use DeltaLog to create a domain metadata easier for DeltaV2Snapshot read testing. + DeltaLog.forTable(spark, path).startTransaction().commit( + DomainMetadata("test.userDomain", """{"key":"value"}""", removed = false) :: Nil, + DeltaOperations.ManualUpdate) + + val snapshot = new DeltaV2Snapshot(loadKernelSnapshot(path)) + + val userDomain = snapshot.domainMetadata.find(_.domain == "test.userDomain") + .getOrElse(fail("expected the test.userDomain domain")) + assert(!userDomain.removed) + assert(userDomain.configuration === """{"key":"value"}""") + // Kernel serves domain metadata without V1 state reconstruction, so it is always "known". + assert(snapshot.domainMetadatasIfKnown.contains(snapshot.domainMetadata)) + } + } + + test("fileSizeHistogram is served from the checksum when one exists") { + withTempDir { dir => + val path = dir.getCanonicalPath + spark.range(0, 1000).write.format("delta").save(path) + spark.range(0, 5).write.format("delta").mode("append").save(path) + val kernelSnapshot = loadKernelSnapshot(path) + assert(kernelSnapshot.getCurrentCrcInfo.get().getFileSizeHistogram.isPresent) + val snapshot = new DeltaV2Snapshot(kernelSnapshot) + + assert(snapshot.fileSizeHistogram.nonEmpty) + assert( + snapshot.fileSizeHistogram.get.sortedBinBoundaries === + FileSizeHistogramUtils.DEFAULT_BINS) + assert( + snapshot.fileSizeHistogram.get.fileCounts.sum === snapshot.numOfFiles + ) + assert( + snapshot.fileSizeHistogram.get.totalBytes.sum === snapshot.sizeInBytes + ) + } + } + + test("fileSizeHistogram is None when the checksum has none") { + withTempDir { dir => + val path = dir.getCanonicalPath + withSQLConf(DeltaSQLConf.DELTA_WRITE_CHECKSUM_ENABLED.key -> "false") { + spark.range(0, 1000).write.format("delta").save(path) + } + val snapshot = new DeltaV2Snapshot(loadKernelSnapshot(path)) + assert(snapshot.fileSizeHistogram.isEmpty) + } + } + + test("sizeInBytes is served from the checksum when one exists") { + withTempDir { dir => + val path = dir.getCanonicalPath + spark.range(0, 1000).write.format("delta").save(path) + + val kernelSnapshot = loadKernelSnapshot(path) + assert(kernelSnapshot.getCurrentCrcInfo.isPresent) + + val snapshot = new DeltaV2Snapshot(kernelSnapshot) + assert(snapshot.sizeInBytesIfKnown.nonEmpty) + assert( + snapshot.sizeInBytes === snapshot.allFiles.collect().map(_.size).sum + ) + } + } + + test("sizeInBytes is -1 when the checksum has none") { + withTempDir { dir => + val path = dir.getCanonicalPath + withSQLConf(DeltaSQLConf.DELTA_WRITE_CHECKSUM_ENABLED.key -> "false") { + spark.range(0, 1000).write.format("delta").save(path) + } + + val snapshot = new DeltaV2Snapshot(loadKernelSnapshot(path)) + assert(snapshot.sizeInBytesIfKnown.isEmpty) + assert(snapshot.sizeInBytes === -1L) + } + } + + test("fileSizeHistogram is None when the histogram config is disabled") { + withTempDir { dir => + val path = dir.getCanonicalPath + spark.range(5).write.format("delta").save(path) + + withSQLConf(DeltaSQLConf.DELTA_FILE_SIZE_HISTOGRAM_ENABLED.key -> "false") { + assert(new DeltaV2Snapshot(loadKernelSnapshot(path)).fileSizeHistogram.isEmpty) + } + } + } + + test("domainMetadata is empty for a table with no domains") { + withTempDir { dir => + val path = dir.getCanonicalPath + spark.range(1).write.format("delta").save(path) + + val snapshot = new DeltaV2Snapshot(loadKernelSnapshot(path)) + assert(snapshot.domainMetadata.isEmpty) + } + } + + test("allFiles uses the active Spark session instead of storing a construction session") { + withTempDir { dir => + val path = dir.getCanonicalPath + spark.range(1).write.format("delta").save(path) + val snapshot = new DeltaV2Snapshot(loadKernelSnapshot(path)) + val activeSession = spark.newSession() + + activeSession.withActive { + assert(snapshot.allFiles.sparkSession eq activeSession) + assert(snapshot.allFiles.count() === 1L) + } + } + } + + test("getKernelSnapshot returns the wrapped Kernel snapshot") { + withTempDir { dir => + val path = dir.getCanonicalPath + spark.range(1).write.format("delta").save(path) + val kernelSnapshot = loadKernelSnapshot(path) + val snapshot = new DeltaV2Snapshot(kernelSnapshot) + + assert(DeltaV2Snapshot.getKernelSnapshot(snapshot) eq kernelSnapshot) + } + } + + test("getKernelSnapshot rejects a non-Kernel-backed V1 snapshot") { + withTempDir { dir => + val path = dir.getCanonicalPath + spark.range(1).write.format("delta").save(path) + val snapshot = DeltaLog.forTable(spark, path).update() + + val error = intercept[IllegalStateException] { + DeltaV2Snapshot.getKernelSnapshot(snapshot) + } + assert(error.getMessage.contains("Expected a DeltaV2Snapshot")) + assert(error.getMessage.contains(snapshot.getClass.getName)) + } + } + + test("getKernelSnapshot rejects null") { + val error = intercept[NullPointerException] { + DeltaV2Snapshot.getKernelSnapshot(null) } + assert(error.getMessage === "snapshot is null") } } diff --git a/spark-unified/src/test/scala/org/apache/spark/sql/delta/v2/interop/KernelSnapshotUtilsSuite.scala b/spark-unified/src/test/scala/org/apache/spark/sql/delta/v2/interop/KernelSnapshotUtilsSuite.scala index ef9ff553a89..634f6c7fe66 100644 --- a/spark-unified/src/test/scala/org/apache/spark/sql/delta/v2/interop/KernelSnapshotUtilsSuite.scala +++ b/spark-unified/src/test/scala/org/apache/spark/sql/delta/v2/interop/KernelSnapshotUtilsSuite.scala @@ -16,24 +16,43 @@ package org.apache.spark.sql.delta.v2.interop +import java.io.File +import java.nio.charset.StandardCharsets.UTF_8 +import java.nio.file.Files + import org.apache.spark.sql.delta.{DeltaConfigs, DeltaLog} import org.apache.spark.sql.delta.actions.AddFile import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest +import io.delta.spark.internal.v2.kernel.KernelEngineFactory import io.delta.kernel.TableManager -import io.delta.kernel.defaults.engine.DefaultEngine import io.delta.kernel.engine.Engine import io.delta.kernel.internal.SnapshotImpl class KernelSnapshotUtilsSuite extends DeltaSQLCommandTest { // scalastyle:off deltahadoopconfiguration - private def engine: Engine = DefaultEngine.create(spark.sessionState.newHadoopConf()) + private def engine: Engine = + KernelEngineFactory.createDefaultEngine(spark.sessionState.newHadoopConf()) // scalastyle:on deltahadoopconfiguration private def kernelSnapshotFor(path: String, engine: Engine): SnapshotImpl = TableManager.loadSnapshot(path).build(engine).asInstanceOf[SnapshotImpl] + private def appendTaggedAddFile(path: File, tags: Map[String, String]): String = { + val dataPath = "tagged.parquet" + val taggedAddFile = AddFile( + path = dataPath, + partitionValues = Map.empty, + size = 1L, + modificationTime = 2L, + dataChange = true, + tags = tags) + val commitFile = new File(path, "_delta_log/00000000000000000001.json") + Files.writeString(commitFile.toPath, taggedAddFile.json, UTF_8) + dataPath + } + private def allFilesFor(path: String): (Array[AddFile], Array[AddFile]) = { val kernelEngine = engine val kernelSnapshot = kernelSnapshotFor(path, kernelEngine) @@ -62,6 +81,7 @@ class KernelSnapshotUtilsSuite extends DeltaSQLCommandTest { assert(kernelFile.modificationTime === v1File.modificationTime) assert(kernelFile.dataChange === v1File.dataChange) assert(kernelFile.stats === v1File.stats) + assert(kernelFile.tags === v1File.tags) assert(kernelFile.deletionVector === v1File.deletionVector) assert(kernelFile.baseRowId === v1File.baseRowId) assert(kernelFile.defaultRowCommitVersion === v1File.defaultRowCommitVersion) @@ -137,4 +157,23 @@ class KernelSnapshotUtilsSuite extends DeltaSQLCommandTest { } } } + + test("buildAllFiles and filesForScan preserve AddFile tags from commit JSON") { + withTempDir { dir => + val path = dir.getAbsolutePath + spark.range(1).coalesce(1).write.format("delta").save(path) + val expectedTags = Map(AddFile.Tags.ZCUBE_ID.name -> "cube-id") + val taggedPath = appendTaggedAddFile(dir, expectedTags) + + val kernelEngine = engine + val kernelSnapshot = kernelSnapshotFor(path, kernelEngine) + val kernelFiles = + KernelSnapshotUtils.buildAllFiles(kernelSnapshot, spark, kernelEngine).collect() + assert(kernelFiles.find(_.path == taggedPath).map(_.tags).contains(expectedTags)) + + val snapshot = new DeltaV2Snapshot(kernelSnapshot) + val selectedFiles = snapshot.filesForScan(Nil).files + assert(selectedFiles.find(_.path == taggedPath).map(_.tags).contains(expectedTags)) + } + } } diff --git a/spark-unified/src/test/scala/org/apache/spark/sql/delta/v2/kernel/KernelActionUtilsSuite.scala b/spark-unified/src/test/scala/org/apache/spark/sql/delta/v2/kernel/KernelActionUtilsSuite.scala new file mode 100644 index 00000000000..7501d53b191 --- /dev/null +++ b/spark-unified/src/test/scala/org/apache/spark/sql/delta/v2/kernel/KernelActionUtilsSuite.scala @@ -0,0 +1,472 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.v2.kernel + +import java.util.Optional + +import scala.collection.JavaConverters._ + +import org.apache.spark.sql.delta.actions.{AddFile, DeletionVectorDescriptor, DomainMetadata} +import io.delta.kernel.{CommitActions => KernelCommitActions} +import io.delta.kernel.data.{ColumnarBatch => KernelColumnarBatch} +import io.delta.kernel.data.{ColumnVector => KernelColumnVector} +import io.delta.kernel.data.{Row => KernelRow} +import io.delta.kernel.data.MapValue +import io.delta.kernel.internal.DeltaLogActionUtils.{DeltaAction => KernelDeltaAction} +import io.delta.kernel.internal.actions.{AddFile => KernelAddFile} +import io.delta.kernel.internal.actions.{CommitInfo => KernelCommitInfo} +import io.delta.kernel.internal.actions.{ + DeletionVectorDescriptor => KernelDeletionVectorDescriptor +} +import io.delta.kernel.internal.actions.{DomainMetadata => KernelDomainMetadata} +import io.delta.kernel.internal.actions.{Format => KernelFormat} +import io.delta.kernel.internal.actions.{Metadata => KernelMetadata} +import io.delta.kernel.internal.actions.{Protocol => KernelProtocol} +import io.delta.kernel.internal.actions.{RemoveFile => KernelRemoveFile} +import io.delta.kernel.internal.actions.{SetTransaction => KernelSetTransaction} +import io.delta.kernel.internal.data.GenericColumnVector +import io.delta.kernel.internal.data.GenericRow +import io.delta.kernel.internal.util.{Utils => KernelUtils} +import io.delta.kernel.internal.util.VectorUtils +import io.delta.kernel.statistics.DataFileStatistics +import io.delta.kernel.types.{StringType, StructType} +import io.delta.kernel.utils.{CloseableIterator => KernelCloseableIterator} + +import org.apache.spark.SparkFunSuite + +class KernelActionUtilsSuite extends SparkFunSuite { + + private val emptySchemaString = """{"type":"struct","fields":[]}""" + + test("metadataFromKernel preserves all populated fields") { + val kernelMetadata = new KernelMetadata( + "table-id", + Optional.of("table-name"), + Optional.of("table-description"), + new KernelFormat("parquet", Map("compression" -> "snappy").asJava), + emptySchemaString, + new StructType(), + VectorUtils.buildArrayValue(Seq("Part2", "part1").asJava, StringType.STRING), + Optional.of(java.lang.Long.valueOf(12345L)), + VectorUtils.stringStringMapValue(Map("delta.appendOnly" -> "true").asJava)) + + val metadata = KernelActionUtils.metadataFromKernel(kernelMetadata) + + assert(metadata.id === "table-id") + assert(metadata.name === "table-name") + assert(metadata.description === "table-description") + assert(metadata.format.provider === "parquet") + assert(metadata.format.options === Map("compression" -> "snappy")) + assert(metadata.schemaString === emptySchemaString) + // Order and case of partition columns must round-trip unchanged. + assert(metadata.partitionColumns === Seq("Part2", "part1")) + assert(metadata.configuration === Map("delta.appendOnly" -> "true")) + assert(metadata.createdTime === Some(12345L)) + } + + test("metadataFromKernel maps absent optionals and empty collections") { + val kernelMetadata = new KernelMetadata( + "table-id", + Optional.empty[String](), + Optional.empty[String](), + new KernelFormat("parquet", Map.empty[String, String].asJava), + emptySchemaString, + new StructType(), + VectorUtils.buildArrayValue(Seq.empty[String].asJava, StringType.STRING), + Optional.empty[java.lang.Long](), + VectorUtils.stringStringMapValue(Map.empty[String, String].asJava)) + + val metadata = KernelActionUtils.metadataFromKernel(kernelMetadata) + + assert(metadata.name === null) + assert(metadata.description === null) + assert(metadata.format.provider === "parquet") + assert(metadata.format.options.isEmpty) + assert(metadata.partitionColumns.isEmpty) + assert(metadata.configuration.isEmpty) + assert(metadata.createdTime === None) + } + + test("protocolFromKernel converts a table-features protocol with reader/writer features") { + val kernelProtocol = new KernelProtocol( + 3, + 7, + Set("v2Checkpoint").asJava, + Set("appendOnly", "invariants").asJava) + + val protocol = KernelActionUtils.protocolFromKernel(kernelProtocol) + + assert(protocol.minReaderVersion === 3) + assert(protocol.minWriterVersion === 7) + assert(protocol.readerFeatures === Some(Set("v2Checkpoint"))) + assert(protocol.writerFeatures === Some(Set("appendOnly", "invariants"))) + } + + test("protocolFromKernel maps a legacy (1, 2) protocol with no table features") { + // A legacy protocol does not support table features (reader < 3, writer < 7), so the + // converted V1 Protocol must carry no feature sets at all. + val kernelProtocol = new KernelProtocol(1, 2) + + val protocol = KernelActionUtils.protocolFromKernel(kernelProtocol) + + assert(protocol.minReaderVersion === 1) + assert(protocol.minWriterVersion === 2) + assert(protocol.readerFeatures === None) + assert(protocol.writerFeatures === None) + } + + test("protocolFromKernel maps a writer-only (1, 7) table-features protocol") { + // Writer version 7 supports writer features while the legacy reader version 1 does not, so + // only writerFeatures is populated and readerFeatures stays None. + val kernelProtocol = new KernelProtocol( + 1, + 7, + Set.empty[String].asJava, + Set("appendOnly", "invariants").asJava) + + val protocol = KernelActionUtils.protocolFromKernel(kernelProtocol) + + assert(protocol.minReaderVersion === 1) + assert(protocol.minWriterVersion === 7) + assert(protocol.readerFeatures === None) + assert(protocol.writerFeatures === Some(Set("appendOnly", "invariants"))) + } + + test("commitInfoFromKernel preserves retry-critical fields") { + val commitInfo = KernelActionUtils.commitInfoFromKernel( + new KernelCommitInfo( + Optional.of(java.lang.Long.valueOf(12345L)), + 23456L, + Optional.of("kernel-engine"), + Optional.of("WRITE"), + Map("mode" -> "Append").asJava, + Optional.of(java.lang.Boolean.TRUE), + Optional.of("txn-123"), + Map("numFiles" -> "1").asJava)) + + assert(commitInfo.inCommitTimestamp === Some(12345L)) + assert(commitInfo.timestamp.getTime === 23456L) + assert(commitInfo.operation === "WRITE") + assert(commitInfo.operationParameters === Map("mode" -> "Append")) + assert(commitInfo.isBlindAppend === Some(true)) + assert(commitInfo.operationMetrics === Some(Map("numFiles" -> "1"))) + assert(commitInfo.engineInfo === Some("kernel-engine")) + assert(commitInfo.txnId === Some("txn-123")) + } + + private def optionalMapValue(values: Map[String, String]): Optional[MapValue] = { + if (values.isEmpty) Optional.empty[MapValue]() + else Optional.of(VectorUtils.stringStringMapValue(values.asJava)) + } + + private def addFileRow( + path: String, + partitionValues: Map[String, String], + size: Long, + modificationTime: Long, + dataChange: Boolean, + tags: Map[String, String] = Map.empty, + deletionVector: Optional[KernelDeletionVectorDescriptor] = Optional.empty(), + baseRowId: Optional[java.lang.Long] = Optional.empty(), + defaultRowCommitVersion: Optional[java.lang.Long] = Optional.empty()): KernelRow = + KernelAddFile.createAddFileRow( + KernelAddFile.SCHEMA_WITHOUT_STATS, + path, + VectorUtils.stringStringMapValue(partitionValues.asJava), + size, + modificationTime, + dataChange, + deletionVector, + optionalMapValue(tags), + baseRowId, + defaultRowCommitVersion, + Optional.empty[DataFileStatistics]()) + + private def kernelAddFile( + path: String, + partitionValues: Map[String, String], + size: Long, + modificationTime: Long, + dataChange: Boolean, + tags: Map[String, String] = Map.empty, + deletionVector: Optional[KernelDeletionVectorDescriptor] = Optional.empty(), + baseRowId: Optional[java.lang.Long] = Optional.empty(), + defaultRowCommitVersion: Optional[java.lang.Long] = Optional.empty()): KernelAddFile = + new KernelAddFile(addFileRow( + path, + partitionValues, + size, + modificationTime, + dataChange, + tags, + deletionVector, + baseRowId, + defaultRowCommitVersion)) + + private def buildKernelRemove( + path: String, + deletionTimestamp: Long, + dataChange: Boolean, + partitionValues: Map[String, String], + size: Long, + tags: Map[String, String] = Map.empty, + stats: String = null, + deletionVector: Optional[KernelDeletionVectorDescriptor] = Optional.empty(), + baseRowId: Optional[java.lang.Long] = Optional.empty(), + defaultRowCommitVersion: Optional[java.lang.Long] = Optional.empty()): KernelRemoveFile = { + // scalastyle:off removeFile // KernelRemoveFile is the Kernel wrapper, not a V1 RemoveFile + val fields = new java.util.HashMap[Integer, Object]() + def put(fieldName: String, value: Object): Unit = + fields.put(Int.box(KernelRemoveFile.FULL_SCHEMA.indexOf(fieldName)), value) + + put("path", path) + put("deletionTimestamp", java.lang.Long.valueOf(deletionTimestamp)) + put("dataChange", java.lang.Boolean.valueOf(dataChange)) + put("extendedFileMetadata", java.lang.Boolean.TRUE) + put("partitionValues", VectorUtils.stringStringMapValue(partitionValues.asJava)) + put("size", java.lang.Long.valueOf(size)) + if (stats != null) put("stats", stats) + optionalMapValue(tags).ifPresent(tags => put("tags", tags)) + deletionVector.ifPresent(dv => put("deletionVector", dv.toRow)) + baseRowId.ifPresent(rowId => put("baseRowId", rowId)) + defaultRowCommitVersion.ifPresent(version => put("defaultRowCommitVersion", version)) + + new KernelRemoveFile(new GenericRow(KernelRemoveFile.FULL_SCHEMA, fields)) + } + // scalastyle:on removeFile + + test("setTransactionFromKernel preserves all fields") { + val setTransaction = KernelActionUtils.setTransactionFromKernel( + new KernelSetTransaction( + "app-id", java.lang.Long.valueOf(7L), Optional.of(java.lang.Long.valueOf(12345L)))) + + assert(setTransaction.appId === "app-id") + assert(setTransaction.version === 7L) + assert(setTransaction.lastUpdated === Some(12345L)) + } + + test("setTransactionFromKernel maps an absent lastUpdated") { + val setTransaction = KernelActionUtils.setTransactionFromKernel( + new KernelSetTransaction("app-id", java.lang.Long.valueOf(7L), Optional.empty())) + + assert(setTransaction.appId === "app-id") + assert(setTransaction.version === 7L) + assert(setTransaction.lastUpdated === None) + } + + test("addFileFromKernel preserves all populated fields") { + val kernelDv = new KernelDeletionVectorDescriptor( + "u", "storage-path", Optional.of(java.lang.Integer.valueOf(10)), 128, 5L) + val addFile = KernelActionUtils.addFileFromKernel( + kernelAddFile( + path = "part-00000.parquet", + partitionValues = Map("p" -> "1", "q" -> "x"), + size = 4096L, + modificationTime = 111L, + dataChange = true, + tags = Map("tag-a" -> "value-a"), + deletionVector = Optional.of(kernelDv), + baseRowId = Optional.of(java.lang.Long.valueOf(42L)), + defaultRowCommitVersion = Optional.of(java.lang.Long.valueOf(7L)))) + + assert(addFile.path === "part-00000.parquet") + assert(addFile.partitionValues === Map("p" -> "1", "q" -> "x")) + assert(addFile.size === 4096L) + assert(addFile.modificationTime === 111L) + // dataChange must round-trip from the log (unlike the snapshot-scan path, which forces false). + assert(addFile.dataChange) + assert(addFile.tagsOrEmpty === Map("tag-a" -> "value-a")) + assert(addFile.baseRowId === Some(42L)) + assert(addFile.defaultRowCommitVersion === Some(7L)) + assert(addFile.deletionVector != null) + assert(addFile.deletionVector.storageType === "u") + assert(addFile.deletionVector.pathOrInlineDv === "storage-path") + assert(addFile.deletionVector.offset === Some(10)) + assert(addFile.deletionVector.sizeInBytes === 128) + assert(addFile.deletionVector.cardinality === 5L) + } + + test("toKernelScanAddFile round-trips fields supported by Kernel") { + val v1AddFile = AddFile( + path = "part-00000.parquet", + partitionValues = Map("p" -> "1", "q" -> "x"), + size = 4096L, + modificationTime = 111L, + dataChange = true, + stats = """{"numRecords":9}""", + tags = Map("tag-a" -> "value-a"), + deletionVector = DeletionVectorDescriptor( + storageType = "u", + pathOrInlineDv = "storage-path", + offset = Some(10), + sizeInBytes = 128, + cardinality = 5L), + baseRowId = Some(42L), + defaultRowCommitVersion = Some(7L)) + + val roundTripped = KernelActionUtils.addFileFromKernel( + KernelActionUtils.toKernelScanAddFile(v1AddFile)) + + assert(roundTripped.path === v1AddFile.path) + assert(roundTripped.partitionValues === v1AddFile.partitionValues) + assert(roundTripped.size === v1AddFile.size) + assert(roundTripped.modificationTime === v1AddFile.modificationTime) + assert(roundTripped.dataChange === v1AddFile.dataChange) + assert(roundTripped.tagsOrEmpty === v1AddFile.tagsOrEmpty) + assert(roundTripped.deletionVector === v1AddFile.deletionVector) + assert(roundTripped.baseRowId === v1AddFile.baseRowId) + assert(roundTripped.defaultRowCommitVersion === v1AddFile.defaultRowCommitVersion) + assert(roundTripped.stats === v1AddFile.stats) + } + + test("addFileFromKernel maps absent optionals and empty partition values") { + val addFile = KernelActionUtils.addFileFromKernel( + kernelAddFile("f.parquet", Map.empty, size = 1L, modificationTime = 1L, dataChange = false)) + + assert(addFile.partitionValues.isEmpty) + assert(!addFile.dataChange) + assert(addFile.deletionVector === null) + assert(addFile.baseRowId === None) + assert(addFile.defaultRowCommitVersion === None) + // No stats were provided (SCHEMA_WITHOUT_STATS), so the V1 stats string is null. + assert(addFile.stats === null) + } + + test("addFileFromKernel preserves a null partition value") { + val addFile = KernelActionUtils.addFileFromKernel( + kernelAddFile("f.parquet", Map("p" -> null), size = 1L, modificationTime = 1L, + dataChange = true)) + + assert(addFile.partitionValues === Map("p" -> null)) + } + + test("removeFileFromKernel preserves all populated fields") { + val kernelDv = new KernelDeletionVectorDescriptor( + "u", "storage-path", Optional.of(java.lang.Integer.valueOf(10)), 128, 5L) + val removeFile = KernelActionUtils.removeFileFromKernel( + buildKernelRemove( + path = "part-00000.parquet", + deletionTimestamp = 222L, + dataChange = true, + partitionValues = Map("p" -> "1"), + size = 4096L, + tags = Map("remove-tag" -> "remove-value"), + stats = """{"numRecords":9}""", + deletionVector = Optional.of(kernelDv), + baseRowId = Optional.of(java.lang.Long.valueOf(84L)), + defaultRowCommitVersion = Optional.of(java.lang.Long.valueOf(8L)))) + + assert(removeFile.path === "part-00000.parquet") + assert(removeFile.deletionTimestamp === Some(222L)) + assert(removeFile.dataChange) + assert(removeFile.extendedFileMetadata === Some(true)) + assert(removeFile.partitionValues === Map("p" -> "1")) + assert(removeFile.size === Some(4096L)) + assert(removeFile.tagsOrEmpty === Map("remove-tag" -> "remove-value")) + assert(removeFile.stats === """{"numRecords":9}""") + assert(removeFile.baseRowId === Some(84L)) + assert(removeFile.defaultRowCommitVersion === Some(8L)) + assert(removeFile.deletionVector != null) + assert(removeFile.deletionVector.storageType === "u") + assert(removeFile.deletionVector.cardinality === 5L) + } + + test("removeFileFromKernel maps an absent deletion vector") { + val removeFile = KernelActionUtils.removeFileFromKernel( + buildKernelRemove( + path = "f.parquet", + deletionTimestamp = 1L, + dataChange = false, + partitionValues = Map.empty, + size = 1L)) + + assert(removeFile.dataChange === false) + assert(removeFile.partitionValues.isEmpty) + assert(removeFile.deletionVector === null) + } + + /** Wraps a single action column into a one-batch [[KernelCommitActions]] for direct decoding. */ + private def singleColumnCommitActions( + colName: String, column: KernelColumnVector): KernelCommitActions = { + val batch = new KernelColumnarBatch { + override def getSchema: StructType = new StructType().add(colName, column.getDataType) + override def getColumnVector(ordinal: Int): KernelColumnVector = column + override def getSize: Int = column.getSize + } + new KernelCommitActions { + override def getVersion: Long = 1L + override def getTimestamp: Long = 0L + override def getActions(): KernelCloseableIterator[KernelColumnarBatch] = + KernelUtils.singletonCloseableIterator(batch) + override def close(): Unit = {} + } + } + + test("readActions decodes an AddFile action from a hand-built commit batch") { + val addColumn = new GenericColumnVector( + java.util.Collections.singletonList(addFileRow( + path = "part-00000.parquet", + partitionValues = Map("p" -> "1"), + size = 4096L, + modificationTime = 111L, + dataChange = true, + tags = Map("tag-a" -> "value-a"), + baseRowId = Optional.of(java.lang.Long.valueOf(42L)), + defaultRowCommitVersion = Optional.of(java.lang.Long.valueOf(7L)))), + KernelAddFile.SCHEMA_WITHOUT_STATS) + + val actions = KernelActionUtils.readActions(singleColumnCommitActions("add", addColumn)) + + val adds = actions.collect { case a: AddFile => a } + assert(adds.size === 1) + assert(adds.head.path === "part-00000.parquet") + assert(adds.head.partitionValues === Map("p" -> "1")) + assert(adds.head.size === 4096L) + assert(adds.head.dataChange) + assert(adds.head.baseRowId === Some(42L)) + assert(adds.head.defaultRowCommitVersion === Some(7L)) + } + + test("readActions decodes domain metadata and its removal flag") { + val configuration = """{"rowIdHighWaterMark":7}""" + val kernelActions = Seq( + new KernelDomainMetadata("delta.rowTracking", configuration, false), + new KernelDomainMetadata("test.userDomain", "{}", true)) + val domainColumn = new GenericColumnVector( + kernelActions.map(_.toRow()).asJava, + KernelDomainMetadata.FULL_SCHEMA) + + val actions = KernelActionUtils.readActions( + singleColumnCommitActions(KernelDeltaAction.DOMAINMETADATA.colName, domainColumn)) + + assert(actions === Seq( + DomainMetadata("delta.rowTracking", configuration, removed = false), + DomainMetadata("test.userDomain", "{}", removed = true))) + } + + test("readActions fails loud on an action type with no decoder (hand-built commit batch)") { + val presentColumn = new GenericColumnVector( + java.util.Collections.singletonList("present"), StringType.STRING) + val action = KernelDeltaAction.CDC + val e = intercept[UnsupportedOperationException] { + KernelActionUtils.readActions(singleColumnCommitActions(action.colName, presentColumn)) + } + assert(e.getMessage.contains("No V1 action from Kernel decoder")) + assert(e.getMessage.contains(action.colName)) + } + +} diff --git a/spark/delta-suite-generator/src/main/scala/io/delta/suitegenerator/SuiteGeneratorConfig.scala b/spark/delta-suite-generator/src/main/scala/io/delta/suitegenerator/SuiteGeneratorConfig.scala index 6b15c92e403..53dc05dc248 100644 --- a/spark/delta-suite-generator/src/main/scala/io/delta/suitegenerator/SuiteGeneratorConfig.scala +++ b/spark/delta-suite-generator/src/main/scala/io/delta/suitegenerator/SuiteGeneratorConfig.scala @@ -139,6 +139,7 @@ object SuiteGeneratorConfig { val ROW_TRACKING_ON = ROW_TRACKING.withValueAsDimension(_.last) val MERGE_PERSISTENT_DV_OFF = DimensionMixin("MergePersistentDV", suffix = "Disabled") val MERGE_ROW_TRACKING_DV = DimensionMixin("RowTrackingMergeDV") + val MERGE_AMT = DimensionMixin("MergeIntoAMT", alias = Some("AMT")) val COLUMN_MAPPING = DimensionWithMultipleValues( "DeltaColumnMappingEnable", List("IdMode", "NameMode"), alias = Some("ColMap")) val UPDATE_SCALA = DimensionMixin("UpdateScala", alias = Some("Scala")) @@ -238,6 +239,14 @@ object SuiteGeneratorConfig { Dims.DATA_SKIP_CHECKPOINT_V2.alone, Dims.COLUMN_MAPPING.withValueAsDimension(_.last).alone ) + ), + // AMT forces id column mapping and uses its own manifest checkpoint, so COLUMN_MAPPING and + // DATA_SKIP_CHECKPOINT_V2 are not used here. + TestConfig( + "DataSkippingDeltaV1AMTTests" :: Nil, + List( + Dims.NONE + ) ) ) ), @@ -318,6 +327,33 @@ object SuiteGeneratorConfig { ) ) ), + TestGroup( + packageName = "mergeamt", + imports = List( + importer"org.apache.spark.sql.delta._", + importer"org.apache.spark.sql.delta.amt._", + importer"org.apache.spark.sql.delta.rowid._" + ), + testConfigs = List( + // The not-matched-by-source CDC suites enable change data feed on the table. Under AMT's + // mandatory column mapping, creating a CDF-enabled table with data is rejected by + // performCdcColumnMappingCheck (DELTA_BLOCK_COLUMN_MAPPING_AND_CDC_OPERATION), so they are + // not part of the AMT variants here. + TestConfig( + (Tests.MERGE_SQL ::: Tests.MERGE_BASE).filterNot(Set( + "MergeIntoNotMatchedBySourceCDCPart1Tests", + "MergeIntoNotMatchedBySourceCDCPart2Tests" + )) ::: List( + "MergeIntoNullTypeTests" + ), + List(List(Dims.MERGE_SQL, Dims.NAME_BASED, Dims.MERGE_AMT)) + ), + TestConfig( + List("RowTrackingMergeCommonTests"), + List(List(Dims.NAME_BASED, Dims.MERGE_AMT, Dims.MERGE_ROW_TRACKING_DV.asOptional)) + ) + ) + ), TestGroup( packageName = "update", imports = List( diff --git a/spark/src/main/antlr4/io/delta/sql/parser/DeltaSqlBase.g4 b/spark/src/main/antlr4/io/delta/sql/parser/DeltaSqlBase.g4 index c176f06aab0..2e98b7d9772 100644 --- a/spark/src/main/antlr4/io/delta/sql/parser/DeltaSqlBase.g4 +++ b/spark/src/main/antlr4/io/delta/sql/parser/DeltaSqlBase.g4 @@ -90,6 +90,9 @@ statement DROP FEATURE featureName=featureNameValue (TRUNCATE HISTORY)? #alterTableDropFeature | ALTER TABLE table=qualifiedName (clusterBySpec | CLUSTER BY NONE) #alterTableClusterBy + | ALTER TABLE table=qualifiedName + REPLACE PARTITIONED BY WITH + (clusterBySpec | CLUSTER BY NONE) #alterTableReplacePartitionedBy | ALTER TABLE table=qualifiedName (ALTER | CHANGE) COLUMN? column=qualifiedName SYNC IDENTITY #alterTableSyncIdentity | OPTIMIZE (path=stringLit | table=qualifiedName) FULL? @@ -272,6 +275,7 @@ nonReserved | CLONE | SHALLOW | FEATURE | TRUNCATE | CLUSTER | NONE + | WITH ; // Define how the keywords above should appear in a user's SQL statement. @@ -345,6 +349,7 @@ USING: 'USING'; VACUUM: 'VACUUM'; VERSION: 'VERSION'; WHERE: 'WHERE'; +WITH: 'WITH'; ZORDER: 'ZORDER'; STATISTICS: 'STATISTICS'; diff --git a/spark/src/main/java/org/apache/spark/sql/delta/RowIndexFilterProvider.java b/spark/src/main/java/org/apache/spark/sql/delta/RowIndexFilterProvider.java new file mode 100644 index 00000000000..88983c839d7 --- /dev/null +++ b/spark/src/main/java/org/apache/spark/sql/delta/RowIndexFilterProvider.java @@ -0,0 +1,40 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta; + +import org.apache.hadoop.conf.Configuration; + +/** + * Provides a way to explicitly load a {@link RowIndexFilter} from a storage format into memory. + * Specific storage formats are implemented in subclasses. + * Implementations attached to file scan tasks must be serializable; filters are loaded on the + * executor. + */ +public interface RowIndexFilterProvider { + /** + * Retrieves the stored filter and loads it into memory. + * + * Different storage formats may have different loading overhead. + * It should be assumed that retrieving a filter may incur a round-trip to cloud storage + * followed by deserialization. + * + * @param hadoopConf Cloud storage configuration for loading filters from disk. + * @return The loaded filter instance. + */ + RowIndexFilter retrieve(Configuration hadoopConf); + +} diff --git a/spark/src/main/java/org/apache/spark/sql/delta/deletionvectors/mumbling/BitPacking.java b/spark/src/main/java/org/apache/spark/sql/delta/deletionvectors/mumbling/BitPacking.java new file mode 100644 index 00000000000..2da008fe6e7 --- /dev/null +++ b/spark/src/main/java/org/apache/spark/sql/delta/deletionvectors/mumbling/BitPacking.java @@ -0,0 +1,793 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.deletionvectors.mumbling; + +import java.nio.ByteBuffer; + +/** + * Bit packing and unpacking for values of 1-7 bits. + * + *

The least-significant bits of each value are packed with the first value occupying the most + * significant bits of the output. Output is padded to the nearest byte with 0s. For example, + * packing values [0b11, 0b10, 0b01] with width 2 produces 0b11100100. + */ +class BitPacking { + + private BitPacking() {} + + /** + * Packs {@code width} least-significant bits of {@code count} values into a data buffer. + * + *

Output is padded to the nearest byte with 0s. + * + *

Values are written to the buffer's underlying storage, but the buffer's position and limit + * are not modified. + * + * @param width number of bits of each value to pack + * @param values array containing source values to pack + * @param valueOffset starting index of values to pack + * @param data an output {@link ByteBuffer} + * @param dataOffset starting index for output in the data buffer + * @param count the number of values to pack + * @return the number of bytes written to the data buffer + */ + static int packBits( + int width, int[] values, int valueOffset, ByteBuffer data, int dataOffset, int count) { + switch (width) { + case 0: + return 0; + case 1: + return packBits1(values, valueOffset, data, dataOffset, count); + case 2: + return packBits2(values, valueOffset, data, dataOffset, count); + case 3: + return packBits3(values, valueOffset, data, dataOffset, count); + case 4: + return packBits4(values, valueOffset, data, dataOffset, count); + case 5: + return packBits5(values, valueOffset, data, dataOffset, count); + case 6: + return packBits6(values, valueOffset, data, dataOffset, count); + case 7: + return packBits7(values, valueOffset, data, dataOffset, count); + case 8: + return copyAsBytes(values, valueOffset, data, dataOffset, count); + default: + throw new IllegalArgumentException("Invalid bit width: " + width); + } + } + + /** + * Unpacks {@code count} values from a data buffer containing {@code width} bits of each value. + * + *

Unused bits in the last input byte are ignored. + * + *

The input buffer's position and limit are not modified. + * + * @param width number of bits of each value to unpack + * @param data an input {@link ByteBuffer} + * @param dataOffset starting index for input in the data buffer + * @param output array for unpacked output values + * @param outputOffset starting index to store values in the output array + * @param count the number of values to unpack + * @return the number of bytes read from the data buffer + */ + static int unpackBits( + int width, ByteBuffer data, int dataOffset, int[] output, int outputOffset, int count) { + switch (width) { + case 0: + return 0; + case 1: + return unpackBits1(data, dataOffset, output, outputOffset, count); + case 2: + return unpackBits2(data, dataOffset, output, outputOffset, count); + case 3: + return unpackBits3(data, dataOffset, output, outputOffset, count); + case 4: + return unpackBits4(data, dataOffset, output, outputOffset, count); + case 5: + return unpackBits5(data, dataOffset, output, outputOffset, count); + case 6: + return unpackBits6(data, dataOffset, output, outputOffset, count); + case 7: + return unpackBits7(data, dataOffset, output, outputOffset, count); + case 8: + return copyAsBytes(data, dataOffset, output, outputOffset, count); + default: + throw new IllegalArgumentException("Invalid bit width: " + width); + } + } + + /** + * Copy byte values from src into a buffer. + * + *

Values must be bytes stored in an integer array. The 3 most significant bytes of the values + * are ignored. + * + * @param source array of source values to copy + * @param sourceOffset starting offset of values to copy + * @param out output buffer values will be copied to + * @param outOffset starting offset in the output buffer + * @param count number of values (bytes) to copy + * @return the number of bytes written to the buffer + */ + private static int copyAsBytes( + int[] source, int sourceOffset, ByteBuffer out, int outOffset, int count) { + for (int i = 0; i < count; i += 1) { + ByteBuffers.writeByte(out, source[sourceOffset + i], outOffset + i); + } + + return count; + } + + /** + * Copy byte values from a buffer into an int[]. + * + * @param data buffer of source values to copy + * @param dataOffset starting offset in the input buffer + * @param out output array values will be copied to + * @param outOffset starting offset in the output buffer + * @param count number of values (bytes) to copy + * @return the number of bytes read from the buffer + */ + private static int copyAsBytes( + ByteBuffer data, int dataOffset, int[] out, int outOffset, int count) { + for (int i = 0; i < count; i += 1) { + out[outOffset + i] = ByteBuffers.readByte(data, dataOffset + i); + } + + return count; + } + + // --------------------------------------------------------------------------- + // Specialized pack: width=1..7 + // Each method packs 8 values into exactly width bytes (full groups), plus a + // partial group of remaining < 8 values in ceil(remaining*width/8) bytes. + // --------------------------------------------------------------------------- + + /** 1-bit values: 8 values into 1 byte. */ + private static int packBits1( + int[] values, int valueOffset, ByteBuffer data, int dataOffset, int count) { + int fullGroups = count / 8; + for (int group = 0; group < fullGroups; group += 1) { + int groupOffset = valueOffset + 8 * group; + int outputOffset = dataOffset + group; + int word = + packWord1( + values[groupOffset], + values[groupOffset + 1], + values[groupOffset + 2], + values[groupOffset + 3], + values[groupOffset + 4], + values[groupOffset + 5], + values[groupOffset + 6], + values[groupOffset + 7]); + ByteBuffers.writeByte(data, word, outputOffset); + } + + int remaining = count % 8; + if (remaining > 0) { + int groupOffset = valueOffset + 8 * fullGroups; + int outputOffset = dataOffset + fullGroups; + int word = + packWord1( + values[groupOffset], + remaining > 1 ? values[groupOffset + 1] : 0, + remaining > 2 ? values[groupOffset + 2] : 0, + remaining > 3 ? values[groupOffset + 3] : 0, + remaining > 4 ? values[groupOffset + 4] : 0, + remaining > 5 ? values[groupOffset + 5] : 0, + remaining > 6 ? values[groupOffset + 6] : 0, + 0); + ByteBuffers.writeByte(data, word, outputOffset); + } + + return byteWidth(count); + } + + @SuppressWarnings("checkstyle:ParameterName") + private static int packWord1(int a, int b, int c, int d, int e, int f, int g, int h) { + return ((a & 0b1) << 7) + | ((b & 0b1) << 6) + | ((c & 0b1) << 5) + | ((d & 0b1) << 4) + | ((e & 0b1) << 3) + | ((f & 0b1) << 2) + | ((g & 0b1) << 1) + | (h & 0b1); + } + + /** 2-bit values: 8 values into 2 bytes. */ + private static int packBits2( + int[] values, int valueOffset, ByteBuffer data, int dataOffset, int count) { + int fullGroups = count / 8; + for (int group = 0; group < fullGroups; group += 1) { + int groupOffset = valueOffset + 8 * group; + int outputOffset = dataOffset + 2 * group; + int word = + packWord2( + values[groupOffset], + values[groupOffset + 1], + values[groupOffset + 2], + values[groupOffset + 3], + values[groupOffset + 4], + values[groupOffset + 5], + values[groupOffset + 6], + values[groupOffset + 7]); + ByteBuffers.writeByte(data, word >>> 8, outputOffset); + ByteBuffers.writeByte(data, word, outputOffset + 1); + } + + int remaining = count % 8; + if (remaining > 0) { + int groupOffset = valueOffset + 8 * fullGroups; + int outputOffset = dataOffset + 2 * fullGroups; + int word = + packWord2( + values[groupOffset], + remaining > 1 ? values[groupOffset + 1] : 0, + remaining > 2 ? values[groupOffset + 2] : 0, + remaining > 3 ? values[groupOffset + 3] : 0, + remaining > 4 ? values[groupOffset + 4] : 0, + remaining > 5 ? values[groupOffset + 5] : 0, + remaining > 6 ? values[groupOffset + 6] : 0, + 0); + ByteBuffers.writeByte(data, word >>> 8, outputOffset); + if (remaining > 4) { + ByteBuffers.writeByte(data, word, outputOffset + 1); + } + } + + return byteWidth(2 * count); + } + + @SuppressWarnings("checkstyle:ParameterName") + private static int packWord2(int a, int b, int c, int d, int e, int f, int g, int h) { + return ((a & 0b11) << 14) + | ((b & 0b11) << 12) + | ((c & 0b11) << 10) + | ((d & 0b11) << 8) + | ((e & 0b11) << 6) + | ((f & 0b11) << 4) + | ((g & 0b11) << 2) + | (h & 0b11); + } + + /** 3-bit values: 8 values into 3 bytes. */ + private static int packBits3( + int[] values, int valueOffset, ByteBuffer data, int dataOffset, int count) { + int fullGroups = count / 8; + for (int group = 0; group < fullGroups; group += 1) { + int groupOffset = valueOffset + 8 * group; + int outputOffset = dataOffset + 3 * group; + int word = + packWord3( + values[groupOffset], + values[groupOffset + 1], + values[groupOffset + 2], + values[groupOffset + 3], + values[groupOffset + 4], + values[groupOffset + 5], + values[groupOffset + 6], + values[groupOffset + 7]); + ByteBuffers.writeByte(data, word >>> 16, outputOffset); + ByteBuffers.writeByte(data, word >>> 8, outputOffset + 1); + ByteBuffers.writeByte(data, word, outputOffset + 2); + } + + int remaining = count % 8; + if (remaining > 0) { + int groupOffset = valueOffset + 8 * fullGroups; + int outputOffset = dataOffset + 3 * fullGroups; + int word = + packWord3( + values[groupOffset], + remaining > 1 ? values[groupOffset + 1] : 0, + remaining > 2 ? values[groupOffset + 2] : 0, + remaining > 3 ? values[groupOffset + 3] : 0, + remaining > 4 ? values[groupOffset + 4] : 0, + remaining > 5 ? values[groupOffset + 5] : 0, + remaining > 6 ? values[groupOffset + 6] : 0, + 0); + int byteCount = byteWidth(3 * remaining); + for (int k = 0; k < byteCount; k += 1) { + ByteBuffers.writeByte(data, word >>> (16 - 8 * k), outputOffset + k); + } + } + + return byteWidth(3 * count); + } + + @SuppressWarnings("checkstyle:ParameterName") + private static int packWord3(int a, int b, int c, int d, int e, int f, int g, int h) { + return ((a & 0b111) << 21) + | ((b & 0b111) << 18) + | ((c & 0b111) << 15) + | ((d & 0b111) << 12) + | ((e & 0b111) << 9) + | ((f & 0b111) << 6) + | ((g & 0b111) << 3) + | (h & 0b111); + } + + /** 4-bit values: 8 values into 4 bytes. */ + private static int packBits4( + int[] values, int valueOffset, ByteBuffer data, int dataOffset, int count) { + int fullGroups = count / 8; + for (int group = 0; group < fullGroups; group += 1) { + int groupOffset = valueOffset + 8 * group; + int outputOffset = dataOffset + 4 * group; + int word = + packWord4( + values[groupOffset], + values[groupOffset + 1], + values[groupOffset + 2], + values[groupOffset + 3], + values[groupOffset + 4], + values[groupOffset + 5], + values[groupOffset + 6], + values[groupOffset + 7]); + ByteBuffers.writeByte(data, word >>> 24, outputOffset); + ByteBuffers.writeByte(data, word >>> 16, outputOffset + 1); + ByteBuffers.writeByte(data, word >>> 8, outputOffset + 2); + ByteBuffers.writeByte(data, word, outputOffset + 3); + } + + int remaining = count % 8; + if (remaining > 0) { + int groupOffset = valueOffset + 8 * fullGroups; + int outputOffset = dataOffset + 4 * fullGroups; + int word = + packWord4( + values[groupOffset], + remaining > 1 ? values[groupOffset + 1] : 0, + remaining > 2 ? values[groupOffset + 2] : 0, + remaining > 3 ? values[groupOffset + 3] : 0, + remaining > 4 ? values[groupOffset + 4] : 0, + remaining > 5 ? values[groupOffset + 5] : 0, + remaining > 6 ? values[groupOffset + 6] : 0, + 0); + int byteCount = byteWidth(4 * remaining); + for (int k = 0; k < byteCount; k += 1) { + ByteBuffers.writeByte(data, word >>> (24 - 8 * k), outputOffset + k); + } + } + + return byteWidth(4 * count); + } + + @SuppressWarnings("checkstyle:ParameterName") + private static int packWord4(int a, int b, int c, int d, int e, int f, int g, int h) { + return ((a & 0b1111) << 28) + | ((b & 0b1111) << 24) + | ((c & 0b1111) << 20) + | ((d & 0b1111) << 16) + | ((e & 0b1111) << 12) + | ((f & 0b1111) << 8) + | ((g & 0b1111) << 4) + | (h & 0b1111); + } + + /** 5-bit values: 8 values into 5 bytes. */ + private static int packBits5( + int[] values, int valueOffset, ByteBuffer data, int dataOffset, int count) { + int fullGroups = count / 8; + for (int group = 0; group < fullGroups; group += 1) { + int groupOffset = valueOffset + 8 * group; + int outputOffset = dataOffset + 5 * group; + long word = + packWord5( + values[groupOffset], + values[groupOffset + 1], + values[groupOffset + 2], + values[groupOffset + 3], + values[groupOffset + 4], + values[groupOffset + 5], + values[groupOffset + 6], + values[groupOffset + 7]); + ByteBuffers.writeByte(data, (int) (word >>> 32), outputOffset); + ByteBuffers.writeByte(data, (int) (word >>> 24), outputOffset + 1); + ByteBuffers.writeByte(data, (int) (word >>> 16), outputOffset + 2); + ByteBuffers.writeByte(data, (int) (word >>> 8), outputOffset + 3); + ByteBuffers.writeByte(data, (int) word, outputOffset + 4); + } + + int remaining = count % 8; + if (remaining > 0) { + int groupOffset = valueOffset + 8 * fullGroups; + int outputOffset = dataOffset + 5 * fullGroups; + long word = + packWord5( + values[groupOffset], + remaining > 1 ? values[groupOffset + 1] : 0, + remaining > 2 ? values[groupOffset + 2] : 0, + remaining > 3 ? values[groupOffset + 3] : 0, + remaining > 4 ? values[groupOffset + 4] : 0, + remaining > 5 ? values[groupOffset + 5] : 0, + remaining > 6 ? values[groupOffset + 6] : 0, + 0); + int byteCount = byteWidth(5 * remaining); + for (int k = 0; k < byteCount; k += 1) { + ByteBuffers.writeByte(data, (int) (word >>> (32 - 8 * k)), outputOffset + k); + } + } + + return byteWidth(5 * count); + } + + @SuppressWarnings("checkstyle:ParameterName") + private static long packWord5(int a, int b, int c, int d, int e, int f, int g, int h) { + return ((long) (a & 0b11111) << 35) + | ((long) (b & 0b11111) << 30) + | ((long) (c & 0b11111) << 25) + | ((long) (d & 0b11111) << 20) + | ((long) (e & 0b11111) << 15) + | ((long) (f & 0b11111) << 10) + | ((long) (g & 0b11111) << 5) + | (long) (h & 0b11111); + } + + /** 6-bit values: 8 values into 6 bytes. */ + private static int packBits6( + int[] values, int valueOffset, ByteBuffer data, int dataOffset, int count) { + int fullGroups = count / 8; + for (int group = 0; group < fullGroups; group += 1) { + int groupOffset = valueOffset + 8 * group; + int outputOffset = dataOffset + 6 * group; + long word = + packWord6( + values[groupOffset], + values[groupOffset + 1], + values[groupOffset + 2], + values[groupOffset + 3], + values[groupOffset + 4], + values[groupOffset + 5], + values[groupOffset + 6], + values[groupOffset + 7]); + ByteBuffers.writeByte(data, (int) (word >>> 40), outputOffset); + ByteBuffers.writeByte(data, (int) (word >>> 32), outputOffset + 1); + ByteBuffers.writeByte(data, (int) (word >>> 24), outputOffset + 2); + ByteBuffers.writeByte(data, (int) (word >>> 16), outputOffset + 3); + ByteBuffers.writeByte(data, (int) (word >>> 8), outputOffset + 4); + ByteBuffers.writeByte(data, (int) word, outputOffset + 5); + } + + int remaining = count % 8; + if (remaining > 0) { + int groupOffset = valueOffset + 8 * fullGroups; + int outputOffset = dataOffset + 6 * fullGroups; + long word = + packWord6( + values[groupOffset], + remaining > 1 ? values[groupOffset + 1] : 0, + remaining > 2 ? values[groupOffset + 2] : 0, + remaining > 3 ? values[groupOffset + 3] : 0, + remaining > 4 ? values[groupOffset + 4] : 0, + remaining > 5 ? values[groupOffset + 5] : 0, + remaining > 6 ? values[groupOffset + 6] : 0, + 0); + int byteCount = byteWidth(6 * remaining); + for (int k = 0; k < byteCount; k += 1) { + ByteBuffers.writeByte(data, (int) (word >>> (40 - 8 * k)), outputOffset + k); + } + } + + return byteWidth(6 * count); + } + + @SuppressWarnings("checkstyle:ParameterName") + private static long packWord6(int a, int b, int c, int d, int e, int f, int g, int h) { + return ((long) (a & 0b111111) << 42) + | ((long) (b & 0b111111) << 36) + | ((long) (c & 0b111111) << 30) + | ((long) (d & 0b111111) << 24) + | ((long) (e & 0b111111) << 18) + | ((long) (f & 0b111111) << 12) + | ((long) (g & 0b111111) << 6) + | (long) (h & 0b111111); + } + + /** 7-bit values: 8 values into 7 bytes. */ + private static int packBits7( + int[] values, int valueOffset, ByteBuffer data, int dataOffset, int count) { + int fullGroups = count / 8; + for (int group = 0; group < fullGroups; group += 1) { + int groupOffset = valueOffset + 8 * group; + int outputOffset = dataOffset + 7 * group; + long word = + packWord7( + values[groupOffset], + values[groupOffset + 1], + values[groupOffset + 2], + values[groupOffset + 3], + values[groupOffset + 4], + values[groupOffset + 5], + values[groupOffset + 6], + values[groupOffset + 7]); + ByteBuffers.writeByte(data, (int) (word >>> 48), outputOffset); + ByteBuffers.writeByte(data, (int) (word >>> 40), outputOffset + 1); + ByteBuffers.writeByte(data, (int) (word >>> 32), outputOffset + 2); + ByteBuffers.writeByte(data, (int) (word >>> 24), outputOffset + 3); + ByteBuffers.writeByte(data, (int) (word >>> 16), outputOffset + 4); + ByteBuffers.writeByte(data, (int) (word >>> 8), outputOffset + 5); + ByteBuffers.writeByte(data, (int) word, outputOffset + 6); + } + + int remaining = count % 8; + if (remaining > 0) { + int groupOffset = valueOffset + 8 * fullGroups; + int outputOffset = dataOffset + 7 * fullGroups; + long word = + packWord7( + values[groupOffset], + remaining > 1 ? values[groupOffset + 1] : 0, + remaining > 2 ? values[groupOffset + 2] : 0, + remaining > 3 ? values[groupOffset + 3] : 0, + remaining > 4 ? values[groupOffset + 4] : 0, + remaining > 5 ? values[groupOffset + 5] : 0, + remaining > 6 ? values[groupOffset + 6] : 0, + 0); + int byteCount = byteWidth(7 * remaining); + for (int k = 0; k < byteCount; k += 1) { + ByteBuffers.writeByte(data, (int) (word >>> (48 - 8 * k)), outputOffset + k); + } + } + + return byteWidth(7 * count); + } + + @SuppressWarnings("checkstyle:ParameterName") + private static long packWord7(int a, int b, int c, int d, int e, int f, int g, int h) { + return ((long) (a & 0b1111111) << 49) + | ((long) (b & 0b1111111) << 42) + | ((long) (c & 0b1111111) << 35) + | ((long) (d & 0b1111111) << 28) + | ((long) (e & 0b1111111) << 21) + | ((long) (f & 0b1111111) << 14) + | ((long) (g & 0b1111111) << 7) + | (long) (h & 0b1111111); + } + + // --------------------------------------------------------------------------- + // Specialized unpack: width=1..7 + // --------------------------------------------------------------------------- + + /** 1-bit values: 1 byte into 8 values. */ + private static int unpackBits1( + ByteBuffer data, int dataOffset, int[] output, int outputOffset, int count) { + int fullGroups = count / 8; + for (int group = 0; group < fullGroups; group += 1) { + int groupOffset = outputOffset + 8 * group; + int word = (int) readWord(data, dataOffset + group, 1); + output[groupOffset] = (word >>> 7) & 0b1; + output[groupOffset + 1] = (word >>> 6) & 0b1; + output[groupOffset + 2] = (word >>> 5) & 0b1; + output[groupOffset + 3] = (word >>> 4) & 0b1; + output[groupOffset + 4] = (word >>> 3) & 0b1; + output[groupOffset + 5] = (word >>> 2) & 0b1; + output[groupOffset + 6] = (word >>> 1) & 0b1; + output[groupOffset + 7] = word & 0b1; + } + + int remaining = count % 8; + if (remaining > 0) { + long word = readWord(data, dataOffset + fullGroups, byteWidth(remaining)); + unpackRemainder(1, word, output, outputOffset + 8 * fullGroups, remaining); + } + + return byteWidth(count); + } + + /** 2-bit values: 2 bytes into 8 values. */ + private static int unpackBits2( + ByteBuffer data, int dataOffset, int[] output, int outputOffset, int count) { + int fullGroups = count / 8; + for (int group = 0; group < fullGroups; group += 1) { + int groupOffset = outputOffset + 8 * group; + int word = (int) readWord(data, dataOffset + 2 * group, 2); + output[groupOffset] = (word >>> 14) & 0b11; + output[groupOffset + 1] = (word >>> 12) & 0b11; + output[groupOffset + 2] = (word >>> 10) & 0b11; + output[groupOffset + 3] = (word >>> 8) & 0b11; + output[groupOffset + 4] = (word >>> 6) & 0b11; + output[groupOffset + 5] = (word >>> 4) & 0b11; + output[groupOffset + 6] = (word >>> 2) & 0b11; + output[groupOffset + 7] = word & 0b11; + } + + int remaining = count % 8; + if (remaining > 0) { + long word = readWord(data, dataOffset + 2 * fullGroups, byteWidth(2 * remaining)); + unpackRemainder(2, word, output, outputOffset + 8 * fullGroups, remaining); + } + + return byteWidth(2 * count); + } + + /** 3-bit values: 3 bytes into 8 values. */ + private static int unpackBits3( + ByteBuffer data, int dataOffset, int[] output, int outputOffset, int count) { + int fullGroups = count / 8; + for (int group = 0; group < fullGroups; group += 1) { + int groupOffset = outputOffset + 8 * group; + int word = (int) readWord(data, dataOffset + 3 * group, 3); + output[groupOffset] = (word >>> 21) & 0b111; + output[groupOffset + 1] = (word >>> 18) & 0b111; + output[groupOffset + 2] = (word >>> 15) & 0b111; + output[groupOffset + 3] = (word >>> 12) & 0b111; + output[groupOffset + 4] = (word >>> 9) & 0b111; + output[groupOffset + 5] = (word >>> 6) & 0b111; + output[groupOffset + 6] = (word >>> 3) & 0b111; + output[groupOffset + 7] = word & 0b111; + } + + int remaining = count % 8; + if (remaining > 0) { + long word = readWord(data, dataOffset + 3 * fullGroups, byteWidth(3 * remaining)); + unpackRemainder(3, word, output, outputOffset + 8 * fullGroups, remaining); + } + + return byteWidth(3 * count); + } + + /** 4-bit values: 4 bytes into 8 values. */ + private static int unpackBits4( + ByteBuffer data, int dataOffset, int[] output, int outputOffset, int count) { + int fullGroups = count / 8; + for (int group = 0; group < fullGroups; group += 1) { + int groupOffset = outputOffset + 8 * group; + long word = readWord(data, dataOffset + 4 * group, 4); + output[groupOffset] = (int) (word >>> 28) & 0b1111; + output[groupOffset + 1] = (int) (word >>> 24) & 0b1111; + output[groupOffset + 2] = (int) (word >>> 20) & 0b1111; + output[groupOffset + 3] = (int) (word >>> 16) & 0b1111; + output[groupOffset + 4] = (int) (word >>> 12) & 0b1111; + output[groupOffset + 5] = (int) (word >>> 8) & 0b1111; + output[groupOffset + 6] = (int) (word >>> 4) & 0b1111; + output[groupOffset + 7] = (int) word & 0b1111; + } + + int remaining = count % 8; + if (remaining > 0) { + long word = readWord(data, dataOffset + 4 * fullGroups, byteWidth(4 * remaining)); + unpackRemainder(4, word, output, outputOffset + 8 * fullGroups, remaining); + } + + return byteWidth(4 * count); + } + + /** 5-bit values: 5 bytes into 8 values. */ + private static int unpackBits5( + ByteBuffer data, int dataOffset, int[] output, int outputOffset, int count) { + int fullGroups = count / 8; + for (int group = 0; group < fullGroups; group += 1) { + int groupOffset = outputOffset + 8 * group; + long word = readWord(data, dataOffset + 5 * group, 5); + output[groupOffset] = (int) (word >>> 35) & 0b11111; + output[groupOffset + 1] = (int) (word >>> 30) & 0b11111; + output[groupOffset + 2] = (int) (word >>> 25) & 0b11111; + output[groupOffset + 3] = (int) (word >>> 20) & 0b11111; + output[groupOffset + 4] = (int) (word >>> 15) & 0b11111; + output[groupOffset + 5] = (int) (word >>> 10) & 0b11111; + output[groupOffset + 6] = (int) (word >>> 5) & 0b11111; + output[groupOffset + 7] = (int) word & 0b11111; + } + + int remaining = count % 8; + if (remaining > 0) { + long word = readWord(data, dataOffset + 5 * fullGroups, byteWidth(5 * remaining)); + unpackRemainder(5, word, output, outputOffset + 8 * fullGroups, remaining); + } + + return byteWidth(5 * count); + } + + /** 6-bit values: 6 bytes into 8 values. */ + private static int unpackBits6( + ByteBuffer data, int dataOffset, int[] output, int outputOffset, int count) { + int fullGroups = count / 8; + for (int group = 0; group < fullGroups; group += 1) { + int groupOffset = outputOffset + 8 * group; + long word = readWord(data, dataOffset + 6 * group, 6); + output[groupOffset] = (int) (word >>> 42) & 0b111111; + output[groupOffset + 1] = (int) (word >>> 36) & 0b111111; + output[groupOffset + 2] = (int) (word >>> 30) & 0b111111; + output[groupOffset + 3] = (int) (word >>> 24) & 0b111111; + output[groupOffset + 4] = (int) (word >>> 18) & 0b111111; + output[groupOffset + 5] = (int) (word >>> 12) & 0b111111; + output[groupOffset + 6] = (int) (word >>> 6) & 0b111111; + output[groupOffset + 7] = (int) word & 0b111111; + } + + int remaining = count % 8; + if (remaining > 0) { + long word = readWord(data, dataOffset + 6 * fullGroups, byteWidth(6 * remaining)); + unpackRemainder(6, word, output, outputOffset + 8 * fullGroups, remaining); + } + + return byteWidth(6 * count); + } + + /** 7-bit values: 7 bytes into 8 values. */ + private static int unpackBits7( + ByteBuffer data, int dataOffset, int[] output, int outputOffset, int count) { + int fullGroups = count / 8; + for (int group = 0; group < fullGroups; group += 1) { + int groupOffset = outputOffset + 8 * group; + long word = readWord(data, dataOffset + 7 * group, 7); + output[groupOffset] = (int) (word >>> 49) & 0b1111111; + output[groupOffset + 1] = (int) (word >>> 42) & 0b1111111; + output[groupOffset + 2] = (int) (word >>> 35) & 0b1111111; + output[groupOffset + 3] = (int) (word >>> 28) & 0b1111111; + output[groupOffset + 4] = (int) (word >>> 21) & 0b1111111; + output[groupOffset + 5] = (int) (word >>> 14) & 0b1111111; + output[groupOffset + 6] = (int) (word >>> 7) & 0b1111111; + output[groupOffset + 7] = (int) word & 0b1111111; + } + + int remaining = count % 8; + if (remaining > 0) { + long word = readWord(data, dataOffset + 7 * fullGroups, byteWidth(7 * remaining)); + unpackRemainder(7, word, output, outputOffset + 8 * fullGroups, remaining); + } + + return byteWidth(7 * count); + } + + /** + * Unpack {@code count < 8} values of {@code width} bits from {@code word}. + * + * @param width number of bits stored for each value + * @param word a long containing the bytes of the remaining values + * @param output array for unpacked output values + * @param outputOffset starting index to store values in the output array + * @param count number of values to unpack from the word + */ + private static void unpackRemainder( + int width, long word, int[] output, int outputOffset, int count) { + int mask = (1 << width) - 1; + // value bits are stored in the last bytes of the word + int valueBytes = byteWidth(count * width); + // the first value is width bits starting with the most-significant bits of the value bytes + int shift = 8 * valueBytes - width; + for (int i = 0; i < count; i += 1) { + output[outputOffset + i] = (int) ((word >>> shift) & mask); + shift -= width; + } + } + + /** + * Read {@code count} bytes of data into the last {@code count} bytes of {@code word}. + * + *

The first input byte occupies the most significant bits of the last {@code count} bytes and + * the last input byte occupies the least significant bits of the word. The remaining most + * significant bytes of the word are 0. + */ + private static long readWord(ByteBuffer data, int offset, int count) { + long word = 0; + for (int k = 0; k < count; k += 1) { + word = (word << 8) | ByteBuffers.readByte(data, offset + k); + } + + return word; + } + + /** Returns the number of whole bytes needed to hold {@code bits} bits. */ + private static int byteWidth(int bits) { + return (bits + 7) / 8; + } +} diff --git a/spark/src/main/java/org/apache/spark/sql/delta/deletionvectors/mumbling/ByteBuffers.java b/spark/src/main/java/org/apache/spark/sql/delta/deletionvectors/mumbling/ByteBuffers.java new file mode 100644 index 00000000000..9a7cea44f22 --- /dev/null +++ b/spark/src/main/java/org/apache/spark/sql/delta/deletionvectors/mumbling/ByteBuffers.java @@ -0,0 +1,50 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.deletionvectors.mumbling; + +import java.nio.ByteBuffer; + +/** + * Absolute single-byte accessors for a {@link ByteBuffer}, indexed relative to the buffer's + * current position and leaving the buffer's position and limit unmodified. + */ +final class ByteBuffers { + + private ByteBuffers() {} + + /** + * Reads the unsigned byte at {@code buffer.position() + offset}. + * + * @param buffer the buffer to read from; position and limit are not modified + * @param offset the offset relative to the buffer's position + * @return the byte value in the range {@code [0, 255]} + */ + static int readByte(ByteBuffer buffer, int offset) { + return buffer.get(buffer.position() + offset) & 0xFF; + } + + /** + * Writes the low 8 bits of {@code value} at {@code buffer.position() + offset}. + * + * @param buffer the buffer to write to; position and limit are not modified + * @param value the value whose low byte is written + * @param offset the offset relative to the buffer's position + */ + static void writeByte(ByteBuffer buffer, int value, int offset) { + buffer.put(buffer.position() + offset, (byte) value); + } +} diff --git a/spark/src/main/java/org/apache/spark/sql/delta/deletionvectors/mumbling/MumblingBitmap.java b/spark/src/main/java/org/apache/spark/sql/delta/deletionvectors/mumbling/MumblingBitmap.java new file mode 100644 index 00000000000..bacc9a7b9dd --- /dev/null +++ b/spark/src/main/java/org/apache/spark/sql/delta/deletionvectors/mumbling/MumblingBitmap.java @@ -0,0 +1,201 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.deletionvectors.mumbling; + +import java.nio.ByteBuffer; + +import org.apache.spark.sql.delta.deletionvectors.MutableMumblingBitmap; +import com.google.common.base.Preconditions; + +/** + * Immutable Mumbling compressed bitmap read from a {@link ByteBuffer}. + * + *

The supplied bytes are copied, but descriptor decoding remains lazy. On the first call to + * {@link #isSet}, the PFOR-encoded descriptor array is decoded and used to build an offsets array + * that maps each container index to its absolute byte position in the buffer. This offsets array is + * the only derived state kept by this class. + * + *

Format (all integers unsigned, little-endian): + * + *

    + *
  • Header (6 bytes): version (1), cardinality (3), container count (2) + *
  • Descriptor array: PFOR-encoded, one byte per container + *
  • Containers: concatenated sparse (0-31 bytes) or dense (32 bytes) containers + *
+ */ +public final class MumblingBitmap { + private final ByteBuffer data; + private final int cardinality; + private final int containerCount; + private volatile int[] descriptors = null; + private volatile int[] offsets = null; + + public MumblingBitmap(ByteBuffer data) { + this(copyByteBuffer(data)); + } + + MumblingBitmap(byte[] data) { + this.data = ByteBuffer.wrap(data).asReadOnlyBuffer(); + + int version = this.data.get(0) & 0xFF; + if (version != MumblingFormat.VERSION) { + throw new UnsupportedOperationException("Unsupported Mumbling bitmap version: " + version); + } + + this.cardinality = + (this.data.get(1) & 0xFF) + | ((this.data.get(2) & 0xFF) << 8) + | ((this.data.get(3) & 0xFF) << 16); + this.containerCount = + (this.data.get(4) & 0xFF) | ((this.data.get(5) & 0xFF) << 8); + + Preconditions.checkState( + containerCount <= MumblingFormat.MAX_CONTAINERS, + "Invalid container count: %s > %s (max)", + containerCount, + MumblingFormat.MAX_CONTAINERS); + Preconditions.checkState( + cardinality <= MumblingFormat.MAX_POSITION_EXCLUSIVE, + "Invalid cardinality: %s > %s (max)", + cardinality, + MumblingFormat.MAX_POSITION_EXCLUSIVE); + } + + public ByteBuffer buffer() { + return data.asReadOnlyBuffer(); + } + + /** Returns an independent mutable copy of this bitmap. */ + public MutableMumblingBitmap toMutable() { + return new MutableMumblingBitmap(this); + } + + /** Returns the number of bits set in the bitmap. */ + public int cardinality() { + return cardinality; + } + + /** + * Returns {@code true} if the bit at {@code pos} is set in the bitmap. + * + *

Positions beyond the range of any container are always unset. + */ + public boolean isSet(int pos) { + Preconditions.checkArgument(pos >= 0, "Invalid bit position: %s < 0", pos); + int containerIndex = pos >>> 8; + int posInContainer = pos & 0xFF; + + if (containerIndex >= containerCount) { + return false; + } + + int containerStart = offset(containerIndex); + int descriptor = descriptor(containerIndex); + + if (MumblingFormat.isDense(descriptor)) { + // Dense: 32-byte bitset, MSB of byte 0 is position 0 + int byteIndex = posInContainer >>> 3; + int bitShift = 7 - (posInContainer & 0b111); + return ((data.get(containerStart + byteIndex) >>> bitShift) & 0b1) == 0b1; + + } else { + // Sparse: sorted list of set positions; scan until found or exceeded + for (int i = 0; i < descriptor; i += 1) { + int stored = data.get(containerStart + i) & 0xFF; + if (stored == posInContainer) { + return true; + } + + if (stored > posInContainer) { + return false; + } + } + + return false; + } + } + + private int descriptor(int containerIndex) { + if (null == descriptors) { + decodeDescriptors(); + } + + return descriptors[containerIndex]; + } + + private int offset(int containerIndex) { + if (null == offsets) { + decodeDescriptors(); + } + + return offsets[containerIndex]; + } + + /** + * Decode the descriptor array and produce an array of absolute container offsets in the buffer. + */ + private void decodeDescriptors() { + int[] descriptorArray = new int[containerCount]; + int bytesRead = + PFOREncoding.decode( + data, MumblingFormat.HEADER_SIZE, descriptorArray, 0, containerCount); + + int[] offsetArray = new int[containerCount + 1]; + int firstContainerOffset = data.position() + MumblingFormat.HEADER_SIZE + bytesRead; + descriptorsToOffsets(firstContainerOffset, descriptorArray, offsetArray); + + // update the references last so that only valid values are available + this.descriptors = descriptorArray; + this.offsets = offsetArray; + } + + private static byte[] copyByteBuffer(ByteBuffer data) { + ByteBuffer source = data.slice(); + byte[] bytes = new byte[source.remaining()]; + source.get(bytes); + return bytes; + } + + /** + * Convert an array of lengths into an array of offsets starting at the given base. + * + *

For example, descriptorsToOffsets(0, [1, 1, 2]) produces [0, 1, 2, 4]. + * + * @param baseOffset initial offset of the first container + * @param descriptors an array of descriptor bytes + * @param offsets output array of offsets + */ + private static void descriptorsToOffsets(int baseOffset, int[] descriptors, int[] offsets) { + Preconditions.checkArgument( + offsets.length > descriptors.length, + "Cannot decode %s lengths into %s offsets (not enough space)", + descriptors.length, + offsets.length); + + offsets[0] = baseOffset; + for (int i = 0; i < descriptors.length; i += 1) { + if (MumblingFormat.isDense(descriptors[i])) { + offsets[i + 1] = offsets[i] + MumblingFormat.DENSE_CONTAINER_SIZE; + } else if (MumblingFormat.isSparse(descriptors[i])) { + offsets[i + 1] = offsets[i] + descriptors[i]; + } else { + throw new IllegalStateException( + "Invalid descriptor, not sparse or dense: " + descriptors[i]); + } + } + } +} diff --git a/spark/src/main/java/org/apache/spark/sql/delta/deletionvectors/mumbling/MumblingBitmapWriter.java b/spark/src/main/java/org/apache/spark/sql/delta/deletionvectors/mumbling/MumblingBitmapWriter.java new file mode 100644 index 00000000000..1ad31f951d7 --- /dev/null +++ b/spark/src/main/java/org/apache/spark/sql/delta/deletionvectors/mumbling/MumblingBitmapWriter.java @@ -0,0 +1,337 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.deletionvectors.mumbling; + +import java.nio.ByteBuffer; + +import com.google.common.base.Preconditions; + +/** + * Serializer for the Mumbling bitmap format read by {@link MumblingBitmap}. + * + *

Iceberg's Mumbling implementation ships only a read-only view; the byte layout produced here + * is the write side, built directly on the (bidirectional) {@link PFOREncoding} descriptor codec so + * it round-trips exactly through {@link MumblingBitmap}. The container assembly mirrors the + * reference layout: a 6-byte header, a PFOR-encoded descriptor array (one descriptor per + * container), then the concatenated container bytes. + * + *

TODO: Replace this class with Iceberg's Mumbling writer when one is available. + */ +public final class MumblingBitmapWriter { + /** + * Positions must be strictly less than this bound. The spec caps a Mumbling bitmap at {@code + * MAX_CONTAINERS} containers of {@code CONTAINER_SIZE} positions (container index = position >>> + * 8), which also bounds the cardinality since positions are distinct. + */ + public static final int MAX_POSITION_EXCLUSIVE = MumblingFormat.MAX_POSITION_EXCLUSIVE; + + private MumblingBitmapWriter() {} + + /** + * Serializes the given positions into a Mumbling bitmap. + * + * @param sortedPositions set positions in strictly ascending order, each in + * {@code [0, MAX_POSITION_EXCLUSIVE)} + * @return the serialized bitmap bytes + */ + public static byte[] serialize(int[] sortedPositions) { + validateSortedPositions(sortedPositions); + int cardinality = sortedPositions.length; + if (cardinality == 0) { + // Header only: version = 1, cardinality = 0, container count = 0. + byte[] empty = new byte[MumblingFormat.HEADER_SIZE]; + empty[0] = (byte) MumblingFormat.VERSION; + return empty; + } + + int maxPosition = sortedPositions[cardinality - 1]; + int containerCount = (maxPosition >>> 8) + 1; + int[] descriptors = new int[containerCount]; + byte[][] containers = new byte[containerCount][]; + int totalContainerBytes = 0; + + int index = 0; + for (int container = 0; container < containerCount; container += 1) { + int start = index; + int upperExclusive = (container + 1) << 8; + while (index < cardinality && sortedPositions[index] < upperExclusive) { + index += 1; + } + int count = index - start; + + byte[] bytes; + if (count <= MumblingFormat.MAX_SPARSE_COUNT) { + // Sparse: descriptor = count, one ascending low-byte position per set bit. + bytes = new byte[count]; + for (int i = 0; i < count; i += 1) { + bytes[i] = (byte) (sortedPositions[start + i] & 0xFF); + } + descriptors[container] = count; + } else { + // Dense: 32-byte bitset, MSB of byte 0 is position 0. + bytes = new byte[MumblingFormat.DENSE_CONTAINER_SIZE]; + for (int i = 0; i < count; i += 1) { + int posInContainer = sortedPositions[start + i] & 0xFF; + int byteIndex = posInContainer >>> 3; + int bitShift = 7 - (posInContainer & 0b111); + bytes[byteIndex] |= (byte) (1 << bitShift); + } + descriptors[container] = MumblingFormat.DENSE_CONTAINER_BIT; + } + containers[container] = bytes; + totalContainerBytes += bytes.length; + } + + return serializeContainers(cardinality, descriptors, containers, totalContainerBytes); + } + + /** + * Merges positions into an existing serialized Mumbling bitmap without materializing its set + * positions. At most one decoded 256-position container is held at a time. + * + * @param sourceBitmap an existing Mumbling bitmap + * @param additionContainers 32-byte addition bitsets indexed by container, with {@code null} + * entries for containers without additions + * @return the merged bitmap + */ + public static MumblingBitmap merge( + MumblingBitmap sourceBitmap, byte[][] additionContainers) { + Preconditions.checkArgument( + additionContainers.length <= MumblingFormat.MAX_CONTAINERS, + "Too many Mumbling addition containers: %s > %s", + additionContainers.length, + MumblingFormat.MAX_CONTAINERS); + int additionContainerCount = additionContainers.length; + while (additionContainerCount > 0 + && additionContainers[additionContainerCount - 1] == null) { + additionContainerCount -= 1; + } + for (int container = 0; container < additionContainerCount; container += 1) { + byte[] additionContainer = additionContainers[container]; + Preconditions.checkArgument( + additionContainer == null + || additionContainer.length == MumblingFormat.DENSE_CONTAINER_SIZE, + "Mumbling addition container %s must contain exactly %s bytes", + container, + MumblingFormat.DENSE_CONTAINER_SIZE); + } + + ByteBuffer source = sourceBitmap.buffer(); + int sourceContainerCount = + (source.get(source.position() + 4) & 0xFF) + | ((source.get(source.position() + 5) & 0xFF) << 8); + int[] sourceDescriptors = new int[sourceContainerCount]; + int descriptorBytes = + PFOREncoding.decode( + source, + MumblingFormat.HEADER_SIZE, + sourceDescriptors, + 0, + sourceContainerCount); + int sourceContainerOffset = + source.position() + MumblingFormat.HEADER_SIZE + descriptorBytes; + + int containerCount = Math.max(sourceContainerCount, additionContainerCount); + int[] descriptors = new int[containerCount]; + byte[][] containers = new byte[containerCount][]; + int totalContainerBytes = 0; + int sourceCardinality = 0; + int mergedCardinality = 0; + + for (int container = 0; container < containerCount; container += 1) { + byte[] additionContainer = + container < additionContainerCount ? additionContainers[container] : null; + + // Fast path 1: copy an untouched source container without decoding and re-encoding it. + if (container < sourceContainerCount && additionContainer == null) { + int descriptor = sourceDescriptors[container]; + int encodedSize = encodedContainerSize(descriptor); + byte[] encodedContainer = copyBytes(source, sourceContainerOffset, encodedSize); + int containerCardinality = + MumblingFormat.isDense(descriptor) ? cardinality(encodedContainer) : descriptor; + + sourceCardinality += containerCardinality; + mergedCardinality += containerCardinality; + descriptors[container] = + MumblingFormat.isDense(descriptor) + ? MumblingFormat.DENSE_CONTAINER_BIT + : descriptor; + containers[container] = encodedContainer; + totalContainerBytes += encodedSize; + sourceContainerOffset += encodedSize; + continue; + } + + // Fast path 2: emit an empty container for a gap before a later addition. + if (container >= sourceContainerCount && additionContainer == null) { + descriptors[container] = 0; + containers[container] = new byte[0]; + continue; + } + + byte[] dense = new byte[MumblingFormat.DENSE_CONTAINER_SIZE]; + if (container < sourceContainerCount) { + int descriptor = sourceDescriptors[container]; + if (MumblingFormat.isDense(descriptor)) { + for (int i = 0; i < MumblingFormat.DENSE_CONTAINER_SIZE; i += 1) { + dense[i] = source.get(sourceContainerOffset + i); + } + sourceContainerOffset += MumblingFormat.DENSE_CONTAINER_SIZE; + } else if (MumblingFormat.isSparse(descriptor)) { + for (int i = 0; i < descriptor; i += 1) { + setBit(dense, source.get(sourceContainerOffset + i) & 0xFF); + } + sourceContainerOffset += descriptor; + } else { + throw new IllegalStateException( + "Invalid descriptor, not sparse or dense: " + descriptor); + } + sourceCardinality += cardinality(dense); + } + + // Null addition containers take one of the fast paths above. + for (int i = 0; i < MumblingFormat.DENSE_CONTAINER_SIZE; i += 1) { + dense[i] |= additionContainer[i]; + } + + int containerCardinality = cardinality(dense); + mergedCardinality += containerCardinality; + byte[] encodedContainer = encodeContainer(dense, containerCardinality); + descriptors[container] = + containerCardinality <= MumblingFormat.MAX_SPARSE_COUNT + ? containerCardinality + : MumblingFormat.DENSE_CONTAINER_BIT; + containers[container] = encodedContainer; + totalContainerBytes += encodedContainer.length; + } + + Preconditions.checkState( + sourceCardinality == sourceBitmap.cardinality(), + "Mumbling cardinality %s does not match decoded cardinality %s", + sourceBitmap.cardinality(), + sourceCardinality); + return new MumblingBitmap( + serializeContainers(mergedCardinality, descriptors, containers, totalContainerBytes)); + } + + private static int encodedContainerSize(int descriptor) { + if (MumblingFormat.isDense(descriptor)) { + return MumblingFormat.DENSE_CONTAINER_SIZE; + } else if (MumblingFormat.isSparse(descriptor)) { + return descriptor; + } + throw new IllegalStateException( + "Invalid descriptor, not sparse or dense: " + descriptor); + } + + private static byte[] copyBytes(ByteBuffer source, int offset, int length) { + byte[] copy = new byte[length]; + for (int i = 0; i < length; i += 1) { + copy[i] = source.get(offset + i); + } + return copy; + } + + private static byte[] serializeContainers( + int cardinality, int[] descriptors, byte[][] containers, int totalContainerBytes) { + int containerCount = descriptors.length; + int sizeEstimate = + MumblingFormat.HEADER_SIZE + + PFOREncoding.estimateEncodedSize(containerCount) + + totalContainerBytes; + ByteBuffer buffer = ByteBuffer.allocate(sizeEstimate); + + // Header: version (1 byte), cardinality (3 bytes LE), container count (2 bytes LE). + buffer.put(0, (byte) MumblingFormat.VERSION); + buffer.put(1, (byte) (cardinality & 0xFF)); + buffer.put(2, (byte) ((cardinality >>> 8) & 0xFF)); + buffer.put(3, (byte) ((cardinality >>> 16) & 0xFF)); + buffer.put(4, (byte) (containerCount & 0xFF)); + buffer.put(5, (byte) ((containerCount >>> 8) & 0xFF)); + + // PFOR-encoded descriptor array. + int descriptorArraySize = + PFOREncoding.encode( + descriptors, 0, buffer, MumblingFormat.HEADER_SIZE, containerCount); + + // Concatenated container bytes. + int containerOffset = MumblingFormat.HEADER_SIZE + descriptorArraySize; + for (byte[] bytes : containers) { + for (int i = 0; i < bytes.length; i += 1) { + buffer.put(containerOffset + i, bytes[i]); + } + containerOffset += bytes.length; + } + + byte[] out = new byte[containerOffset]; + System.arraycopy(buffer.array(), 0, out, 0, containerOffset); + return out; + } + + private static byte[] encodeContainer(byte[] dense, int containerCardinality) { + if (containerCardinality > MumblingFormat.MAX_SPARSE_COUNT) { + return dense; + } + + byte[] sparse = new byte[containerCardinality]; + int index = 0; + for (int position = 0; position < MumblingFormat.CONTAINER_SIZE; position += 1) { + if (isSet(dense, position)) { + sparse[index] = (byte) position; + index += 1; + } + } + return sparse; + } + + private static int cardinality(byte[] dense) { + int cardinality = 0; + for (byte value : dense) { + cardinality += Integer.bitCount(value & 0xFF); + } + return cardinality; + } + + private static void setBit(byte[] dense, int position) { + int byteIndex = position >>> 3; + int bitShift = 7 - (position & 0b111); + dense[byteIndex] |= (byte) (1 << bitShift); + } + + private static boolean isSet(byte[] dense, int position) { + int byteIndex = position >>> 3; + int bitShift = 7 - (position & 0b111); + return ((dense[byteIndex] >>> bitShift) & 0b1) == 0b1; + } + + private static void validateSortedPositions(int[] sortedPositions) { + int previous = -1; + for (int position : sortedPositions) { + Preconditions.checkArgument( + position >= 0 && position < MAX_POSITION_EXCLUSIVE, + "Mumbling position must be within [0, %s): %s", + MAX_POSITION_EXCLUSIVE, + position); + Preconditions.checkArgument( + position > previous, + "Mumbling positions must be strictly increasing: %s after %s", + position, + previous); + previous = position; + } + } +} diff --git a/spark/src/main/java/org/apache/spark/sql/delta/deletionvectors/mumbling/MumblingFormat.java b/spark/src/main/java/org/apache/spark/sql/delta/deletionvectors/mumbling/MumblingFormat.java new file mode 100644 index 00000000000..056bdbfb2bc --- /dev/null +++ b/spark/src/main/java/org/apache/spark/sql/delta/deletionvectors/mumbling/MumblingFormat.java @@ -0,0 +1,39 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.deletionvectors.mumbling; + +/** Shared constants and helpers for the Mumbling v1 wire format. */ +final class MumblingFormat { + static final int VERSION = 1; + static final int HEADER_SIZE = 6; + static final int DENSE_CONTAINER_BIT = 0b0010_0000; + static final int DENSE_CONTAINER_SIZE = 32; + static final int MAX_SPARSE_COUNT = DENSE_CONTAINER_SIZE - 1; + static final int CONTAINER_SIZE = 256; + static final int MAX_CONTAINERS = 8192; + static final int MAX_POSITION_EXCLUSIVE = MAX_CONTAINERS * CONTAINER_SIZE; + + private MumblingFormat() {} + + static boolean isDense(int descriptor) { + return (descriptor & DENSE_CONTAINER_BIT) != 0; + } + + static boolean isSparse(int descriptor) { + return descriptor < DENSE_CONTAINER_SIZE; + } +} diff --git a/spark/src/main/java/org/apache/spark/sql/delta/deletionvectors/mumbling/PFOREncoding.java b/spark/src/main/java/org/apache/spark/sql/delta/deletionvectors/mumbling/PFOREncoding.java new file mode 100644 index 00000000000..c8fe1e85e92 --- /dev/null +++ b/spark/src/main/java/org/apache/spark/sql/delta/deletionvectors/mumbling/PFOREncoding.java @@ -0,0 +1,403 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.deletionvectors.mumbling; + +import java.nio.ByteBuffer; + +import com.google.common.base.Preconditions; + +/** + * Patched Frame of Reference (PFOR) encoding for arrays of unsigned byte values. + * + *

Implements the encoding described in Appendix A of the Mumbling bitmap specification. The + * input array is split into 256-value chunks (the last chunk may be shorter). Each chunk is + * independently encoded using 4 configuration values: + * + *

    + *
  • {@code b1}: number of bits stored in the primary array for every normalized value + *
  • {@code b2}: number of bits stored per exception value (normalized value of > b1 bits) + *
  • {@code e}: number of exceptions with more than b1 bits + *
  • {@code m}: chunk-local minimum value, subtracted from all values to normalize + *
+ * + *

Each chunk is stored as: + * + *

    + *
  • 3-byte header: {@code b1|b2} primary and exception bit widths (byte 0), {@code e} exception + * count (byte 1), {@code m} normalization base value (byte 2) + *
  • Primary array: the low {@code b1} bits of every normalized value, packed MSB-first ({@code + * b1 * n} bits, padded to a byte) + *
  • Exception offsets: chunk-relative positions of exception values ({@code e} bytes) + *
  • Exception values: the high {@code b2} bits of every exception value, packed MSB-first + * ({@code e * b2} bits, padded to a byte. + *
+ */ +class PFOREncoding { + private static final int CHUNK_SIZE = 256; + + private PFOREncoding() {} + + /** + * Encodes {@code count} values from an array of unsigned byte values. + * + * @param values unsigned byte values to encode + * @param count number of values to encode + * @return a {@link ByteBuffer} of the encoded values with position and limit set for reading + */ + static ByteBuffer encode(int[] values, int count) { + ByteBuffer out = ByteBuffer.allocate(estimateEncodedSize(count)); + int bytesWritten = encode(values, 0, out, 0, count); + out.limit(bytesWritten); + return out; + } + + /** + * Encode {@code count} unsigned byte values from {@code values} into a buffer. + * + *

The buffer's position and limit are not modified. + * + * @param values unsigned byte values to encode + * @param valueOffset starting offset of values to encode + * @param out buffer to write encoded values to + * @param outOffset starting offset in the output buffer + * @param count number of values to encode + * @return the number of bytes written to the buffer + */ + static int encode(int[] values, int valueOffset, ByteBuffer out, int outOffset, int count) { + // outOffset is relative to the buffer's position; check the encoded data fits + Preconditions.checkArgument(outOffset >= 0, "Cannot encode at negative offset: %s", outOffset); + Preconditions.checkArgument( + estimateEncodedSize(count) <= out.remaining() - outOffset, + "Cannot encode %s values to buffer with %s remaining space", + count, + out.remaining() - outOffset); + + int bytesWritten = 0; + int valuesEncoded = 0; + + while (valuesEncoded < count) { + int chunkLength = Math.min(CHUNK_SIZE, count - valuesEncoded); + bytesWritten += + encodeChunk( + values, valueOffset + valuesEncoded, out, outOffset + bytesWritten, chunkLength); + valuesEncoded += chunkLength; + } + + return bytesWritten; + } + + /** + * Decode to produce unsigned byte values. + * + *

Decodes starting at {@code encoded.position()} and does not modify the input buffer. + * + * @param encoded PFOR-encoded ByteBuffer produced by {@link #encode} + * @param count total number of values to decode + * @return decoded unsigned byte values + */ + static int[] decode(ByteBuffer encoded, int count) { + int[] out = new int[count]; + decode(encoded, 0, out, 0, count); + return out; + } + + /** + * Decode {@code count} unsigned bytes from a buffer into {@code out}. + * + *

This does not modify the input buffer. + * + * @param encoded a buffer containing encoded data + * @param offset starting offset of encoded values + * @param out an output value array + * @param outOffset starting offset in the output array + * @param count number of values to decode + * @return the number of bytes read from the encoded buffer + */ + static int decode(ByteBuffer encoded, int offset, int[] out, int outOffset, int count) { + Preconditions.checkArgument(offset >= 0, "Cannot decode at negative offset: %s", offset); + + int bytesRead = 0; + int valuesRead = 0; + + while (valuesRead < count) { + int chunkSize = Math.min(CHUNK_SIZE, count - valuesRead); + bytesRead += decodeChunk(encoded, offset + bytesRead, out, outOffset + valuesRead, chunkSize); + valuesRead += chunkSize; + } + + return bytesRead; + } + + /** + * Encode one chunk into {@code out} starting at absolute position {@code outPos}. + * + * @param values array containing source values to encode + * @param valueOffset starting index of values to encode + * @param out an output {@link ByteBuffer} + * @param outOffset starting index for output in the out buffer + * @param count number of values to encode + * @return the number of bytes written to the output buffer + */ + private static int encodeChunk( + int[] values, int valueOffset, ByteBuffer out, int outOffset, int count) { + Preconditions.checkArgument(count >= 0, "Invalid value count to encode: %s", count); + Preconditions.checkArgument( + valueOffset + count <= values.length, + "Cannot encode %s values starting at %s from int[%s]: not enough values", + count, + valueOffset, + values.length); + + // find base=min(values) for normalization + int base = min(values, valueOffset, count); + + // normalize by subtracting base + int[] normalized = new int[count]; + int setBits = 0; + int normalizedSetBits = 0; + for (int i = 0; i < count; i += 1) { + setBits |= values[valueOffset + i]; + normalized[i] = values[valueOffset + i] - base; + normalizedSetBits |= normalized[i]; + } + + Preconditions.checkArgument( + width(setBits) <= 8, + "Cannot encode values wider than 8 bits: %s bits needed", + width(setBits)); + + // Choose b1 to minimize total encoded data size (excluding 3-byte header) + int maxWidth = width(normalizedSetBits); + int b1 = chooseBitWidth(normalized, count, maxWidth); + int b2 = maxWidth - b1; + int excCount = countExceptions(normalized, count, b1); + + // check that there is enough space in the buffer for the encoded data + int requiredSize = encodedSize(count, b1, b2, excCount); + Preconditions.checkArgument( + outOffset + requiredSize <= out.remaining(), + "Cannot encode %s values (%s bytes) into buffer with %s remaining bytes", + count, + requiredSize, + out.remaining() - outOffset); + + // Special case: b1=8 means store original values as raw bytes with b2, e, and m set to 0. + if (b1 == 8) { + writeHeader(out, outOffset, b1, 0 /* b2 */, 0 /* excCount */, 0 /* m */); + return 3 + BitPacking.packBits(8, values, valueOffset, out, outOffset + 3, count); + } + + int bytesWritten = writeHeader(out, outOffset, b1, b2, excCount, base); + + // Primary array: low b1 bits of every value + bytesWritten += BitPacking.packBits(b1, normalized, 0, out, outOffset + bytesWritten, count); + + // b2 is the bit width of exception values: (maxWidth - b1) bits of each exception + if (excCount > 0) { + int[] excOffsets = new int[excCount]; + int[] excValues = new int[excCount]; + + // Collect exceptions (values that do not fit in b1 bits) + int excIndex = 0; + int threshold = 1 << b1; + for (int i = 0; i < count; i += 1) { + if (normalized[i] >= threshold) { + excOffsets[excIndex] = i; + excValues[excIndex] = normalized[i] >>> b1; + excIndex += 1; + } + } + + // Exception offsets (one byte per exception) + bytesWritten += + BitPacking.packBits(8, excOffsets, 0, out, outOffset + bytesWritten, excCount); + + // Exception values: remaining high b2 bits of each exception + bytesWritten += + BitPacking.packBits(b2, excValues, 0, out, outOffset + bytesWritten, excCount); + } + + return bytesWritten; + } + + /** + * Decode one chunk of encoded data, writing decoded values into an output array. + * + * @param data buffer containing source data to decode + * @param dataOffset starting index in the buffer to decode + * @param out an output {@link ByteBuffer} + * @param outOffset starting index for output in the out buffer + * @param count number of values to decode + * @return the number of bytes read from {@code data} + */ + private static int decodeChunk( + ByteBuffer data, int dataOffset, int[] out, int outOffset, int count) { + Preconditions.checkArgument(count >= 0, "Invalid value count to decode: %s", count); + Preconditions.checkArgument( + outOffset + count <= out.length, + "Cannot decode %s values starting at %s into int[%s]: not enough space", + count, + outOffset, + out.length); + + int b1 = ByteBuffers.readByte(data, dataOffset) & 0x0F; + int b2 = (ByteBuffers.readByte(data, dataOffset) >>> 4) & 0x0F; + int excCount = ByteBuffers.readByte(data, dataOffset + 1); + int base = ByteBuffers.readByte(data, dataOffset + 2); + int bytesRead = 3; + + // after reading the header, check that the full chunk is present + int expectedSize = encodedSize(count, b1, b2, excCount); + Preconditions.checkArgument( + dataOffset + expectedSize <= data.remaining(), + "Cannot decode %s values (%s bytes) from buffer with %s remaining bytes", + count, + expectedSize, + data.remaining() - dataOffset); + + // Read primary array: low b1 bits of each value + bytesRead += BitPacking.unpackBits(b1, data, dataOffset + bytesRead, out, outOffset, count); + + // Read exceptions and update output values + if (excCount > 0) { + int[] excOffsets = new int[excCount]; + int[] excValues = new int[excCount]; + int excListOffset = dataOffset + bytesRead; + int excDataOffset = dataOffset + bytesRead + excCount; + + // Read exception indexes + bytesRead += BitPacking.unpackBits(8, data, excListOffset, excOffsets, 0, excCount); + + // Read exception values and patch the primary values + bytesRead += BitPacking.unpackBits(b2, data, excDataOffset, excValues, 0, excCount); + + // Update output values + for (int i = 0; i < excCount; i += 1) { + out[outOffset + excOffsets[i]] |= excValues[i] << b1; + } + } + + // Add back the chunk minimum + for (int i = 0; i < count; i += 1) { + out[outOffset + i] += base; + } + + return bytesRead; + } + + private static int writeHeader( + ByteBuffer out, int outOffset, int b1, int b2, int excCount, int base) { + // Header: b1 in low nibble, b2 in high nibble, then e, then m + ByteBuffers.writeByte(out, (b2 << 4) | (b1 & 0b1111), outOffset); + ByteBuffers.writeByte(out, excCount, outOffset + 1); + ByteBuffers.writeByte(out, base, outOffset + 2); + + return 3; + } + + /** + * Choose the primary bit width {@code b1} that minimizes total encoded chunk size. + * + *

Larger width is preferred on ties to reduce the number of exceptions. + * + * @param normalized value array to encode, after normalization + * @param length number of values in the array to encode + * @param maxWidth the largest bit width of normalized values + * @return the chosen width + */ + private static int chooseBitWidth(int[] normalized, int length, int maxWidth) { + int bestWidth = 0; + int bestSize = Integer.MAX_VALUE; + + for (int candidateWidth = 0; candidateWidth <= maxWidth; candidateWidth += 1) { + int excCount = countExceptions(normalized, length, candidateWidth); + int b2 = maxWidth - candidateWidth; + int size = byteWidth(length * candidateWidth) + excCount + byteWidth(excCount * b2); + + if (size <= bestSize) { + bestSize = size; + bestWidth = candidateWidth; + } + } + + return bestWidth; + } + + private static int countExceptions(int[] normalized, int length, int bitWidth) { + if (bitWidth >= 8) { + return 0; + } + + int excCount = 0; + int threshold = 1 << bitWidth; + for (int i = 0; i < length; i += 1) { + if (normalized[i] >= threshold) { + excCount += 1; + } + } + return excCount; + } + + /** + * Return the lowest byte value from the array slice [start, start + length). + * + *

If length is < 1, the result will be larger than Byte.MAX_VALUE. + * + * @param values array of values + * @param start starting index + * @param length number of values to check + * @return the min of the values in the array slice + */ + private static int min(int[] values, int start, int length) { + // Use min > Byte.MAX_VALUE to signal no min (length < 1) + int min = 256; + for (int i = start; i < start + length; i += 1) { + if (values[i] < min) { + min = values[i]; + } + } + + return min; + } + + /** Returns the number of bytes required in the worst case to encode {@code valueCount} values. */ + static int estimateEncodedSize(int valueCount) { + // Worst-case per chunk is b1=8: 3-byte header + 1 byte per value. Any other b1 chosen by the + // encoder costs <= n bytes of data (otherwise b1=8 would have been selected instead). + int numChunks = ceilDiv(valueCount, CHUNK_SIZE); + return 3 * numChunks + valueCount; + } + + /** Returns the number of bytes required to encode a chunk of values. */ + private static int encodedSize(int count, int b1, int b2, int excCount) { + return 3 + byteWidth(b1 * count) + excCount + byteWidth(b2 * excCount); + } + + /** Returns the number of bits required to represent {@code v} (0 for v=0). */ + static int width(int value) { + return 32 - Integer.numberOfLeadingZeros(value); + } + + private static int byteWidth(int bits) { + return ceilDiv(bits, 8); + } + + @SuppressWarnings("checkstyle:ParameterName") + private static int ceilDiv(int a, int b) { + return (a + b - 1) / b; + } +} diff --git a/spark/src/main/resources/error/delta-error-classes.json b/spark/src/main/resources/error/delta-error-classes.json index f1b3e260234..df1f057d1fa 100644 --- a/spark/src/main/resources/error/delta-error-classes.json +++ b/spark/src/main/resources/error/delta-error-classes.json @@ -154,8 +154,30 @@ }, "DELTA_CANNOT_CHANGE_DATA_TYPE" : { "message" : [ - "Cannot change data type: " + "Cannot change the table schema." ], + "subClass" : { + "ADD_NON_NULLABLE_COLUMN" : { + "message" : [ + "Adding non-nullable column is not supported." + ] + }, + "CHANGE_DATA_TYPE" : { + "message" : [ + "Changing the data type of column from to is not supported." + ] + }, + "DROP_COLUMNS" : { + "message" : [ + "Dropping column(s) is not supported." + ] + }, + "TIGHTEN_NULLABILITY" : { + "message" : [ + "Tightening the nullability of column is not supported." + ] + } + }, "sqlState" : "429BQ" }, "DELTA_CANNOT_CHANGE_LOCATION" : { @@ -711,6 +733,12 @@ ], "sqlState" : "42P01" }, + "DELTA_COMMIT_UNEXPECTED_DATA_CHANGE" : { + "message" : [ + "Operation committed a with dataChange = , but expected ." + ], + "sqlState" : "XXKDS" + }, "DELTA_COMPLEX_TYPE_COLUMN_CONTAINS_NULL_TYPE" : { "message" : [ " Found nested NullType in column which is of . Delta doesn't support writing NullType in complex types." @@ -801,6 +829,12 @@ ], "sqlState" : "56038" }, + "DELTA_CONFLICTING_METADATA_DOMAIN" : { + "message" : [ + "A concurrent transaction added a conflicting metadata domain ." + ], + "sqlState" : "2D521" + }, "DELTA_CONFLICT_SET_COLUMN" : { "message" : [ "There is a conflict from these SET columns: ." @@ -1283,7 +1317,7 @@ "Using this SQL configuration could lead to accidental data loss, therefore we do", "not recommend the use of this flag unless this is for testing purposes." ], - "sqlState" : "XXKDS" + "sqlState" : "22KD1" }, "DELTA_FEATURES_PROTOCOL_METADATA_MISMATCH" : { "message" : [ @@ -1673,8 +1707,76 @@ }, "DELTA_ILLEGAL_OPTION" : { "message" : [ - "Invalid value '' for option '', " + "Invalid value '' for option ''." ], + "subClass" : { + "DYNAMIC_PARTITION_OVERWRITE_ONLY" : { + "message" : [ + "This option can be specified only in dynamic partition overwrite mode." + ] + }, + "INVALID_PARQUET_FORMAT_VERSION" : { + "message" : [ + "", + "" + ] + }, + "INVALID_REORG_PARQUET_FORMAT_VERSION" : { + "message" : [ + "This is not a valid target Parquet format version for a REORG command." + ] + }, + "MUST_BE_INTEGER" : { + "message" : [ + "Must be an integer." + ] + }, + "MUST_BE_NON_NEGATIVE_NUMBER" : { + "message" : [ + "Must be a non-negative number." + ] + }, + "MUST_BE_ONE_OF" : { + "message" : [ + "Must be one of the following: ." + ] + }, + "MUST_BE_POSITIVE_NUMBER" : { + "message" : [ + "Must be a positive number." + ] + }, + "MUST_BE_SIZE_CONFIGURATION" : { + "message" : [ + "Must be a size configuration such as '10g'." + ] + }, + "NO_EMPTY_COLUMN_NAMES" : { + "message" : [ + "Must not contain empty column names." + ] + }, + "PARQUET_COMPRESSION_CODEC_CONFLICT" : { + "message" : [ + "Conflicts with the table property '' set to ''." + ] + }, + "PARQUET_WRITER_VERSION_CONFLICT" : { + "message" : [ + "Conflicts with the table property '' set to '', which resolves to writer version ''." + ] + }, + "SCHEMA_MODE_WITH_TIME_TRAVEL" : { + "message" : [ + "This schema mode cannot be used with time travel options." + ] + }, + "UNRECOGNIZED_NAMED_ARGUMENT" : { + "message" : [ + "Unrecognized named argument for function ''. Valid named arguments: ." + ] + } + }, "sqlState" : "42616" }, "DELTA_ILLEGAL_USAGE" : { @@ -1839,8 +1941,20 @@ }, "DELTA_INVALID_IDEMPOTENT_WRITES_OPTIONS" : { "message" : [ - "Invalid options for idempotent Dataframe writes: " + "Invalid options for idempotent DataFrame writes:" ], + "subClass" : { + "MISSING_DATAFRAME_WRITE_OPTIONS" : { + "message" : [ + "Both txnVersion and txnAppId must be specified for idempotent DataFrame writes." + ] + }, + "MISSING_SESSION_CONFS" : { + "message" : [ + "Both spark.databricks.delta.write.txnAppId and spark.databricks.delta.write.txnVersion must be specified for idempotent Delta writes." + ] + } + }, "sqlState" : "42616" }, "DELTA_INVALID_INTERVAL" : { @@ -2313,8 +2427,20 @@ }, "DELTA_NON_PARTITION_COLUMN_ABSENT" : { "message" : [ - "Data written into Delta needs to contain at least one non-partitioned column.

" + "Data written into Delta needs to contain at least one non-partitioned column." ], + "subClass" : { + "ALL_PARTITION_COLUMNS" : { + "message" : [ + "All of the provided columns are partition columns." + ] + }, + "NULL_TYPE_COLUMNS_DROPPED" : { + "message" : [ + "Columns which are of NullType have been dropped." + ] + } + }, "sqlState" : "KD005" }, "DELTA_NON_PARTITION_COLUMN_REFERENCE" : { @@ -2557,8 +2683,23 @@ }, "DELTA_PROTOCOL_CHANGED" : { "message" : [ - "ProtocolChangedException: The protocol version of the Delta table has been changed by a concurrent update. \nRefer to for more details." + "ProtocolChangedException: The protocol version of the Delta table has been changed by a concurrent update. Refer to for more details." ], + "subClass" : { + "CONFLICTING_COMMIT" : { + "message" : [ + "Conflicting commit: ", + "" + ] + }, + "WRITE_TO_EMPTY_DIRECTORY" : { + "message" : [ + "This happens when multiple writers are writing to an empty directory. Creating the table ahead of time will avoid this conflict.", + "Conflicting commit: ", + "" + ] + } + }, "sqlState" : "2D521" }, "DELTA_PROTOCOL_PROPERTY_NOT_INT" : { @@ -2613,6 +2754,19 @@ ], "sqlState" : "44000" }, + "DELTA_REPLACE_PARTITIONED_BY_ON_UNPARTITIONED_TABLE" : { + "message" : [ + "ALTER TABLE REPLACE PARTITIONED BY can only be applied to a partitioned table." + ], + "sqlState" : "42000" + }, + "DELTA_REPLACE_PARTITIONED_BY_PARTITION_COLUMNS_NOT_MATERIALIZED" : { + "message" : [ + "Cannot replace the partitioning of the table because the partition column(s) are not stored in data file(s), for example: .", + "Rewrite these files so that they contain the partition column values before running ALTER TABLE REPLACE PARTITIONED BY." + ], + "sqlState" : "55000" + }, "DELTA_REPLACE_TABLE_WITH_CATALOG_MANAGED_NOT_SUPPORTED" : { "message" : [ "REPLACE TABLE cannot enable catalog-managed on the existing non-catalog-managed table . Upgrade or recreate the table instead." @@ -2627,9 +2781,21 @@ }, "DELTA_REPLACE_WHERE_MISMATCH" : { "message" : [ - "Written data does not conform to partial table overwrite condition or constraint ''.", - "" + "Written data does not conform to partial table overwrite condition or constraint ''." ], + "subClass" : { + "INVALID_PARTITIONS" : { + "message" : [ + "Invalid data would be written to partitions ." + ] + }, + "INVARIANT_VIOLATION" : { + "message" : [ + "", + "" + ] + } + }, "sqlState" : "44000" }, "DELTA_REPLACE_WHERE_WITH_DYNAMIC_PARTITION_OVERWRITE" : { @@ -2859,13 +3025,6 @@ ], "sqlState" : "42KD4" }, - "DELTA_STREAMING_INITIAL_SNAPSHOT_TOO_LARGE" : { - "message" : [ - "Initial snapshot for Delta streaming at table '' (version ) contains files, which exceeds the maximum allowed files.", - "" - ], - "sqlState" : "54000" - }, "DELTA_STREAMING_METADATA_EVOLUTION" : { "message" : [ "The schema, table configuration or protocol of your Delta table has changed during streaming.", @@ -2931,12 +3090,12 @@ "DELTA_STREAMING_SCHEMA_MISMATCH_ON_RESTART" : { "message" : [ "The streaming query's schema does not match the current table schema.", + "", "Query schema (from analysis): ", "", "Current table schema: ", "", - "The table schema has changed since this streaming query was created.", - "Please create a new streaming DataFrame to pick up the current schema." + "The table schema has changed since this streaming query was created. Create a new streaming DataFrame to pick up the current schema." ], "sqlState" : "KD007" }, @@ -3147,14 +3306,44 @@ }, "DELTA_UNIVERSAL_FORMAT_CONVERSION_FAILED" : { "message" : [ - "Failed to convert the table version to the universal format . " + "Failed to convert the table version to the universal format ." ], + "subClass" : { + "CAUSED_BY_ERROR" : { + "message" : [ + "", + "" + ] + }, + "UNEXPECTED_PARTITION_DATA_TYPE" : { + "message" : [ + "Unexpected partition data type ." + ] + } + }, "sqlState" : "KD00E" }, "DELTA_UNIVERSAL_FORMAT_VIOLATION" : { "message" : [ - "The validation of Universal Format () has failed: " + "The validation of Universal Format () has failed:" ], + "subClass" : { + "HUDI_DELETE_VECTORS_NOT_SUPPORTED" : { + "message" : [ + "Requires deletion vectors to be disabled." + ] + }, + "HUDI_UNSUPPORTED_DATA_TYPE" : { + "message" : [ + "DataType is not currently supported." + ] + }, + "ICEBERG_COMPAT_REQUIRED" : { + "message" : [ + "Requires IcebergCompat to be explicitly enabled in order for Universal Format (Iceberg) to be enabled on an existing table. Supported versions are IcebergCompatV1 and IcebergCompatV2." + ] + } + }, "sqlState" : "KD00E" }, "DELTA_UNKNOWN_CONFIGURATION" : { @@ -3220,13 +3409,35 @@ }, "DELTA_UNSUPPORTED_ALTER_TABLE_REPLACE_COL_OP" : { "message" : [ - "Unsupported ALTER TABLE REPLACE COLUMNS operation. Reason:
", - "", + "Unsupported ALTER TABLE REPLACE COLUMNS operation.", "Failed to change schema from:", "", "to:", - "" + "", + "Reason:" ], + "subClass" : { + "ADD_NON_NULLABLE_COLUMN" : { + "message" : [ + "Adding non-nullable column is not supported." + ] + }, + "CHANGE_DATA_TYPE" : { + "message" : [ + "Changing the data type of column from to is not supported." + ] + }, + "DROP_COLUMNS" : { + "message" : [ + "Dropping column(s) is not supported." + ] + }, + "TIGHTEN_NULLABILITY" : { + "message" : [ + "Tightening the nullability of column is not supported." + ] + } + }, "sqlState" : "0AKDC" }, "DELTA_UNSUPPORTED_CATALOG_MANAGED_TABLE_CREATION" : { @@ -3323,8 +3534,17 @@ }, "DELTA_UNSUPPORTED_DROP_COLUMN" : { "message" : [ - "DROP COLUMN is not supported for your Delta table. " + "DROP COLUMN is not supported for your Delta table." ], + "subClass" : { + "ENABLE_COLUMN_MAPPING" : { + "message" : [ + "Enable Column Mapping on your Delta table with mapping mode 'name' by running:", + "ALTER TABLE table_name SET TBLPROPERTIES ('delta.columnMapping.mode' = 'name').", + "If your table is not on the required protocol version it will be upgraded. Column mapping requires at least protocol (, )." + ] + } + }, "sqlState" : "0AKDC" }, "DELTA_UNSUPPORTED_DROP_NESTED_COLUMN_FROM_NON_STRUCT_TYPE" : { @@ -3468,8 +3688,17 @@ }, "DELTA_UNSUPPORTED_RENAME_COLUMN" : { "message" : [ - "Column rename is not supported for your Delta table. " + "Column rename is not supported for your Delta table." ], + "subClass" : { + "ENABLE_COLUMN_MAPPING" : { + "message" : [ + "Enable Column Mapping on your Delta table with mapping mode 'name' by running:", + "ALTER TABLE table_name SET TBLPROPERTIES ('delta.columnMapping.mode' = 'name').", + "If your table is not on the required protocol version it will be upgraded. Column mapping requires at least protocol (, )." + ] + } + }, "sqlState" : "0AKDC" }, "DELTA_UNSUPPORTED_SCHEMA_DURING_READ" : { diff --git a/spark/src/main/scala-shims/spark-4.0-4.1/AllowedUserProvidedExpressionsShims.scala b/spark/src/main/scala-shims/spark-4.0-4.1/AllowedUserProvidedExpressionsShims.scala new file mode 100644 index 00000000000..ac86373bda4 --- /dev/null +++ b/spark/src/main/scala-shims/spark-4.0-4.1/AllowedUserProvidedExpressionsShims.scala @@ -0,0 +1,22 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.catalyst.expressions + +/** Additional classes allowed during user-provided expression validation in Spark 4.0 and 4.1. */ +object AllowedUserProvidedExpressionsShims { + val additionalExpressions: Set[Class[_]] = Set.empty +} diff --git a/spark/src/main/scala-shims/spark-4.2/AllowedUserProvidedExpressionsShims.scala b/spark/src/main/scala-shims/spark-4.2/AllowedUserProvidedExpressionsShims.scala new file mode 100644 index 00000000000..005cc3485d2 --- /dev/null +++ b/spark/src/main/scala-shims/spark-4.2/AllowedUserProvidedExpressionsShims.scala @@ -0,0 +1,24 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.catalyst.expressions + +/** Additional classes allowed during user-provided expression validation in Spark 4.2. */ +object AllowedUserProvidedExpressionsShims { + val additionalExpressions: Set[Class[_]] = Set( + classOf[TypedNullLiteral], + classOf[TimestampAddInterval]) +} diff --git a/spark/src/main/scala/com/databricks/spark/util/TestBarrier.scala b/spark/src/main/scala/com/databricks/spark/util/TestBarrier.scala new file mode 100644 index 00000000000..2dc3ae0212e --- /dev/null +++ b/spark/src/main/scala/com/databricks/spark/util/TestBarrier.scala @@ -0,0 +1,182 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.databricks.spark.util + +import java.util.concurrent.{Semaphore, TimeoutException, TimeUnit} + +import scala.collection.mutable + + +/** This class is used to wait for threads blocked on `label` and unblock these threads. */ +class TestBarrier private[util] ( + val label: String, + private val semaphore: Semaphore, + private var hitCount: Int = 1) { + /** The threads blocked by this TestBarrier object. */ + private val blockingThreads = new mutable.HashSet[Long]() + /** + * True if this barrier is enabled. If it is disabled, any thread that enters this barrier + * wouldn't be blocked. + */ + private var isEnabled = true + private var blockingNewThreads = true + /** The interval a thread need to sleep before checking the current status of barrier. */ + private final val DEFAULT_TIMEOUT_DURATION_MS = 100 + private final val DEFAULT_WAIT_TIMEOUT_MS = 120000 + + def getNumBlockedThreads: Int = synchronized { blockingThreads.size } + + private[util] def isBlockingNewThreads: Boolean = synchronized { blockingNewThreads } + + /** Wait until at least `numThreads` threads are blocked. */ + def waitUntilBlocked( + numThreads: Int = 1, + checkIntervalMs: Long = DEFAULT_TIMEOUT_DURATION_MS, + timeout: Long = DEFAULT_WAIT_TIMEOUT_MS): Unit = { + val startTimeMs = System.currentTimeMillis() + var keepSpinning = true + while (keepSpinning) { + val numBlocked = getNumBlockedThreads + keepSpinning = numBlocked < numThreads + if (keepSpinning) { + if (System.currentTimeMillis() - startTimeMs > timeout) { + throw new TimeoutException( + s"Only $getNumBlockedThreads(expect $numThreads) threads are blocked " + + s"after $timeout milliseconds.") + } + Thread.sleep(checkIntervalMs) + } + } + } + + /** Release all threads that are blocked by this TestBarrier. */ + def releaseAll(): Unit = synchronized { + semaphore.release(getNumBlockedThreads) + blockingThreads.clear() + isEnabled = false + } + + /** Release one thread that are blocked by this TestBarrier. */ + def releaseOne(): Unit = synchronized { + // Keep the barrier enabled, only wake a single waiter. + semaphore.release(1) + // The awakened thread will remove itself from `blockingThreads` in `enter`'s `finally`. + } + + /** + * Disable the barrier so that any thread that enters this barrier wouldn't be blocked, but the + * threads that are already blocked by this barrier still need to be released. + */ + private[util] def disableWithoutReleasing(): Unit = synchronized { blockingNewThreads = false } + + def enable(): Unit = synchronized { + isEnabled = true + // Drain any leftover permits from previous barrier use to prevent threads from + // acquiring spurious permits when the barrier is reused. + semaphore.drainPermits() + } + + /** Register current thread to this TestBarrier and block on it. */ + private[util] def enter(): Unit = { + val threadId = Thread.currentThread().getId + // To avoid this method racing with `releaseAll()`, this method unblocks itself if isEnabled is + // false. + synchronized { + if (isEnabled) { + blockingThreads.add(threadId) + } else { + // Barrier is disabled, return directly. + return + } + } + try { + var done = false + while (!done) { + val stillEnabled = synchronized { isEnabled } + if (!stillEnabled) { + // Barrier disabled globally, every blocked thread can exit. + done = true + } else if (semaphore.tryAcquire(DEFAULT_TIMEOUT_DURATION_MS, TimeUnit.MILLISECONDS)) { + // This thread got a permit to exit. + done = true + } + // else, no permit yet, loop again + } + } finally { + synchronized { + // This is safe with releaseAll() because removing non-existing threadId from the set + // is no-op. + blockingThreads.remove(threadId) + } + } + } + + /** + * Hit the barrier for the given times. Will return true if the barrier hitCount is zero or less, + * which means the barrier can be destroyed. + */ + def hit(times: Int = 1): Boolean = synchronized { + hitCount -= times + return hitCount <= 0 + } +} + +/** Registry for labeled barriers shared by test-specific entry points. */ +object TestBarrierRegistry { + @volatile private var enabled = false + private val barriersByLabel = new mutable.HashMap[String, TestBarrier]() + + def createBarrier(label: String, hitCount: Int, isTesting: Boolean): TestBarrier = synchronized { + assert(isTesting) + enabled = true + val barrier = new TestBarrier(label, new Semaphore(0), hitCount) + barriersByLabel.put(label, barrier) + barrier + } + + def waitIfEnabled(label: String, isTesting: Boolean): Unit = { + if (!enabled || !isTesting) return + val barrier = synchronized { barriersByLabel.get(label) } + barrier.filter(_.isBlockingNewThreads).foreach(_.enter()) + } + + def disableBarrierWithoutReleasing(label: String): Unit = synchronized { + barriersByLabel.get(label).foreach(_.disableWithoutReleasing()) + } + + def hitBarrier(label: String, times: Int = 1): Unit = synchronized { + barriersByLabel.get(label).foreach { barrier => + if (barrier.hit(times)) { + destroyBarrier(label) + } + } + } + + def enableBarrier(label: String): Unit = synchronized { + barriersByLabel.get(label).foreach(_.enable()) + } + + def destroyBarrier(label: String): Unit = synchronized { + barriersByLabel.remove(label).foreach(_.releaseAll()) + enabled = barriersByLabel.nonEmpty + } + + def isBarrierEnabled(label: String): Boolean = synchronized { + barriersByLabel.contains(label) + } +} + diff --git a/spark/src/main/scala/io/delta/exceptions/DeltaConcurrentExceptions.scala b/spark/src/main/scala/io/delta/exceptions/DeltaConcurrentExceptions.scala index bb0114e1aa1..84bb2ef412a 100644 --- a/spark/src/main/scala/io/delta/exceptions/DeltaConcurrentExceptions.scala +++ b/spark/src/main/scala/io/delta/exceptions/DeltaConcurrentExceptions.scala @@ -88,14 +88,33 @@ class MetadataChangedException private ( * @since 1.0.0 */ @Evolving -class ProtocolChangedException(message: String) +class ProtocolChangedException private ( + message: String, + errorClass: String, + messageParameters: Array[String] = Array.empty) extends org.apache.spark.sql.delta.ProtocolChangedException(message) with DeltaThrowable { + def this(message: String) = this(message, "DELTA_PROTOCOL_CHANGED", Array.empty) def this(messageParameters: Array[String]) = { - this(DeltaThrowableHelper.getMessage("DELTA_PROTOCOL_CHANGED", messageParameters)) + this( + DeltaThrowableHelper.getMessage("DELTA_PROTOCOL_CHANGED", messageParameters), + "DELTA_PROTOCOL_CHANGED", + messageParameters) } - override def getErrorClass: String = "DELTA_PROTOCOL_CHANGED" + override def getErrorClass: String = errorClass override def getMessage: String = message + + override def getMessageParameters: java.util.Map[String, String] = { + DeltaThrowableHelper.getMessageParameters(errorClass, errorSubClass = null, messageParameters) + } +} + +object ProtocolChangedException { + def apply(subClass: String, messageParameters: Array[String]): ProtocolChangedException = { + val errorClass = s"DELTA_PROTOCOL_CHANGED.$subClass" + val message = DeltaThrowableHelper.getMessage(errorClass, messageParameters) + new ProtocolChangedException(message, errorClass, messageParameters) + } } /** diff --git a/spark/src/main/scala/io/delta/sql/parser/DeltaSqlParser.scala b/spark/src/main/scala/io/delta/sql/parser/DeltaSqlParser.scala index 567017a89fa..6dba173f96e 100644 --- a/spark/src/main/scala/io/delta/sql/parser/DeltaSqlParser.scala +++ b/spark/src/main/scala/io/delta/sql/parser/DeltaSqlParser.scala @@ -43,7 +43,7 @@ import java.util.Locale import scala.collection.JavaConverters._ import org.apache.spark.sql.catalyst.TimeTravel -import org.apache.spark.sql.delta.skipping.clustering.temp.{AlterTableClusterBy, ClusterByParserUtils, ClusterByPlan, ClusterBySpec} +import org.apache.spark.sql.delta.skipping.clustering.temp.{AlterTableClusterBy, AlterTableReplacePartitionedByWithClusterBy, ClusterByParserUtils, ClusterByPlan, ClusterBySpec} import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.commands._ @@ -638,6 +638,20 @@ class DeltaSqlAstBuilder extends DeltaSqlBaseBaseVisitor[AnyRef] { } } + /** + * Parse `ALTER TABLE ... REPLACE PARTITIONED BY WITH CLUSTER BY (col1, ...) | NONE`. + */ + override def visitAlterTableReplacePartitionedBy( + ctx: AlterTableReplacePartitionedByContext): LogicalPlan = withOrigin(ctx) { + val table = UnresolvedTable( + ctx.table.identifier.asScala.map(_.getText).toSeq, + "ALTER TABLE ... REPLACE PARTITIONED BY WITH CLUSTER BY") + val clusterBySpec = Option(ctx.clusterBySpec()).map { spec => + ClusterBySpec(spec.interleave.asScala.map(_.identifier.asScala.map(_.getText).toSeq).toSeq) + } + AlterTableReplacePartitionedByWithClusterBy(table, clusterBySpec) + } + protected def typedVisit[T](ctx: ParseTree): T = { ctx.accept(this).asInstanceOf[T] } diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/AllowedUserProvidedExpressions.scala b/spark/src/main/scala/org/apache/spark/sql/delta/AllowedUserProvidedExpressions.scala index c0fafb01882..734daf67ef7 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/AllowedUserProvidedExpressions.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/AllowedUserProvidedExpressions.scala @@ -334,9 +334,12 @@ object AllowedUserProvidedExpressions { expression[StructsToCsv]("to_csv"), // Special expressions that are not built-in expressions. + classOf[With], + classOf[CommonExpressionDef], + classOf[CommonExpressionRef], expression[AttributeReference]("col"), expression[Literal]("lit") - ) + ) ++ AllowedUserProvidedExpressionsShims.additionalExpressions val checkConstraintExpressions: Set[Class[_]] = Set( expression[Contains]("contains"), diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/CatalogManagedTableMaintenanceOperation.scala b/spark/src/main/scala/org/apache/spark/sql/delta/CatalogManagedTableMaintenanceOperation.scala new file mode 100644 index 00000000000..7ad7e35d469 --- /dev/null +++ b/spark/src/main/scala/org/apache/spark/sql/delta/CatalogManagedTableMaintenanceOperation.scala @@ -0,0 +1,40 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta + +object CatalogManagedTableMaintenanceOperation { + /** Storage property carrying the catalog-permitted maintenance operations, comma-separated. */ + val ALLOWED_OPERATIONS_PROPERTY = "delta.clientMaintenanceOperations" + + /** + * Deletes data files that no retained table version needs. It deletes the files only after the + * retention period. + */ + val DATA_CLEANUP = "DATA_CLEANUP" + + /** + * Rewrites data files that the current table version uses. It changes how the files store and + * group the data. It does not change the table's current rows. + */ + val DATA_REORGANIZATION = "DATA_REORGANIZATION" + + /** + * Deletes expired files from `_delta_log`. It keeps the files that are necessary to read retained + * table versions. + */ + val METADATA_CLEANUP = "METADATA_CLEANUP" +} diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/CheckpointProvider.scala b/spark/src/main/scala/org/apache/spark/sql/delta/CheckpointProvider.scala index ba7e249e35c..42f93c5e50e 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/CheckpointProvider.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/CheckpointProvider.scala @@ -23,7 +23,7 @@ import scala.util.control.NonFatal import org.apache.spark.sql.delta.ClassicColumnConversions._ import org.apache.spark.sql.delta.DataFrameUtils import org.apache.spark.sql.delta.DeltaLogFileIndex.COMMIT_VERSION_COLUMN -import org.apache.spark.sql.delta.SnapshotManagement.checkpointV2ThreadPool +import org.apache.spark.sql.delta.SnapshotManagement.checkpointThreadPool import org.apache.spark.sql.delta.actions._ import org.apache.spark.sql.delta.expressions.EncodeNestedVariantAsZ85String import org.apache.spark.sql.delta.metering.DeltaLogging @@ -244,7 +244,7 @@ object CheckpointProvider extends DeltaLogging { // a lazy checkpoint provider wrapping the future. we won't wait on the future unless/until // somebody calls a complete checkpoint provider method. val deltaLog = snapshotDescriptor.deltaLog - val future = checkpointV2ThreadPool.submitNonFateSharing { spark: SparkSession => + val future = checkpointThreadPool.submitNonFateSharing { spark: SparkSession => provider.readV2Actions( spark, deltaLog.store, deltaLog.newDeltaHadoopConf(), deltaLog.options) } @@ -571,7 +571,7 @@ case class UninitializedV2CheckpointProvider( val nonFateSharingCheckpointReadFuture : NonFateSharingFuture[(Option[CheckpointMetadata], Seq[SidecarFile])] = { - checkpointV2ThreadPool.submitNonFateSharing { spark: SparkSession => + checkpointThreadPool.submitNonFateSharing { spark: SparkSession => loadV2Actions(spark, logStore, hadoopConf, deltaLogOptions) } } @@ -712,7 +712,7 @@ object V2CheckpointProvider { def getSidecarSchemaFetcher: () => Option[StructType] = { val sidecarSchemaFromMetadata = checkpointMetadata.sidecarFileSchema val nonFateSharingSidecarSchemaFuture: NonFateSharingFuture[Option[StructType]] = { - checkpointV2ThreadPool.submitNonFateSharing { spark: SparkSession => + checkpointThreadPool.submitNonFateSharing { spark: SparkSession => sidecarFiles.headOption.map { sidecarFile => val sidecarFileStatus = sidecarFile.toFileStatus(uninitializedV2LikeCheckpointProvider.logPath) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/Checkpoints.scala b/spark/src/main/scala/org/apache/spark/sql/delta/Checkpoints.scala index c56fd71772a..e3b0d08884b 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/Checkpoints.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/Checkpoints.scala @@ -18,20 +18,23 @@ package org.apache.spark.sql.delta import java.io.FileNotFoundException import java.util.UUID +import java.util.concurrent.Future import scala.collection.mutable +import scala.concurrent.duration.Duration import scala.math.Ordering.Implicits._ import scala.util.Try import scala.util.control.NonFatal // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta.ClassicColumnConversions._ -import org.apache.spark.sql.delta.actions.{Action, CheckpointMetadata, Metadata, SidecarFile, SingleAction} +import org.apache.spark.sql.delta.actions.{Action, Checkpoint, CheckpointMetadata, CommitInfo, LastManifestCommit, Metadata, SidecarFile, SingleAction} +import org.apache.spark.sql.delta.amt.{AMTCheckpointProvider, AMTTriggerMode, AMTUtils, AMTWriteResult, AMTWriterManager} import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.storage.LogStore -import org.apache.spark.sql.delta.util.{DeltaFileOperations, DeltaLogGroupingIterator, FileNames} +import org.apache.spark.sql.delta.util.{DeltaCommitFileProvider, DeltaFileOperations, DeltaLogGroupingIterator, FileNames} import org.apache.spark.sql.delta.util.{Utils => DeltaUtils} import org.apache.spark.sql.delta.util.FileNames._ import org.apache.spark.sql.delta.util.JsonUtils @@ -40,7 +43,6 @@ import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileStatus, FileSystem, Path} import org.apache.hadoop.mapred.{JobConf, TaskAttemptContextImpl, TaskAttemptID} import org.apache.hadoop.mapreduce.{Job, TaskType} -import org.apache.parquet.hadoop.ParquetOutputFormat import org.apache.spark.TaskContext import org.apache.spark.internal.MDC @@ -60,7 +62,7 @@ import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat import org.apache.spark.sql.functions.{coalesce, col, struct, when} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.StructType -import org.apache.spark.util.{LongAccumulator, SerializableConfiguration, Utils} +import org.apache.spark.util.{LongAccumulator, SerializableConfiguration, ThreadUtils, Utils} /** * A class to help with comparing checkpoints with each other, where we may have had concurrent @@ -75,18 +77,25 @@ case class CheckpointInstance( version: Long, format: CheckpointInstance.Format, fileName: Option[String] = None, - numParts: Option[Int] = None) extends Ordered[CheckpointInstance] { + numParts: Option[Int] = None, + manifestCommitVersion: Option[Long] = None) extends Ordered[CheckpointInstance] { - // Assert that numParts are present when checkpoint format is Format.WITH_PARTS. + import CheckpointInstance.Format + + // Assert that numParts are present when checkpoint format is Format.WITH_PARTS or Format.AMT. // For other formats, numParts must be None. - require((format == CheckpointInstance.Format.WITH_PARTS) == numParts.isDefined, - s"numParts ($numParts) must be present for checkpoint format" + - s" ${CheckpointInstance.Format.WITH_PARTS.name}") + require(Set(Format.WITH_PARTS, Format.AMT).contains(format) == numParts.isDefined, + s"numParts ($numParts) must be present for checkpoint formats " + + s"${Format.WITH_PARTS.name} and ${Format.AMT.name}") // Assert that filePath is present only when checkpoint format is Format.V2. // For other formats, filePath must be None. - require((format == CheckpointInstance.Format.V2) == fileName.isDefined, - s"fileName ($fileName) must be present for checkpoint format" + - s" ${CheckpointInstance.Format.V2.name}") + require((format == Format.V2) == fileName.isDefined, + s"fileName ($fileName) must be present for checkpoint format ${Format.V2.name}") + // Assert that lastManifestCommit is present only when checkpoint format is Format.AMT. + // For other formats, lastManifestCommit must be None. + require((format == CheckpointInstance.Format.AMT) == manifestCommitVersion.isDefined, + s"manifestCommitVersion ($manifestCommitVersion) must only be present for checkpoint format" + + s"${Format.AMT.name}") /** * Returns a [[CheckpointProvider]] which can tell the files corresponding to this @@ -103,6 +112,20 @@ case class CheckpointInstance( val lastCheckpointInfo = lastCheckpointInfoHint.filter(cm => CheckpointInstance(cm) == this) val cpFiles = filterFiles(deltaLog, filesForCheckpointConstruction) format match { + case CheckpointInstance.Format.AMT => + val lastAMTCheckpointInfo = + lastCheckpointInfo.flatMap(_.amtCheckpoint).getOrElse { + throw new IllegalStateException( + s"AMT checkpoint instance at version $version requires a LastCheckpointInfo with " + + s"an amtCheckpoint, but none was provided.") + } + val checkpointAction = lastAMTCheckpointInfo.checkpoint.getOrElse { + // When we omit the checkpoint action to prevent _last_checkpoint file from being too + // large, we need a fallback mechanism. For now, we expect it to be always present. + throw new IllegalStateException("AMT checkpoint action is expected to be always present.") + } + AMTCheckpointProvider.fromCheckpoint( + deltaLog, checkpointAction, lastAMTCheckpointInfo.manifestCommitVersion) // Treat single file checkpoints also as V2 Checkpoints because we don't know if it is // actually a V2 checkpoint until we read it. case CheckpointInstance.Format.V2 | CheckpointInstance.Format.SINGLE => @@ -124,6 +147,9 @@ case class CheckpointInstance( } case CheckpointInstance.Format.WITH_PARTS => PreloadedCheckpointProvider(cpFiles, lastCheckpointInfo) + case CheckpointInstance.Format.AMT => + throw DeltaErrors.assertionFailedError( + s"checkpoint format ${CheckpointInstance.Format.AMT.name} is not readable yet") case CheckpointInstance.Format.SENTINEL => throw DeltaErrors.assertionFailedError( s"invalid checkpoint format ${CheckpointInstance.Format.SENTINEL}") @@ -134,6 +160,9 @@ case class CheckpointInstance( filesForCheckpointConstruction: Seq[FileStatus]) : Seq[FileStatus] = { val logPath = deltaLog.logPath format match { + // An AMT checkpoint has no standalone checkpoint file to select from the listing; the + // manifest tree is reached through the inline Checkpoint action instead. + case CheckpointInstance.Format.AMT => Seq.empty // Treat Single File checkpoints also as V2 Checkpoints because we don't know if it is // actually a V2 checkpoint until we read it. case format if format.usesSidecars => @@ -197,6 +226,7 @@ object CheckpointInstance { case SINGLE.name => Some(SINGLE) case WITH_PARTS.name => Some(WITH_PARTS) case V2.name => Some(V2) + case AMT.name => Some(AMT) case _ => None } @@ -206,6 +236,8 @@ object CheckpointInstance { object WITH_PARTS extends Format(1, "WITH_PARTS") /** V2 Checkpoint format */ object V2 extends Format(2, "V2") with FormatUsesSidecars + /** AMT checkpoint format. */ + object AMT extends Format(3, "AMT") /** Sentinel, for internal use only */ object SENTINEL extends Format(Int.MaxValue, "SENTINEL") } @@ -240,7 +272,17 @@ object CheckpointInstance { version = metadata.version, format = metadata.getFormatEnum(), fileName = metadata.v2Checkpoint.map(_.path), - numParts = metadata.parts) + numParts = metadata.parts, + manifestCommitVersion = metadata.amtCheckpoint.map(_.manifestCommitVersion)) + } + + def apply(amtCheckpointProvider: AMTCheckpointProvider): CheckpointInstance = { + CheckpointInstance( + version = amtCheckpointProvider.version, + format = Format.AMT, + fileName = None, + numParts = Some(amtCheckpointProvider.leaves.size), + manifestCommitVersion = Some(amtCheckpointProvider.manifestCommitVersion)) } val MaxValue: CheckpointInstance = sentinelValue(versionOpt = None) @@ -310,13 +352,17 @@ trait Checkpoints extends DeltaLogging { */ def checkpoint( snapshotToCheckpoint: Snapshot, - catalogTableOpt: Option[CatalogTable] = None): Unit = + catalogTableOpt: Option[CatalogTable] = None, + amtTriggerModeOpt: Option[AMTTriggerMode] = None): Unit = recordDeltaOperation(this, "delta.checkpoint") { withCheckpointExceptionHandling(snapshotToCheckpoint.deltaLog, "delta.checkpoint.sync.error") { if (snapshotToCheckpoint.version < 0) { throw DeltaErrors.checkpointNonExistTable(dataPath) } - checkpointAndCleanUpDeltaLog(snapshotToCheckpoint, catalogTableOpt) + checkpointAndCleanUpDeltaLog( + snapshotToCheckpoint, + catalogTableOpt = catalogTableOpt, + amtTriggerModeOpt = amtTriggerModeOpt) } } @@ -337,13 +383,281 @@ trait Checkpoints extends DeltaLogging { def checkpointAndCleanUpDeltaLog( snapshotToCheckpoint: Snapshot, - catalogTableOpt: Option[CatalogTable]): Unit = { + catalogTableOpt: Option[CatalogTable], + amtTriggerModeOpt: Option[AMTTriggerMode] = None): Unit = { + if (AMTUtils.amtEnabled(snapshotToCheckpoint)) { + // Note: This also takes care of writing the last checkpoint file via optimistic transaction + AMTWriterManager.emitAMTCheckpoint(snapshotToCheckpoint, catalogTableOpt, amtTriggerModeOpt) + return + } val lastCheckpointInfo = writeCheckpointFiles(snapshotToCheckpoint, catalogTableOpt) writeLastCheckpointFile( snapshotToCheckpoint.deltaLog, lastCheckpointInfo, LastCheckpointInfo.checksumEnabled(spark)) doLogCleanup(snapshotToCheckpoint, catalogTableOpt) } + /** Builds the LastCheckpointInfo for AMT and writes it to `_last_checkpoint` file. */ + def writeLastCheckpointFileForAMT( + manifestCommitVersion: Long, + writeResult: AMTWriteResult): Unit = { + val lastCheckpointInfo = Checkpoints.buildLastCheckpointInfoForAMT( + manifestCommitVersion, writeResult) + writeLastCheckpointFile(this, lastCheckpointInfo, LastCheckpointInfo.checksumEnabled(spark)) + } + + /** + * Resolves the AMT checkpoint instances from the last checkpoint info and old checkpoint + * provider. + */ + protected def resolveAMTCheckpointInstances( + lastCheckpointInfo: Option[LastCheckpointInfo], + oldCheckpointProviderOpt: Option[UninitializedCheckpointProvider]) + : Seq[CheckpointInstance] = { + val instanceFromLastCheckpointInfo = lastCheckpointInfo + .filter(_.amtCheckpoint.isDefined) + .map(CheckpointInstance.apply) + val instanceFromOldCheckpointProvider = oldCheckpointProviderOpt.collect { + case amt: AMTCheckpointProvider => CheckpointInstance.apply(amt) + } + // If the hint and the old provider describe the same AMT checkpoint, they'll both be discarded + // in `getLatestCompleteCheckpointFromList`. Dedup so a single instance survives. Prioritize the + // old checkpoint provider instance for better semantics, as we will reuse the old checkpoint + // provider in `getLogSegmentForVersion` if the version matches. + instanceFromLastCheckpointInfo + .filterNot(info => instanceFromOldCheckpointProvider.exists(_.version == info.version)) + .toSeq ++ instanceFromOldCheckpointProvider + } + + /** Starts an async read of the last commit's CommitInfo if needed and flag enabled. */ + protected def maybeReadLastCommitInfoAsync( + logSegment: LogSegment, + shouldReconcileAMTCheckpointProvider: Boolean): Option[Future[Option[CommitInfo]]] = { + val shouldKickOffAsyncCommitInfoRead = shouldReconcileAMTCheckpointProvider && + spark.conf.get(DeltaSQLConf.AMT_SNAPSHOT_DISCOVERY_ASYNC_COMMIT_INFO_READ_ENABLED) + Option.when(shouldKickOffAsyncCommitInfoRead) { + SnapshotManagement.checkpointThreadPool.submit(spark) { readLastCommitInfo(logSegment) } + } + } + + /** + * Reconciles the AMT checkpoint provider (if any) of the log segment against the last manifest + * commit reference, updating/trimming the segment as needed and throwing on inconsistency. + * + * @param initSegment initial log segment; may contain some initial checkpoint provider. + * @param lastChecksumOpt optional checksum at the log segment's version. + * @param lastCommitInfoFuture future of the CommitInfo from the last commit in the log segment. + * + * @return updated log segment with reconciled AMT checkpoint provider and trimmed deltas. + */ + protected def reconcileAMTCheckpointProvider( + initSegment: LogSegment, + lastChecksumOpt: Option[VersionChecksum], + lastCommitInfoFutureOpt: Option[Future[Option[CommitInfo]]]): LogSegment = { + val lastManifestCommitOpt = (lastChecksumOpt, lastCommitInfoFutureOpt) match { + case (Some(checksum), _) => + lastCommitInfoFutureOpt.foreach { lastCommitInfoFuture => + if (DeltaUtils.isTesting) { + // In testing, we always complete the async read to avoid flaky file operation counts. + ThreadUtils.awaitResult(lastCommitInfoFuture, Duration.Inf) + } else { + // In prod, the async CommitInfo read is no longer needed and we can cancel it. + // We trust the CRC and the CommitInfo have the same lastManifestCommit. + lastCommitInfoFuture.cancel(false) + } + } + checksum.lastManifestCommit + + case (None, Some(lastCommitInfoFuture)) => + // No CRC is available, so fall back to the reference from the last commit's CommitInfo. + // Block on the read that was started in parallel with the CRC read. + ThreadUtils.awaitResult(lastCommitInfoFuture, Duration.Inf).flatMap(_.lastManifestCommit) + + case (None, None) => + // Without either a CRC or a CommitInfo, we cannot differentiate between: + // 1. There is no lastManifestCommit at all; + // 2. There is a lastManifestCommit, but we chose not to read it (flag disabled). + // We return None in both cases, but note that case-2 could lead to an existing AMT + // checkpoint provider not discovered, and cause missing file actions. + None + } + + (initSegment.checkpointProvider, lastManifestCommitOpt) match { + case (amtCheckpointProvider: AMTCheckpointProvider, lastManifestCommitOpt) if + amtCheckpointProvider.manifestCommitVersion > initSegment.version => + // Exemption: the initial AMT checkpoint provider is introduced by a later manifest commit, + // undiscoverable at the target version. + // + // This could happen during `deltaLog.createSnapshotAtInit()` when we haven't initialized + // the table's commit-coordinator before we come up with an initial snapshot. In this case, + // if a manifest commit at version M describing content root version R is unbackfilled, it + // won't show up in the initial listing, so the initial log segment version will be M - 1 + // (we explicitly backfill all previous files before writing any manifest commit). But if + // _last_checkpoint has already been updated with this AMT at version R, the initial log + // segment creation will still adopt this hint as the listing floor. We will end up with an + // initial AMT describing version R, whose publishing manifest commit is not reflected in + // the initial snapshot’s lastManifestCommit. + // + // AMT guarantees that content root versions are non-decreasing across manifest commits. + // Since manifest commit M describes version R, no earlier manifest commit can describe a + // content root greater than R. Inline manifest commits describe their own versions, so none + // of them could exist in (R, X], where X is the snapshot version and R <= X < M. Therefore, + // keeping the initial AMT will not miss any file actions. + assert(amtCheckpointProvider.version <= initSegment.version) + lastManifestCommitOpt.foreach { lastManifestCommit => + // This AMT checkpoint provider is introduced by a later manifest commit, so it must be + // no less than the content root version described by the current lastManifestCommit. + assert(amtCheckpointProvider.version >= lastManifestCommit.contentRootVersion) + } + + case (amtCheckpointProvider: AMTCheckpointProvider, Some(lastManifestCommit)) => + if (amtCheckpointProvider.version == lastManifestCommit.contentRootVersion) { + // Happy-path: the versions match. Safe to use the initial AMT checkpoint provider. + return initSegment + } else if (amtCheckpointProvider.version < lastManifestCommit.contentRootVersion) { + // The initial AMT checkpoint provider is stale, which could happen during: + // - `deltaLog.createSnapshotAtInit()` when _last_checkpoint is stale; + // - `deltaLog.update()` when a manifest commit has landed after last cached snapshot. + // + // We must NOT use the stale AMT checkpoint provider, because: + // - A newer content root version is discoverable at the target version, meaning that at + // least one newer manifest commit has landed. + // - If that newer manifest commit is an inline one, having it in the trailing deltas + // would lead to missing file actions. + return readManifestCommitAndUpdateAMTCheckpointProvider(initSegment, lastManifestCommit) + } else if (amtCheckpointProvider.version > lastManifestCommit.contentRootVersion) { + // This should not happen. + throw new IllegalStateException( + s"AMT checkpoint mismatch for table $logPath: the AMT checkpoint provider is at " + + s"version ${amtCheckpointProvider.version}, but the checksum's lastManifestCommit " + + s"describes content root version ${lastManifestCommit.contentRootVersion}.") + } + + case (amtCheckpointProvider: AMTCheckpointProvider, None) => + throw new IllegalStateException( + s"An AMT checkpoint provider at version ${amtCheckpointProvider.version} was " + + s"discovered for table $logPath, but no lastManifestCommit is available from either " + + s"the CRC or the latest commit's CommitInfo to corroborate it. Refusing to trust the " + + s"AMT checkpoint.") + + case (otherCheckpointProvider, Some(lastManifestCommit)) => + // Treat the lastManifestCommit as the source of truth for the AMT checkpoint provider, + // even when the initial log segment has a non-AMT or empty checkpoint provider. + if (otherCheckpointProvider.version <= lastManifestCommit.contentRootVersion) { + // We have all the deltas needed after the accurate content root version, so we update. + return readManifestCommitAndUpdateAMTCheckpointProvider(initSegment, lastManifestCommit) + } else { + // The initial log segment somehow has a non-AMT provider, while the lastManifestCommit + // claims that there is some AMT content root available describing an earlier version. + // This might happen across upgrade/downgrade code paths. Throw until we support them. + throw new IllegalStateException( + s"The LastManifestCommit of version ${initSegment.version} carries a content root " + + s"version ${lastManifestCommit.contentRootVersion} that is ahead of the initial " + + s"non-AMT checkpoint version ${otherCheckpointProvider.version}. Refusing to " + + s"trust the checkpoint.") + } + + // A non-AMT / empty provider with no lastManifestCommit: nothing to reconcile. + case _ => () + } + + // No changes needed. + initSegment + } + + /** Reads the [[CommitInfo]] from the last commit file of the given log segment. */ + protected def readLastCommitInfo(logSegment: LogSegment): Option[CommitInfo] = { + val commitFile = DeltaCommitFileProvider(logPath, logSegment).deltaFile(logSegment.version) + DeltaHistoryManager.getCommitInfoOpt(store, commitFile, newDeltaHadoopConf()) + } + + /** + * Reads the Checkpoint action from the manifest commit and builds the AMT checkpoint provider. + * Returns the updated log segment with the new AMT checkpoint provider and trimmed deltas. + * + * One I/O is performed. + */ + private def readManifestCommitAndUpdateAMTCheckpointProvider( + logSegment: LogSegment, + lastManifestCommit: LastManifestCommit): LogSegment = { + recordDeltaOperation(this, "delta.v4amt.readManifestCommitAndUpdateAMTCheckpointProvider") { + val LastManifestCommit(manifestCommitVersion, contentRootVersion) = lastManifestCommit + // The initial log segment must have all the deltas after the content root version. + require(logSegment.checkpointProvider.version <= contentRootVersion) + val checkpoint = readCheckpointActionFromCommit(logSegment, lastManifestCommit) + val newCheckpointProvider = AMTCheckpointProvider.fromCheckpoint( + this, checkpoint, manifestCommitVersion) + trimLogSegmentToAMTCheckpoint(logSegment, newCheckpointProvider) + } + } + + /** + * Trims the log segment to install the new checkpoint provider. Pads the gap with non-compacted + * deltas if the new checkpoint provider sits in the middle of a compacted delta. + */ + private[delta] def trimLogSegmentToAMTCheckpoint( + logSegment: LogSegment, + newCheckpointProvider: AMTCheckpointProvider): LogSegment = { + require(newCheckpointProvider.version >= logSegment.checkpointProvider.version) + if (newCheckpointProvider.version == logSegment.checkpointProvider.version) { + // An inline AMT and an immediate rewrite can describe the same version with different trees. + // Use the new checkpoint provider with the latest tree. Other fields don't need trimming. + return logSegment.copy(checkpointProvider = newCheckpointProvider) + } + + val nonCompactedDeltas = logSegment.nonCompactedDeltasOpt.getOrElse { + throw new IllegalStateException( + s"The AMT log segment at version ${logSegment.version} has no non-compacted deltas.") + } + val deltasAndCompactedDeltas = (logSegment.deltas ++ nonCompactedDeltas) + .distinct.sortBy(f => f.getPath.getName) + val deltasAfterCheckpoint = nonCompactedDeltas + .filter(deltaVersion(_) > newCheckpointProvider.version).toArray + val trimmedDeltasAndCompactedDeltas = useCompactedDeltasForLogSegment( + deltasAndCompactedDeltas = deltasAndCompactedDeltas, + deltasAfterCheckpoint = deltasAfterCheckpoint, + latestCommitVersion = logSegment.version, + checkpointVersionToUse = newCheckpointProvider.version) + val deltaAtCheckpointVersionOpt = nonCompactedDeltas + .find(deltaVersion(_) == newCheckpointProvider.version) + if (deltaAtCheckpointVersionOpt.isEmpty) { + throw new IllegalStateException( + s"The trimmed AMT log segment has no delta at the new checkpoint version " + + s"${newCheckpointProvider.version}.\nOld log segment: ${logSegment}") + } + logSegment.copy( + deltas = trimmedDeltasAndCompactedDeltas, + nonCompactedDeltasOpt = Some(deltasAfterCheckpoint), + deltaAtCheckpointVersionOpt = deltaAtCheckpointVersionOpt, + checkpointProvider = newCheckpointProvider) + } + + /** Reads the [[actions.Checkpoint]] action from the manifest commit. */ + private def readCheckpointActionFromCommit( + logSegment: LogSegment, + lastManifestCommit: LastManifestCommit): Checkpoint = { + val LastManifestCommit(manifestCommitVersion, contentRootVersion) = lastManifestCommit + val commitFile = DeltaCommitFileProvider(logPath, logSegment).deltaFile(manifestCommitVersion) + val actions = store.readAsIterator(commitFile, newDeltaHadoopConf()) + val checkpoint = try { + actions + .map(Action.fromJson) + .collectFirst { case cp: Checkpoint => cp } + .getOrElse { + throw new IllegalStateException( + s"The checksum at version ${logSegment.version} names manifest commit version " + + s"${manifestCommitVersion} as the source of content root version " + + s"${contentRootVersion}, but that commit carries no Checkpoint action.") + } + } finally { + actions.close() + } + // It reads an inline (non-standalone) Checkpoint from a manifest commit, + // so both Checkpoint.version and ContentRoot.version must match the CRC pointer. + assert(checkpoint.version == contentRootVersion) + assert(checkpoint.contentRoot.version == contentRootVersion) + checkpoint + } + protected[delta] def writeLastCheckpointFile( deltaLog: DeltaLog, lastCheckpointInfo: LastCheckpointInfo, @@ -584,6 +898,8 @@ trait Checkpoints extends DeltaLogging { matchingCheckpointInstances.length == ci.numParts.get case CheckpointInstance.Format.V2 => matchingCheckpointInstances.length == 1 + case CheckpointInstance.Format.AMT => + matchingCheckpointInstances.length == 1 case CheckpointInstance.Format.SENTINEL => false } @@ -660,6 +976,27 @@ object Checkpoints Some(schema).filter(s => JsonUtils.toJson(s).length <= checkpointSchemaSizeThreshold) } + private[delta] def buildLastCheckpointInfoForAMT( + manifestCommitVersion: Long, + writeResult: AMTWriteResult): LastCheckpointInfo = { + val AMTWriteResult(contentRootVersion, checkpoint, leaves, _, _) = writeResult + val lastAMTCheckpoint = LastAMTCheckpoint( + manifestCommitVersion = manifestCommitVersion, + checkpoint = Some(checkpoint), + leaves = Some(leaves)) + LastCheckpointInfo( + version = contentRootVersion, + // `size` (total action count) and `numOfAddFiles` are unavailable for AMT: the file actions + // live in the manifest tree, not in memory here. + size = -1, + parts = Some(leaves.size), + sizeInBytes = Some(checkpoint.contentRoot.sizeInBytes + leaves.map(_.file_size_in_bytes).sum), + numOfAddFiles = None, + checkpointSchema = None, + checkpointType = Some(LastCheckpointInfo.CheckpointType.AMT), + amtCheckpoint = Some(lastAMTCheckpoint)) + } + /** * Writes out the contents of a [[Snapshot]] into a checkpoint file that * can be used to short-circuit future replays of the log. @@ -1051,11 +1388,9 @@ object Checkpoints * (the default, classic-checkpoint behavior) it is `df.schema.asNullable` (fully * nullable). The AMT manifest writer passes its id-carrying schema. The resolved * schema is the value returned to the caller. - * @param useDeltaParquetWriteSupport When true, hooks in [[DeltaParquetWriteSupport]] as the - * Parquet write support class so that list-element / map key-value field ids - * carried on `outputSchema` via `parquet.field.nested.ids` are emitted to the - * file's Parquet schema (the stock `ParquetWriteSupport` does not). Needed for - * the AMT Iceberg-V4 manifest schema. Default false uses the standard support. + * @param format The Parquet file format whose `prepareWrite` configures the write. Defaults to + * the standard [[ParquetFileFormat]]. The AMT manifest writer passes + * `AMTParquetFileFormat`, which applies the Iceberg-V4 manifest write settings. * @return The schema actually written. */ def writeAtomicCheckpointParquetFile( @@ -1065,18 +1400,12 @@ object Checkpoints hadoopConf: Configuration, useRename: Boolean, outputSchema: Option[StructType] = None, - useDeltaParquetWriteSupport: Boolean = false): StructType = + format: ParquetFileFormat = new ParquetFileFormat()): StructType = recordFrameProfile( "Checkpoints", "writeAtomicCheckpointParquetFile") { val schema = outputSchema.getOrElse(df.schema.asNullable) - val format = new ParquetFileFormat() val job = Job.getInstance(hadoopConf) val factory = format.prepareWrite(spark, job, Map.empty, schema) - if (useDeltaParquetWriteSupport) { - // Emit nested (list-element / map key-value) field ids, which the stock ParquetWriteSupport - // does not. Set after prepareWrite so it overrides the write-support class prepareWrite set. - ParquetOutputFormat.setWriteSupportClass(job, classOf[DeltaParquetWriteSupport]) - } val serConf = new SerializableConfiguration(job.getConfiguration) val finalSparkPath = SparkPath.fromPath(finalPath) @@ -1287,6 +1616,8 @@ object Checkpoints additionalCols ++= partitionValues additionalCols ++= Checkpoints.extractStats(snapshot.statsSchema, "add.stats") } + // amtResidue and backReference are dropped on purpose here: V1/V2 checkpoints are incompatible + // with AMT, so these AMT-era extras do not belong in the checkpoint `add` struct. val withAdd = state.withColumn("add", when(col("add").isNotNull, struct(Seq( col("add.path"), diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/Checksum.scala b/spark/src/main/scala/org/apache/spark/sql/delta/Checksum.scala index cb87262f4f4..f8e9d27cf9d 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/Checksum.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/Checksum.scala @@ -28,6 +28,7 @@ import scala.util.control.NonFatal import org.apache.spark.sql.delta.Relocated._ import org.apache.spark.sql.delta.actions._ +import org.apache.spark.sql.delta.amt.AMTUtils import org.apache.spark.sql.delta.commands.DeletionVectorUtils import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.metering.DeltaLogging @@ -368,6 +369,7 @@ trait RecordChecksum extends DeltaLogging { var numFiles = oldVersionChecksum.numFiles var protocol = oldVersionChecksum.protocol var metadata = oldVersionChecksum.metadata + var lastManifestCommit: Option[LastManifestCommit] = oldVersionChecksum.lastManifestCommit val fileSizeHistogram = if (spark.conf.get(DeltaSQLConf.DELTA_FILE_SIZE_HISTOGRAM_ENABLED)) { oldVersionChecksum.fileSizeHistogram.map { h => FileSizeHistogram(h.sortedBinBoundaries, h.fileCounts.clone(), h.totalBytes.clone()) @@ -425,7 +427,7 @@ trait RecordChecksum extends DeltaLogging { case _: RemoveFile if ignoreRemoveFiles => () - // extendedFileMetadata == true implies fields partitionValues, size, and tags are present + // extendedFileMetadata == true implies fields partitionValues and size are present case r: RemoveFile if r.extendedFileMetadata == Some(true) => val size = r.size.get tableSizeBytes -= size @@ -454,6 +456,7 @@ trait RecordChecksum extends DeltaLogging { metadata = m case ci: CommitInfo => inCommitTimestamp = ci.inCommitTimestamp + lastManifestCommit = ci.lastManifestCommit case _ => } @@ -549,7 +552,7 @@ trait RecordChecksum extends DeltaLogging { allFiles = allFiles, deletedRecordCountsHistogramOpt = deletedRecordCountsHistogramOpt, fileSizeHistogram = fileSizeHistogram, - lastManifestCommit = None + lastManifestCommit = lastManifestCommit )) } @@ -602,7 +605,11 @@ trait RecordChecksum extends DeltaLogging { // We can also ignore file retention because that only affects [[RemoveFile]] actions. val logReplay = new InMemoryLogReplay( minFileRetentionTimestamp = None, - minSetTransactionRetentionTimestamp = None) + minSetTransactionRetentionTimestamp = None, + tableRoot = deltaLog.dataPath, + // Using object identity or not doesn't matter here for computing SetTransactions. + useDeletionVectorObjectIdentity = spark.sessionState.conf.getConf( + DeltaSQLConf.DELETION_VECTORS_USE_OBJECT_IDENTITY_FOR_INCREMENTAL_CRC)) logReplay.append(attemptVersion - 1, oldSetTransactions.toIterator) logReplay.append(attemptVersion, setTransactionsToCommit.toIterator) @@ -631,7 +638,11 @@ trait RecordChecksum extends DeltaLogging { // We only work with DomainMetadata, so RemoveFile and SetTransaction retention don't matter. val logReplay = new InMemoryLogReplay( minFileRetentionTimestamp = None, - minSetTransactionRetentionTimestamp = None) + minSetTransactionRetentionTimestamp = None, + tableRoot = deltaLog.dataPath, + // Using object identity or not doesn't matter here for computing DomainMetadata. + useDeletionVectorObjectIdentity = spark.sessionState.conf.getConf( + DeltaSQLConf.DELETION_VECTORS_USE_OBJECT_IDENTITY_FOR_INCREMENTAL_CRC)) val threshold = spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_MAX_DOMAIN_METADATAS_IN_CRC) @@ -682,7 +693,9 @@ trait RecordChecksum extends DeltaLogging { .orElse { recordFrameProfile( "Delta", "VersionChecksum.incrementallyComputeAddFiles") { - oldSnapshot.map(_.allFiles.collect().toSeq) + // Reconstructing from the read snapshot can surface stale leaf back references when that + // snapshot's AMT tree still had leaf manifests so strip them here. + oldSnapshot.map(_.allFiles.collect().toSeq.map(_.copy(backReference = None))) } } .getOrElse { return None } @@ -697,7 +710,10 @@ trait RecordChecksum extends DeltaLogging { // We only work with AddFile, so RemoveFile and SetTransaction retention don't matter. val logReplay = new InMemoryLogReplay( minFileRetentionTimestamp = None, - minSetTransactionRetentionTimestamp = None) + minSetTransactionRetentionTimestamp = None, + tableRoot = deltaLog.dataPath, + useDeletionVectorObjectIdentity = spark.sessionState.conf.getConf( + DeltaSQLConf.DELETION_VECTORS_USE_OBJECT_IDENTITY_FOR_INCREMENTAL_CRC)) logReplay.append(attemptVersion - 1, oldAllFiles.map(normalizePath).toIterator) logReplay.append(attemptVersion, actionsToCommit.map(normalizePath).toIterator) @@ -855,11 +871,29 @@ trait ValidateChecksum extends DeltaLogging { self: Snapshot => */ def validateFileListAgainstCRC(checksum: VersionChecksum, contextOpt: Option[String]): Boolean = { val fileSortKey = (f: AddFile) => (f.path, f.modificationTime, f.size) - val filesFromCrc = checksum.allFiles.map(_.sortBy(fileSortKey)).getOrElse { return true } - val filesFromStateReconstruction = recordFrameProfile( + var filesFromCrc = checksum.allFiles.map(_.sortBy(fileSortKey)).getOrElse { return true } + var filesFromStateReconstruction = recordFrameProfile( "Delta", "snapshot.allFiles") { allFilesViaStateReconstruction.collect().toSeq.sortBy(fileSortKey) } + // AMT manifest trees do not yet round-trip every AddFile field: `DataEntry.toAddFile` zeroes + // `modificationTime`, forces `dataChange = false`, drops `tags`, and reduces `stats` to + // `{"numRecords":n}`. Project those lossy fields before comparing. + if (AMTUtils.amtEnabled(self)) { + def normalizeForAmtTreeRoundTrip(f: AddFile): AddFile = f.copy( + // CRC and AMT reconstruction may use different path encodings for the same on-disk DV, + // so normalize both to absolute paths before comparing. + deletionVector = Option(f.deletionVector) + .map(_.copyWithAbsolutePath(deltaLog.dataPath)) + .orNull, + modificationTime = 0L, + dataChange = false, + tags = null, + stats = f.numPhysicalRecords.map(n => s"""{"numRecords":$n}""").getOrElse(null)) + filesFromCrc = filesFromCrc.map(normalizeForAmtTreeRoundTrip).sortBy(fileSortKey) + filesFromStateReconstruction = + filesFromStateReconstruction.map(normalizeForAmtTreeRoundTrip).sortBy(fileSortKey) + } if (filesFromCrc == filesFromStateReconstruction) return true val filesFromCrcWithoutStats = filesFromCrc.map(_.copy(stats = "")) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/ConflictChecker.scala b/spark/src/main/scala/org/apache/spark/sql/delta/ConflictChecker.scala index a875e4219df..72f973b3810 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/ConflictChecker.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/ConflictChecker.scala @@ -24,6 +24,7 @@ import scala.collection.mutable import org.apache.spark.sql.delta.DeltaOperations.{OP_SET_TBLPROPERTIES, ROW_TRACKING_BACKFILL_OPERATION_NAME, ROW_TRACKING_UNBACKFILL_OPERATION_NAME} import org.apache.spark.sql.delta.RowId.RowTrackingMetadataDomain import org.apache.spark.sql.delta.actions._ +import org.apache.spark.sql.delta.amt.AMTUtils import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.metering.DeltaLogging @@ -34,7 +35,7 @@ import org.apache.spark.sql.delta.util.FileNames import io.delta.storage.commit.UpdatedActions import io.delta.storage.commit.uccommitcoordinator.UCCommitCoordinatorClient import io.delta.storage.commit.uniform.UniformMetadata -import org.apache.hadoop.fs.FileStatus +import org.apache.hadoop.fs.{FileStatus, Path} import org.apache.spark.internal.{MDC, MessageWithContext} import org.apache.spark.sql.{DataFrame, SparkSession} @@ -69,7 +70,8 @@ private[delta] case class CurrentTransactionInfo( val catalogTable: Option[CatalogTable], val domainMetadata: Seq[DomainMetadata], val op: DeltaOperations.Operation, - val preCommitLatestAMTCheckpointOpt: Option[Checkpoint] = None + val preCommitLatestAMTCheckpointOpt: Option[Checkpoint] = None, + val currentCommitAttemptAMTCheckpointOpt: Option[Checkpoint] = None , val convertedIcebergMetadata: Option[UniformMetadata] = None , idempotentCommitAlreadyLandedAt: Option[Long] = None ) { @@ -194,6 +196,54 @@ private[delta] class WinningCommitSummary( .exists(_.toBoolean) } +/** Compact information about [[WinningCommitSummary]]. */ +private[delta] case class WinningCommitMetrics( + isBlindAppend: Boolean, + minDefaultRowCommitVersion: Option[Long], + numAdds: Int, + numRemoves: Int, + numAddFilesWithBackreferences: Int, + numRemoveFilesWithBackreferences: Int, + checkpointAction: Option[Checkpoint], + commitInfo: Option[CommitInfo]) { + + /** + * Whether every Add/Remove file action in this winning commit was created strictly after + * `baseVersion` (min `defaultRowCommitVersion` > `baseVersion`) + */ + def allFileActionsHaveDefaultCommitVersionNewerThan(baseVersion: Long): Boolean = + (numAdds + numRemoves == 0) || minDefaultRowCommitVersion.exists(_ > baseVersion) +} + +object WinningCommitMetrics { + def fromWinningCommitSummary(summary: WinningCommitSummary): WinningCommitMetrics = { + val defaultRowCommitVersions: Seq[Option[Long]] = + summary.addedFiles.map(_.defaultRowCommitVersion) ++ + summary.removedFiles.map(_.defaultRowCommitVersion) + // If any AddFile/RemoveFile is missing defaultRowCommitVersion, then make + // minDefaultRowCommitVersion = None. + val minDefaultRowCommitVersion = + if (defaultRowCommitVersions.nonEmpty && defaultRowCommitVersions.forall(_.isDefined)) { + Some(defaultRowCommitVersions.flatten.min) + } else { + None + } + val numAdds = summary.addedFiles.size + val numRemoves = summary.removedFiles.size + val numAddFilesWithBackreferences = summary.addedFiles.count(_.backReference.isDefined) + val numRemoveFilesWithBackreferences = summary.removedFiles.count(_.backReference.isDefined) + WinningCommitMetrics( + isBlindAppend = summary.isBlindAppendOption.getOrElse(false), + minDefaultRowCommitVersion = minDefaultRowCommitVersion, + numAdds = numAdds, + numRemoves = numRemoves, + numAddFilesWithBackreferences = numAddFilesWithBackreferences, + numRemoveFilesWithBackreferences = numRemoveFilesWithBackreferences, + checkpointAction = summary.amtCheckpoint, + commitInfo = summary.commitInfo) + } +} + object WinningCommitSummary { /** @@ -226,6 +276,7 @@ private[delta] class ConflictChecker( isolationLevel: IsolationLevel) extends DeltaLogging with ConflictCheckerPredicateElimination { + protected val dataPath = initialCurrentTransactionInfo.readSnapshot.dataPath protected val winningCommitVersion = winningCommitSummary.commitVersion protected val startTimeMs = System.currentTimeMillis() protected val timingStats = mutable.HashMap[String, Long]() @@ -261,14 +312,11 @@ private[delta] class ConflictChecker( /** Run invariants on new set of actions in case they changed. */ private def checkInvariants(updatedInfo: CurrentTransactionInfo): Unit = { if (!hasActionsChanged(updatedInfo)) return - ConflictChecker.checkNoDuplicateActions(spark, updatedInfo.actions.iterator) + val useDVObjectIdentity = FileAction.useDeletionVectorObjectIdentity( + updatedInfo.metadata, updatedInfo.protocol, spark) + ConflictChecker.checkNoDuplicateActions( + spark, updatedInfo.actions.iterator, dataPath, useDVObjectIdentity) .foreach(_ => ()) - ConflictChecker.trackConsistentDataChange( - spark, - updatedInfo.actions.iterator, - deltaLog, - updatedInfo.op, - callerContext = "checkConflictsAndValidateActions").foreach(_ => ()) } /** @@ -310,12 +358,11 @@ private[delta] class ConflictChecker( checkForDeletedFilesAgainstCurrentTxnDeletedFiles() resolveTimestampOrderingConflicts() - // If the winning commit emitted an inline AMT checkpoint, it is now the latest checkpoint - // before the next commit attempt. - winningCommitSummary.amtCheckpoint.foreach { checkpoint => - currentTransactionInfo = - currentTransactionInfo.copy(preCommitLatestAMTCheckpointOpt = Some(checkpoint)) - } + // Fold the winning commit into our transaction info before the next attempt: this advances + // `preCommitLatestAMTCheckpointOpt` to the winner's inline AMT checkpoint (when the winner + // wrote one) and clears `currentCommitAttemptAMTCheckpointOpt`. + currentTransactionInfo = + AMTUtils.updateCurrentTransactionInfo(currentTransactionInfo, winningCommitSummary) logMetrics() currentTransactionInfo @@ -387,7 +434,7 @@ private[delta] class ConflictChecker( oldProtocol = readProtocol, table = DeltaTableV2( spark = spark, - path = deltaLog.dataPath, + path = dataPath, catalogTable = currentTransactionInfo.catalogTable), snapshot = winningSnapshot) if (!isDowngradeCommitValid) { @@ -1263,9 +1310,8 @@ private[delta] class ConflictChecker( case (domain, _) if RowTrackingMetadataDomain.isSameDomain(domain) => domain case (_, Some(_)) => // Any conflict not specifically handled by a previous case must fail the transaction. - throw new io.delta.exceptions.ConcurrentTransactionException( - s"A conflicting metadata domain ${domainMetadataFromCurrentTransaction.domain} is " + - "added.") + throw DeltaErrors.conflictingMetadataDomainException( + domainMetadataFromCurrentTransaction.domain) } val mergedDomainMetadata = mutable.Buffer.empty[DomainMetadata] @@ -1539,60 +1585,125 @@ private[delta] class ConflictChecker( private[delta] object ConflictChecker extends DeltaLogging { /** - * Returns an iterator that validates all [[AddFile]] and [[RemoveFile]] actions in - * `actions` share a consistent `dataChange` value. [[AddCDCFile]] is excluded because - * change-data-feed files are always emitted with `dataChange = false`. + * Returns an iterator that validates two `dataChange` invariants in one pass: + * - all [[AddFile]] and [[RemoveFile]] actions share a consistent value; and + * - every file action matches [[DeltaOperations.Operation.expectedFileDataChange]], when set. * - * Behavior is controlled by - * [[DeltaSQLConf.DELTA_COMMIT_VALIDATE_CONSISTENT_DATA_CHANGE_MODE]]: - * - `off`: skip the check entirely. - * - `log`: record a Delta event on violation but do not throw. - * - `fatal`: record a Delta event and then throw an [[IllegalStateException]]. + * [[AddCDCFile]] is excluded because change-data-feed files always carry `dataChange = false`. + * Each check is independently controlled by its corresponding commit-validation mode. A + * violation is reported only once per invariant, and fatal mode throws on the first violation. * - * Single pass, no materialization; the throw fires on the first detected inconsistency. + * When consistency validation is enabled and the commit carries a [[CommitInfo]] with a + * [[CommitInfo.dataChange]], this also records whether that value matches the file actions. */ - def trackConsistentDataChange( + def trackDataChange( spark: SparkSession, actions: Iterator[Action], deltaLog: DeltaLog, op: DeltaOperations.Operation, callerContext: String): Iterator[Action] = { - val mode = - DeltaSQLConf.ConsistentDataChangeValidationMode.fromConf(spark.sessionState.conf) - if (mode == DeltaSQLConf.ConsistentDataChangeValidationMode.OFF) return actions + val consistentMode = DeltaSQLConf.DataChangeValidationMode.consistentDataChangeMode( + spark.sessionState.conf) + val expectedMode = DeltaSQLConf.DataChangeValidationMode.expectedDataChangeMode( + spark.sessionState.conf) + val expectedDataChange = op.expectedFileDataChange + + val consistentCheckEnabled = + consistentMode != DeltaSQLConf.DataChangeValidationMode.OFF + val expectedCheckEnabled = + expectedMode != DeltaSQLConf.DataChangeValidationMode.OFF && + expectedDataChange.isDefined + + if (!consistentCheckEnabled && !expectedCheckEnabled) return actions + var firstDataChangeAction: Option[FileAction] = None var firstNoDataChangeAction: Option[FileAction] = None - var violationReported = false - actions.map { action => - action match { - case f: FileAction if !f.isInstanceOf[AddCDCFile] => - if (f.dataChange) { - if (firstDataChangeAction.isEmpty) firstDataChangeAction = Some(f) - } else { - if (firstNoDataChangeAction.isEmpty) firstNoDataChangeAction = Some(f) + var inconsistentDataChangeReported = false + var unexpectedDataChangeReported = false + var commitInfoMismatchReported = false + var declaredDataChange: Option[Boolean] = None + + def isCommitInfoDataChangeViolated(): Boolean = + declaredDataChange.exists { declared => + (declared && firstNoDataChangeAction.isDefined) || + (!declared && firstDataChangeAction.isDefined) || + (!actions.hasNext && declared && firstDataChangeAction.isEmpty) + } + + new Iterator[Action] { + override def hasNext: Boolean = actions.hasNext + override def next(): Action = { + val action = actions.next() + action match { + case c: CommitInfo if consistentCheckEnabled => + declaredDataChange = c.dataChange + case f: FileAction if consistentCheckEnabled && !f.isInstanceOf[AddCDCFile] => + if (f.dataChange) { + if (firstDataChangeAction.isEmpty) firstDataChangeAction = Some(f) + } else { + if (firstNoDataChangeAction.isEmpty) firstNoDataChangeAction = Some(f) + } + case _ => + } + if (consistentCheckEnabled && !inconsistentDataChangeReported && + firstDataChangeAction.isDefined && firstNoDataChangeAction.isDefined) { + inconsistentDataChangeReported = true + val message = "All FileActions in a single commit must share a consistent " + + "dataChange value, but this commit mixes dataChange = true and " + + "dataChange = false actions." + recordDeltaEvent( + deltaLog, + "delta.commit.inconsistentDataChange", + data = Map( + "callerContext" -> callerContext, + "operation" -> op.name, + "operationParameters" -> op.jsonEncodedValues, + "firstDataChangeAction" -> firstDataChangeAction, + "firstNoDataChangeAction" -> firstNoDataChangeAction)) + if (consistentMode == DeltaSQLConf.DataChangeValidationMode.FATAL) { + throw new IllegalStateException(message) } - if (!violationReported && - firstDataChangeAction.isDefined && firstNoDataChangeAction.isDefined) { - violationReported = true - val message = "All FileActions in a single commit must share a consistent " + - "dataChange value, but this commit mixes dataChange = true and " + - "dataChange = false actions." + } else if (consistentCheckEnabled && !commitInfoMismatchReported && + isCommitInfoDataChangeViolated()) { + commitInfoMismatchReported = true + val declared = declaredDataChange.get + recordDeltaEvent( + deltaLog, + "delta.commit.commitInfoDataChangeMismatch", + data = Map( + "callerContext" -> callerContext, + "operation" -> op.name, + "operationParameters" -> op.jsonEncodedValues, + "declaredDataChange" -> declared, + "derivedDataChange" -> firstDataChangeAction.isDefined, + "firstDataChangeAction" -> firstDataChangeAction, + "firstNoDataChangeAction" -> firstNoDataChangeAction)) + } + + action match { + case f: FileAction if expectedCheckEnabled && !f.isInstanceOf[AddCDCFile] && + !unexpectedDataChangeReported && expectedDataChange.exists(_ != f.dataChange) => + unexpectedDataChangeReported = true + val expected = expectedDataChange.get + val error = DeltaErrors.unexpectedCommittedDataChange( + op.name, f.getClass.getSimpleName, f.dataChange, expected) recordDeltaEvent( deltaLog, - "delta.commit.inconsistentDataChange", + "delta.commit.unexpectedDataChange", data = Map( "callerContext" -> callerContext, "operation" -> op.name, "operationParameters" -> op.jsonEncodedValues, - "firstDataChangeAction" -> firstDataChangeAction, - "firstNoDataChangeAction" -> firstNoDataChangeAction)) - if (mode == DeltaSQLConf.ConsistentDataChangeValidationMode.FATAL) { - throw new IllegalStateException(message) + "actualDataChange" -> f.dataChange, + "path" -> f.path)) + if (expectedMode == DeltaSQLConf.DataChangeValidationMode.FATAL) { + throw error } - } - case _ => + logError(error.getMessage) + case _ => + } + action } - action } } @@ -1601,13 +1712,22 @@ private[delta] object ConflictChecker extends DeltaLogging { * streams. Checks: duplicate adds, duplicate removes, and same path+DV both * added and removed. Single pass, no materialization. Returns `actions` * unchanged when [[DeltaSQLConf.DELTA_DUPLICATE_ACTION_CHECK_ENABLED]] is off. + * + * This check is performed using AMT-aware object identity mode. In most cases, + * using object identity or not doesn't influence outcome given actions comes + * from the same snapshot. Some callers, e.g. conflict resolution, must pass + * `true` for AMT compatibility. */ def checkNoDuplicateActions( spark: SparkSession, - actions: Iterator[Action]): Iterator[Action] = { + actions: Iterator[Action], + tableRoot: Path, + useDVObjectIdentity: Boolean): Iterator[Action] = { if (!spark.conf.get(DeltaSQLConf.DELTA_DUPLICATE_ACTION_CHECK_ENABLED)) return actions val addPaths = mutable.Map.empty[String, Option[String]] val removePaths = mutable.Map.empty[String, Option[String]] + def dvIdOf(fileAction: FileAction): Option[String] = + Option(fileAction.deletionVector).map(_.uniqueId(tableRoot, useDVObjectIdentity)) def pathAndDVString(path: String, dvIdOpt: Option[String]): String = { dvIdOpt.map(dvId => s"$path DV $dvId").getOrElse(path) } @@ -1623,7 +1743,7 @@ private[delta] object ConflictChecker extends DeltaLogging { actions.map { action => action match { case add: AddFile => - val dvId = add.getDeletionVectorUniqueId + val dvId = dvIdOf(add) addPaths.put(add.path, dvId).foreach { existingDVId => failDuplicate("add", add.path, dvId, existingDVId) } @@ -1634,7 +1754,7 @@ private[delta] object ConflictChecker extends DeltaLogging { } } case remove: RemoveFile => - val dvId = remove.getDeletionVectorUniqueId + val dvId = dvIdOf(remove) removePaths.put(remove.path, dvId).foreach { existingDVId => failDuplicate("remove", remove.path, dvId, existingDVId) } diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaErrors.scala b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaErrors.scala index 179f1c32a5f..572db6ca242 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaErrors.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaErrors.scala @@ -37,9 +37,10 @@ import org.apache.spark.sql.delta.schema.{DeltaInvariantViolationException, Inva import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util.JsonUtils import io.delta.exceptions +import org.apache.commons.lang3.exception.ExceptionUtils import org.apache.hadoop.fs.{ChecksumException, Path} -import org.apache.spark.{SparkConf, SparkEnv, SparkException, SparkThrowable} +import org.apache.spark.{ReadOnlySparkConf, SparkConf, SparkEnv, SparkException, SparkThrowable} import org.apache.spark.sql.{AnalysisException, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute @@ -53,12 +54,55 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{DataType, StructField, StructType} +/** + * Enumerates the ways a data type change can be rejected. Each violation maps to an error + * subclass shared by `DELTA_CANNOT_CHANGE_DATA_TYPE` and + * `DELTA_UNSUPPORTED_ALTER_TABLE_REPLACE_COL_OP`. + */ +sealed trait DataTypeChangeViolation { + /** Name of the error subclass that describes this violation. */ + def subClass: String + + /** Message parameters for this violation, in the order placeholders appear in the subclass. */ + def parameters: Array[String] +} + +object DataTypeChangeViolation { + case class TightenNullability(column: String) extends DataTypeChangeViolation { + override val subClass: String = "TIGHTEN_NULLABILITY" + override def parameters: Array[String] = Array(column) + } + + case class AddNonNullableColumn(column: String) extends DataTypeChangeViolation { + override val subClass: String = "ADD_NON_NULLABLE_COLUMN" + override def parameters: Array[String] = Array(column) + } + + case class DropColumns(columns: Seq[String]) extends DataTypeChangeViolation { + override val subClass: String = "DROP_COLUMNS" + override def parameters: Array[String] = Array(columns.mkString(", ")) + } + + case class ChangeDataType(column: String, fromType: DataType, toType: DataType) + extends DataTypeChangeViolation { + override val subClass: String = "CHANGE_DATA_TYPE" + // Render the types with their SQL names (e.g. INT, BIGINT) rather than the internal + // `DataType.toString` (IntegerType, LongType) for a user-facing message. + override def parameters: Array[String] = Array(column, fromType.sql, toType.sql) + } +} + +class DeltaCannotChangeDataTypeException(val violation: DataTypeChangeViolation) + extends DeltaAnalysisException( + errorClass = s"DELTA_CANNOT_CHANGE_DATA_TYPE.${violation.subClass}", + messageParameters = violation.parameters) + trait DocsPath { /** * The URL for the base path of Delta's docs. When changing this path, ensure that the new path * works with the error messages below. */ - protected def baseDocsPath(conf: SparkConf): String = "https://docs.delta.io/latest" + protected def baseDocsPath(conf: ReadOnlySparkConf): String = "https://docs.delta.io/latest" def assertValidCallingFunction(): Unit = { val callingMethods = Thread.currentThread.getStackTrace @@ -84,7 +128,7 @@ trait DocsPath { * @return The entire URL of the documentation link */ def generateDocsLink( - conf: SparkConf, + conf: ReadOnlySparkConf, relativePath: String, skipValidation: Boolean = false): String = { require(conf != null) @@ -98,7 +142,7 @@ trait DocsPath { relativePath: String, skipValidation: Boolean = false): Option[String] = Option(spark.sparkContext) - .map(context => generateDocsLink(context.getConf, relativePath, skipValidation)) + .map(context => generateDocsLink(context.getReadOnlyConf, relativePath, skipValidation)) /** * List of error function names for all errors that have URLs. When adding your error to this list @@ -123,7 +167,6 @@ trait DocsPath { "concurrentModificationExceptionMsg", "incorrectLogStoreImplementationException", "sourceNotDeterministicInMergeException", - "columnMappingAdviceMessage", "icebergClassMissing", "tableFeatureReadRequiresWriteException", "tableFeatureRequiresHigherReaderProtocolVersion", @@ -145,7 +188,7 @@ trait DeltaErrorsBase with DeltaLogging with QueryErrorsBase { - def baseDocsPath(spark: SparkSession): String = baseDocsPath(spark.sparkContext.getConf) + def baseDocsPath(spark: SparkSession): String = baseDocsPath(spark.sparkContext.getReadOnlyConf) val faqRelativePath: String = "/delta-intro.html#frequently-asked-questions" @@ -170,29 +213,6 @@ trait DeltaErrorsBase ) } - def initialSnapshotTooLargeForStreaming( - snapshotVersion: Long, - numFiles: Long, - maxFiles: Int, - tablePath: String): Throwable = { - new DeltaUnsupportedOperationException( - errorClass = "DELTA_STREAMING_INITIAL_SNAPSHOT_TOO_LARGE", - messageParameters = Array( - tablePath, - snapshotVersion.toString, - numFiles.toString, - maxFiles.toString, - s"""To fix this issue, choose one of: - | - | 1. Increase spark.databricks.delta.streaming.initialSnapshotMaxFiles - | (current: $maxFiles) - | - | 2. Use 'startingVersion' option to skip the initial snapshot and start - | from a specific version""".stripMargin - ) - ) - } - def deltaSourceIgnoreChangesError( version: Long, changeInfo: String, @@ -273,6 +293,17 @@ trait DeltaErrorsBase messageParameters = Array(columns.mkString("[", ",", "]"))) } + def unexpectedCommittedDataChange( + operation: String, + actionType: String, + actualDataChange: Boolean, + expectedDataChange: Boolean): Throwable = { + new DeltaIllegalStateException( + errorClass = "DELTA_COMMIT_UNEXPECTED_DATA_CHANGE", + messageParameters = + Array(operation, actionType, actualDataChange.toString, expectedDataChange.toString)) + } + /** * Thrown when a CDC query contains conflict 'starting' or 'ending' options, e.g. when both * starting version and starting timestamp are specified. @@ -966,10 +997,11 @@ trait DeltaErrorsBase def alterTableReplaceColumnsException( oldSchema: StructType, newSchema: StructType, - reason: String): Throwable = { + violation: DataTypeChangeViolation): Throwable = { new DeltaAnalysisException( - errorClass = "DELTA_UNSUPPORTED_ALTER_TABLE_REPLACE_COL_OP", - messageParameters = Array(reason, formatSchema(oldSchema), formatSchema(newSchema)) + errorClass = s"DELTA_UNSUPPORTED_ALTER_TABLE_REPLACE_COL_OP.${violation.subClass}", + messageParameters = + Array(formatSchema(oldSchema), formatSchema(newSchema)) ++ violation.parameters ) } @@ -1271,14 +1303,14 @@ trait DeltaErrorsBase } def nonPartitionColumnAbsentException(colsDropped: Boolean): Throwable = { - val msg = if (colsDropped) { - " Columns which are of NullType have been dropped." + val errorClass = if (colsDropped) { + "DELTA_NON_PARTITION_COLUMN_ABSENT.NULL_TYPE_COLUMNS_DROPPED" } else { - "" + "DELTA_NON_PARTITION_COLUMN_ABSENT.ALL_PARTITION_COLUMNS" } new DeltaAnalysisException( - errorClass = "DELTA_NON_PARTITION_COLUMN_ABSENT", - messageParameters = Array(msg) + errorClass = errorClass, + messageParameters = Array.empty ) } @@ -1286,16 +1318,15 @@ trait DeltaErrorsBase replaceWhere: String, invariantViolation: InvariantViolationException): Throwable = { new DeltaAnalysisException( - errorClass = "DELTA_REPLACE_WHERE_MISMATCH", + errorClass = "DELTA_REPLACE_WHERE_MISMATCH.INVARIANT_VIOLATION", messageParameters = Array(replaceWhere, invariantViolation.getMessage), cause = Some(invariantViolation)) } def replaceWhereMismatchException(replaceWhere: String, badPartitions: String): Throwable = { new DeltaAnalysisException( - errorClass = "DELTA_REPLACE_WHERE_MISMATCH", - messageParameters = Array(replaceWhere, - s"Invalid data would be written to partitions $badPartitions.")) + errorClass = "DELTA_REPLACE_WHERE_MISMATCH.INVALID_PARTITIONS", + messageParameters = Array(replaceWhere, badPartitions)) } def illegalFilesFound(file: String): Throwable = { @@ -1304,16 +1335,98 @@ trait DeltaErrorsBase messageParameters = Array(file)) } - def illegalDeltaOptionException(name: String, input: String, explain: String): Throwable = { + private def illegalDeltaOption( + subClass: String, + name: String, + input: String, + subclassParameters: Map[String, String] = Map.empty, + cause: Throwable = null): Throwable = { + val errorClass = s"DELTA_ILLEGAL_OPTION.$subClass" + val allParameters = Map("input" -> input, "name" -> name) ++ subclassParameters + // Order the parameter values to match the parameter names in the (combined main and + // subclass) message template, so callers pass parameters by name rather than by position. + val orderedParameters = + DeltaThrowableHelper.getParameterNames(errorClass, errorSubClass = null).map(allParameters) + new DeltaIllegalArgumentException( + errorClass = errorClass, + messageParameters = orderedParameters, + cause = cause) + } + + def illegalDeltaOptionMustBeOneOf( + name: String, input: String, validValues: Seq[String]): Throwable = + illegalDeltaOption( + "MUST_BE_ONE_OF", name, input, Map("validValues" -> validValues.mkString(", "))) + + def illegalDeltaOptionMustBeBoolean(name: String, input: String): Throwable = + illegalDeltaOptionMustBeOneOf(name, input, Seq("'true'", "'false'")) + + def illegalDeltaOptionMustBeInteger(name: String, input: String): Throwable = + illegalDeltaOption("MUST_BE_INTEGER", name, input) + + def illegalDeltaOptionMustBeNonNegativeNumber(name: String, input: String): Throwable = + illegalDeltaOption("MUST_BE_NON_NEGATIVE_NUMBER", name, input) + + def illegalDeltaOptionMustBePositiveNumber(name: String, input: String): Throwable = + illegalDeltaOption("MUST_BE_POSITIVE_NUMBER", name, input) + + def illegalDeltaOptionNoEmptyColumnNames(name: String, input: String): Throwable = + illegalDeltaOption("NO_EMPTY_COLUMN_NAMES", name, input) + + def illegalDeltaOptionMustBeSizeConfiguration(name: String, input: String): Throwable = + illegalDeltaOption("MUST_BE_SIZE_CONFIGURATION", name, input) + + def illegalDeltaOptionDynamicPartitionOverwriteOnly(name: String, input: String): Throwable = + illegalDeltaOption("DYNAMIC_PARTITION_OVERWRITE_ONLY", name, input) + + def illegalDeltaOptionSchemaModeWithTimeTravel(name: String, input: String): Throwable = + illegalDeltaOption("SCHEMA_MODE_WITH_TIME_TRAVEL", name, input) + + def illegalDeltaOptionInvalidReorgParquetFormatVersion( + name: String, input: String, cause: Throwable): Throwable = + illegalDeltaOption("INVALID_REORG_PARQUET_FORMAT_VERSION", name, input, cause = cause) + + def illegalDeltaOptionInvalidParquetFormatVersion( + name: String, input: String, cause: Throwable): Throwable = + illegalDeltaOption( + "INVALID_PARQUET_FORMAT_VERSION", name, input, + subclassParameters = Map("causeExceptionMessage" -> cause.getMessage), cause = cause) + + def illegalDeltaOptionUnrecognizedNamedArgument( + name: String, input: String, functionName: String, validArguments: String): Throwable = + illegalDeltaOption( + "UNRECOGNIZED_NAMED_ARGUMENT", name, input, + Map("functionName" -> functionName, "validArguments" -> validArguments)) + + def illegalDeltaOptionParquetCompressionCodecConflict( + name: String, input: String, property: String, propertyValue: String): Throwable = + illegalDeltaOption( + "PARQUET_COMPRESSION_CODEC_CONFLICT", name, input, + Map("property" -> property, "propertyValue" -> propertyValue)) + + def illegalDeltaOptionParquetWriterVersionConflict( + name: String, + input: String, + property: String, + propertyValue: String, + resolvedVersion: String): Throwable = + illegalDeltaOption( + "PARQUET_WRITER_VERSION_CONFLICT", name, input, + Map( + "property" -> property, + "propertyValue" -> propertyValue, + "resolvedVersion" -> resolvedVersion)) + + def invalidIdempotentWritesMissingWriteOptionsException(): Throwable = { new DeltaIllegalArgumentException( - errorClass = "DELTA_ILLEGAL_OPTION", - messageParameters = Array(input, name, explain)) + errorClass = "DELTA_INVALID_IDEMPOTENT_WRITES_OPTIONS.MISSING_DATAFRAME_WRITE_OPTIONS", + messageParameters = Array.empty) } - def invalidIdempotentWritesOptionsException(explain: String): Throwable = { + def invalidIdempotentWritesMissingSessionConfsException(): Throwable = { new DeltaIllegalArgumentException( - errorClass = "DELTA_INVALID_IDEMPOTENT_WRITES_OPTIONS", - messageParameters = Array(explain)) + errorClass = "DELTA_INVALID_IDEMPOTENT_WRITES_OPTIONS.MISSING_SESSION_CONFS", + messageParameters = Array.empty) } def invalidInterval(interval: String): Throwable = { @@ -2302,11 +2415,9 @@ trait DeltaErrorsBase ) } - def cannotChangeDataType(msg: String): Throwable = { - new DeltaAnalysisException( - errorClass = "DELTA_CANNOT_CHANGE_DATA_TYPE", - messageParameters = Array(msg) - ) + def cannotChangeDataType( + violation: DataTypeChangeViolation): DeltaCannotChangeDataTypeException = { + new DeltaCannotChangeDataTypeException(violation) } def ambiguousDataTypeChange(column: String, from: StructType, to: StructType): Throwable = { @@ -2550,29 +2661,32 @@ trait DeltaErrorsBase mode.name)) } - protected def columnMappingAdviceMessage( - requiredProtocol: Protocol = ColumnMappingTableFeature.minProtocolVersion): String = { - val readerVersion = requiredProtocol.minReaderVersion - val writerVersion = requiredProtocol.minWriterVersion - s""" - |Please enable Column Mapping on your Delta table with mapping mode 'name'. - |You can use one of the following commands. - | - |ALTER TABLE table_name SET TBLPROPERTIES ('delta.columnMapping.mode' = 'name') - | - |Note, if your table is not on the required protocol version it will be upgraded. - |Column mapping requires at least protocol ($readerVersion, $writerVersion) - |""".stripMargin + /** + * Returns the error class (either `mainErrorClass` or `mainErrorClass.`) and message + * parameters for the "enable Column Mapping" advice. When `suggestUpgrade` is false the advice is + * omitted and the bare `mainErrorClass` is returned. + */ + protected def withColumnMappingAdviceSubClass( + mainErrorClass: String, suggestUpgrade: Boolean): (String, Array[String]) = { + if (!suggestUpgrade) { + return (mainErrorClass, Array.empty[String]) + } + val readerVersion = ColumnMappingTableFeature.minProtocolVersion.minReaderVersion + val writerVersion = ColumnMappingTableFeature.minProtocolVersion.minWriterVersion + (s"$mainErrorClass.ENABLE_COLUMN_MAPPING", + Array(readerVersion.toString, writerVersion.toString)) } def columnRenameNotSupported: Throwable = { - val adviceMsg = columnMappingAdviceMessage() - new DeltaAnalysisException("DELTA_UNSUPPORTED_RENAME_COLUMN", Array(adviceMsg)) + val (errorClass, params) = + withColumnMappingAdviceSubClass("DELTA_UNSUPPORTED_RENAME_COLUMN", suggestUpgrade = true) + new DeltaAnalysisException(errorClass, params) } def dropColumnNotSupported(suggestUpgrade: Boolean): Throwable = { - val adviceMsg = if (suggestUpgrade) columnMappingAdviceMessage() else "" - new DeltaAnalysisException("DELTA_UNSUPPORTED_DROP_COLUMN", Array(adviceMsg)) + val (errorClass, params) = + withColumnMappingAdviceSubClass("DELTA_UNSUPPORTED_DROP_COLUMN", suggestUpgrade) + new DeltaAnalysisException(errorClass, params) } def dropNestedColumnsFromNonStructTypeException(struct : DataType) : Throwable = { @@ -2672,6 +2786,16 @@ trait DeltaErrorsBase ) } + def fullAMTWriteFailedWithConflict( + conflictingCommitVersion: Long): FullAMTWriteFailedWithConflict = + new FullAMTWriteFailedWithConflict(conflictingCommitVersion) + + def concurrentAMTCheckpointLandedException( + latestManifestCommitVersion: Long, + latestContentRootVersion: Long): ConcurrentAMTCheckpointLandedException = + new ConcurrentAMTCheckpointLandedException( + latestManifestCommitVersion, latestContentRootVersion) + def metadataChangedException( table: String, conflictingCommit: Option[CommitInfo]): io.delta.exceptions.MetadataChangedException = { @@ -2691,21 +2815,17 @@ trait DeltaErrorsBase def protocolChangedException( conflictingCommit: Option[CommitInfo]): io.delta.exceptions.ProtocolChangedException = { - val additionalInfo = conflictingCommit.map { v => - if (v.version.getOrElse(-1) == 0) { - "This happens when multiple writers are writing to an empty directory. " + - "Creating the table ahead of time will avoid this conflict. " - } else { - "" - } - }.getOrElse("") - new io.delta.exceptions.ProtocolChangedException( - Array( - additionalInfo, - conflictingCommit.map(ci => s"\nConflicting commit: ${JsonUtils.toJson(ci)}").getOrElse(""), - DeltaErrors.generateDocsLink(SparkEnv.get.conf, "/concurrency-control.html") - ) - ) + val docLink = DeltaErrors.generateDocsLink(SparkEnv.get.conf, "/concurrency-control.html") + conflictingCommit match { + case Some(ci) if ci.version.getOrElse(-1L) == 0 => + io.delta.exceptions.ProtocolChangedException( + "WRITE_TO_EMPTY_DIRECTORY", Array(docLink, JsonUtils.toJson(ci))) + case Some(ci) => + io.delta.exceptions.ProtocolChangedException( + "CONFLICTING_COMMIT", Array(docLink, JsonUtils.toJson(ci))) + case None => + new io.delta.exceptions.ProtocolChangedException(Array(docLink)) + } } def unsupportedReaderTableFeaturesInTableException( @@ -2971,6 +3091,11 @@ trait DeltaErrorsBase ) } + def conflictingMetadataDomainException( + domain: String): ConflictingMetadataDomainException = { + new ConflictingMetadataDomainException(Array(domain)) + } + def restoreMissedDataFilesError(missedFiles: Array[String], version: Long): Throwable = new IllegalArgumentException( s"""Not all files from version $version are available in file system. @@ -3778,33 +3903,22 @@ trait DeltaErrorsBase def uniFormIcebergRequiresIcebergCompat(): Throwable = { new DeltaUnsupportedOperationException( - errorClass = "DELTA_UNIVERSAL_FORMAT_VIOLATION", - messageParameters = Array( - UniversalFormat.ICEBERG_FORMAT, - "Requires IcebergCompat to be explicitly enabled in order for Universal Format (Iceberg) " + - "to be enabled on an existing table. Supported versions are IcebergCompatV1 and " + - "IcebergCompatV2." - ) + errorClass = "DELTA_UNIVERSAL_FORMAT_VIOLATION.ICEBERG_COMPAT_REQUIRED", + messageParameters = Array(UniversalFormat.ICEBERG_FORMAT) ) } def uniFormHudiDeleteVectorCompat(): Throwable = { new DeltaUnsupportedOperationException( - errorClass = "DELTA_UNIVERSAL_FORMAT_VIOLATION", - messageParameters = Array( - UniversalFormat.HUDI_FORMAT, - "Requires delete vectors to be disabled." - ) + errorClass = "DELTA_UNIVERSAL_FORMAT_VIOLATION.HUDI_DELETE_VECTORS_NOT_SUPPORTED", + messageParameters = Array(UniversalFormat.HUDI_FORMAT) ) } def uniFormHudiSchemaCompat(unsupportedType: DataType): Throwable = { new DeltaUnsupportedOperationException( - errorClass = "DELTA_UNIVERSAL_FORMAT_VIOLATION", - messageParameters = Array( - UniversalFormat.HUDI_FORMAT, - s"DataType: $unsupportedType is not currently supported." - ) + errorClass = "DELTA_UNIVERSAL_FORMAT_VIOLATION.HUDI_UNSUPPORTED_DATA_TYPE", + messageParameters = Array(UniversalFormat.HUDI_FORMAT, unsupportedType.toString) ) } @@ -3988,10 +4102,20 @@ trait DeltaErrorsBase def universalFormatConversionFailedException( failedOnCommitVersion: Long, format: String, - errorMessage: String): Throwable = { + error: Throwable): Throwable = { + new DeltaRuntimeException( + errorClass = "DELTA_UNIVERSAL_FORMAT_CONVERSION_FAILED.CAUSED_BY_ERROR", + messageParameters = Array(s"$failedOnCommitVersion", format, ExceptionUtils.getMessage(error)) + ).initCause(error) + } + + def universalFormatConversionFailedUnexpectedPartitionDataTypeException( + failedOnCommitVersion: Long, + format: String, + dataType: DataType): Throwable = { new DeltaRuntimeException( - errorClass = "DELTA_UNIVERSAL_FORMAT_CONVERSION_FAILED", - messageParameters = Array(s"$failedOnCommitVersion", format, errorMessage) + errorClass = "DELTA_UNIVERSAL_FORMAT_CONVERSION_FAILED.UNEXPECTED_PARTITION_DATA_TYPE", + messageParameters = Array(s"$failedOnCommitVersion", format, dataType.toString) ) } @@ -4147,6 +4271,22 @@ trait DeltaErrorsBase messageParameters = Array.empty) } + def replacePartitionedByOnUnpartitionedTableException(): Throwable = { + new DeltaAnalysisException( + errorClass = "DELTA_REPLACE_PARTITIONED_BY_ON_UNPARTITIONED_TABLE", + messageParameters = Array.empty) + } + + def replacePartitionedByPartitionColumnsNotMaterializedException( + partitionColumns: Seq[String], + numFiles: Long, + examplePaths: Seq[String]): Throwable = { + new DeltaAnalysisException( + errorClass = "DELTA_REPLACE_PARTITIONED_BY_PARTITION_COLUMNS_NOT_MATERIALIZED", + messageParameters = Array( + partitionColumns.mkString(", "), numFiles.toString, examplePaths.mkString(", "))) + } + def createTableWithDifferentClusteringException( path: Path, specifiedClusterBySpec: Option[ClusterBySpec], @@ -4182,6 +4322,22 @@ trait DeltaErrorsBase messageParameters = Array(operation)) } + def checkCatalogManagedTableOperationAllowed( + operation: String, + snapshot: SnapshotDescriptor, + catalogTableOpt: Option[CatalogTable]): Unit = { + if (snapshot.isCatalogOwned) { + val allowedOperations = catalogTableOpt + .flatMap(_.storage.properties.get( + CatalogManagedTableMaintenanceOperation.ALLOWED_OPERATIONS_PROPERTY)) + .map(_.split(",")) + .getOrElse(Array.empty[String]) + if (!allowedOperations.contains(operation)) { + throw operationBlockedOnCatalogManagedTable(operation) + } + } + } + def deltaCannotCreateCatalogManagedTable(): Throwable = { new DeltaUnsupportedOperationException( errorClass = "DELTA_UNSUPPORTED_CATALOG_MANAGED_TABLE_CREATION", @@ -4288,6 +4444,41 @@ class ConcurrentWriteException(message: String) conflictingCommit)) } +/** + * Thrown by the AMT write path when a losing full maintenance OPTIMIZE checkpoint + * cannot be rebased and the commit fails. The caller could retry the full maintenance + * OPTIMIZE checkpoint if needed. + * + * @param conflictingCommitVersion the version at which the conflicting winner committed. + */ +class FullAMTWriteFailedWithConflict( + val conflictingCommitVersion: Long) + extends io.delta.exceptions.DeltaConcurrentModificationException( + s"A concurrent commit at version $conflictingCommitVersion changed content this full AMT " + + "checkpoint describes; it must be regenerated against the updated snapshot.") + +/** + * Thrown by the AMT write path when a losing maintenance OPTIMIZE checkpoint finds that a + * concurrent winning commit already installed a new AMT tree that makes this checkpoint redundant. + * + * Unlike [[io.delta.exceptions.ConcurrentWriteException]], this is never surfaced to the caller: + * the maintenance checkpoint commit site ([[org.apache.spark.sql.delta.amt.AMTUtils]] + * `emitAMTCheckpoint`) catches it and treats the checkpoint as a graceful no-op, because the + * winner's tree -- committed at `manifestCommitVersion`, describing content-root version + * `contentRootVersion` -- already provides an up-to-date AMT. + * Since this is not user-facing, it does not use an `errorClass`. + * + * @param manifestCommitVersion the version at which the winner committed the superseding AMT tree. + * @param contentRootVersion the table version that winner's content root describes. + */ +class ConcurrentAMTCheckpointLandedException( + val manifestCommitVersion: Long, + val contentRootVersion: Long) + extends io.delta.exceptions.DeltaConcurrentModificationException( + s"A concurrent commit already installed an up-to-date AMT tree (manifest commit version " + + s"$manifestCommitVersion, content-root version $contentRootVersion); the losing " + + "maintenance checkpoint is redundant and was skipped.") + /** * Thrown when time travelling to a version that does not exist in the Delta Log. * @param userVersion - the version time travelling to @@ -4390,6 +4581,22 @@ class ConcurrentTransactionException(message: String) conflictingCommit)) } +/** + * Thrown when the current transaction adds a metadata domain that conflicts with a metadata domain + * added by a concurrent transaction. Subclasses + * [[io.delta.exceptions.ConcurrentTransactionException]] for backward compatibility, but reports + * its own error class. + */ +class ConflictingMetadataDomainException(messageParameters: Array[String] = Array.empty) + extends io.delta.exceptions.ConcurrentTransactionException( + DeltaThrowableHelper.getMessage("DELTA_CONFLICTING_METADATA_DOMAIN", messageParameters)) + with DeltaThrowable { + override def getErrorClass: String = "DELTA_CONFLICTING_METADATA_DOMAIN" + override def getMessageParameters: java.util.Map[String, String] = + DeltaThrowableHelper.getMessageParameters( + "DELTA_CONFLICTING_METADATA_DOMAIN", errorSubClass = null, messageParameters) +} + /** A helper class in building a helpful error message in case of metadata mismatches. */ class MetadataMismatchErrorBuilder { private var subErrors: Seq[(String, Array[String])] = Nil diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaFileProviderUtils.scala b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaFileProviderUtils.scala index c8ec841a6ac..4832506cdf2 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaFileProviderUtils.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaFileProviderUtils.scala @@ -102,9 +102,8 @@ object DeltaFileProviderUtils extends DeltaLogging { */ def parallelReadAndParseDeltaFilesAsIterator( spark: SparkSession, - deltaLog: DeltaLog, commits: Seq[SingleCommit]): Seq[ClosableIterator[Action]] = - parallelReadDeltaFiles(spark, deltaLog, commits)(_.getActionsIterator()) + parallelReadDeltaFiles(spark, commits)(_.getActionsIterator()) /** * Parallel-read a set of commits off the driver, materializing each commit's [[Action]]s into a @@ -112,23 +111,28 @@ object DeltaFileProviderUtils extends DeltaLogging { */ def parallelReadAndParseDeltaFilesAsSeq( spark: SparkSession, - deltaLog: DeltaLog, commits: Seq[SingleCommit]): Seq[Seq[Action]] = - parallelReadDeltaFiles(spark, deltaLog, commits) { commit => + parallelReadDeltaFiles(spark, commits) { commit => commit.getActionsIterator().processAndClose(_.toSeq) } + /** + * Like [[parallelReadAndParseDeltaFilesAsSeq]], but asserts each commit is a log commit. + */ + private[delta] def parallelReadAndParseLogCommitsAsSeqUnsafe( + spark: SparkSession, + commits: Seq[SingleCommit]): Seq[Seq[Action]] = + parallelReadDeltaFiles(spark, commits) { commit => + commit.getLogCommitActionsIteratorUnsafe().processAndClose(_.toSeq) + } + /** * Parallel-read a set of commits off the driver, applying `f` to each. The commits are read * concurrently on the driver delta-log thread pool; results are returned in the input order. + * Only derive `A` from `commits`. */ protected def parallelReadDeltaFiles[A]( spark: SparkSession, - deltaLog: DeltaLog, - commits: Seq[SingleCommit])(f: SingleCommit => A): Seq[A] = { - val hadoopConf = deltaLog.newDeltaHadoopConf() - readThreadPool.parallelMap(spark, commits) { commit => - f(commit) - }.toSeq - } + commits: Seq[SingleCommit])(f: SingleCommit => A): Seq[A] = + readThreadPool.parallelMap(spark, commits)(f).toSeq } diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaHistoryManager.scala b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaHistoryManager.scala index 7bab91f2d31..45f2a43183f 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaHistoryManager.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaHistoryManager.scala @@ -25,7 +25,7 @@ import scala.collection.mutable import scala.concurrent.{ExecutionContext, ExecutionContextExecutorService, Future} import scala.concurrent.duration.Duration -import org.apache.spark.sql.delta.actions.{Action, CommitInfo, CommitMarker, JobInfo, NotebookInfo} +import org.apache.spark.sql.delta.actions.{Action, CommitInfo, CommitMarker, JobInfo, NotebookInfo, RemoveFile, SingleAction} import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.sources.DeltaSQLConf @@ -39,8 +39,10 @@ import org.apache.hadoop.fs.{FileStatus, Path} import org.apache.spark.SparkEnv import org.apache.spark.internal.MDC +import org.apache.spark.sql.Dataset import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.catalog.CatalogTable +import org.apache.spark.sql.functions.col import org.apache.spark.sql.internal.SQLConf import org.apache.spark.util.{SerializableConfiguration, ThreadUtils} @@ -65,6 +67,77 @@ class DeltaHistoryManager( import DeltaHistoryManager._ + /** + * Returns all staged and committed file actions (as raw `SingleAction`s) in the commits in the + * inclusive version range `[startVersion, endVersion]`, read directly by traversing the commit + * files. + * + * This includes actions from unbackfilled commit files under `_delta_log/_commits/`, which may + * be staged commits that were never actually committed to the table. + * + * This reads the file actions straight from the commit JSONs. Note that a commit JSON is not + * required to carry its file actions: the Iceberg V4 metadata protocol does not mandate them, + * writes from external Iceberg clients may already omit them, and the current implementation + * that writes them inline (`includeActionsInCommitJson = true`, alongside the manifest tree) may + * stop doing so in the future. When the actions are absent this method will not surface them and + * callers must source them elsewhere. The authoritative spec is: + * https://github.com/delta-io/delta/blob/master/protocol_rfcs/iceberg-v4-metadata.md + */ + def getAllStagedAndCommittedFileActions( + startVersion: Long, + endVersion: Long): Dataset[SingleAction] = { + import org.apache.spark.sql.delta.implicits._ + require(startVersion <= endVersion, + s"startVersion ($startVersion) must be <= endVersion ($endVersion)") + val hadoopConf = deltaLog.newDeltaHadoopConf() + + // Backfilled commits in `_delta_log/`. + val backfilledCommitFiles = deltaLog.store + .listFrom(listingPrefix(deltaLog.logPath, startVersion), hadoopConf) + .collect { case DeltaFile(f, version) => (f, version) } + .takeWhile(_._2 <= endVersion) + .toSeq + + // Unbackfilled commits in `_delta_log/_commits/`. Start listing from the last backfilled + // version we found so the two listings bridge without gaps. The directory may not exist on + // tables created in older versions. + val commitDirPath = FileNames.commitDirPath(deltaLog.logPath) + val fs = deltaLog.logPath.getFileSystem(hadoopConf) + val updatedStartVersion = + backfilledCommitFiles.lastOption.map(_._2).getOrElse(startVersion) + val unbackfilledCommitFiles = if (fs.exists(commitDirPath)) { + deltaLog.store + .listFrom(listingPrefix(commitDirPath, updatedStartVersion), hadoopConf) + .collect { case UnbackfilledDeltaFile(f, version, _) => (f, version) } + .takeWhile(_._2 <= endVersion) + .toSeq + } else { + Seq.empty + } + + val allCommitFiles = (backfilledCommitFiles ++ unbackfilledCommitFiles).map(_._1) + if (allCommitFiles.isEmpty) { + return spark.emptyDataset[SingleAction] + } + val commitFileIndex = + DeltaLogFileIndex(DeltaLogFileIndex.COMMIT_FILE_FORMAT, allCommitFiles).get + deltaLog.loadIndex(commitFileIndex).as[SingleAction] + } + + /** + * Returns the `RemoveFile` (tombstone) actions for the removes within the inclusive version + * range `[startVersion, endVersion]`, derived by traversing the commits in the range (the + * removes-only view of [[getAllStagedAndCommittedFileActions]]). + */ + def getRemoveFileActions( + startVersion: Long, + endVersion: Long): Dataset[RemoveFile] = { + import org.apache.spark.sql.delta.implicits._ + getAllStagedAndCommittedFileActions(startVersion, endVersion) + .where("remove IS NOT NULL") + .select(col("remove").as[RemoveFile]) + } + /** * Returns the information of the latest `limit` commits made to this table in reverse * chronological order. diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaLog.scala b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaLog.scala index 14126e62e67..15ac5bfaa4f 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaLog.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaLog.scala @@ -29,7 +29,6 @@ import scala.util.control.NonFatal import com.databricks.spark.util.TagDefinition import com.databricks.spark.util.TagDefinitions._ -import org.apache.spark.sql.delta.v2.interop.DeltaV2TableManager import org.apache.spark.sql.delta.DataFrameUtils import org.apache.spark.sql.delta.ClassicColumnConversions._ import org.apache.spark.sql.delta.actions._ @@ -41,7 +40,7 @@ import org.apache.spark.sql.delta.redirect.RedirectFeature import org.apache.spark.sql.delta.schema.{SchemaMergingUtils, SchemaUtils} import org.apache.spark.sql.delta.sources._ import org.apache.spark.sql.delta.storage.LogStoreProvider -import org.apache.spark.sql.delta.util.{FileNames, PathWithFileSystem, Utils => DeltaUtils} +import org.apache.spark.sql.delta.util.{DeltaFileSystemOptions, FileNames, PathWithFileSystem, Utils => DeltaUtils} import com.google.common.cache.{Cache, CacheBuilder, RemovalNotification} import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileStatus, FileSystem, Path} @@ -91,8 +90,7 @@ class DeltaLog private( with DeltaFileFormat with ProvidesUniFormConverters with ReadChecksum - with DeltaLoggingProvider - with DeltaV2TableManager { + with DeltaLoggingProvider { import org.apache.spark.sql.delta.files.TahoeFileIndex import org.apache.spark.sql.delta.util.FileNames._ @@ -1060,15 +1058,7 @@ object DeltaLog extends DeltaLogging { spark: SparkSession, options: Map[String, String], rootPath: Path): Path = { - val fileSystemOptions: Map[String, String] = - if (spark.sessionState.conf.getConf( - DeltaSQLConf.LOAD_FILE_SYSTEM_CONFIGS_FROM_DATAFRAME_OPTIONS)) { - options.filterKeys { k => - DeltaTableUtils.validDeltaTableHadoopPrefixes.exists(k.startsWith) - }.toMap - } else { - Map.empty - } + val fileSystemOptions = DeltaFileSystemOptions.buildFsOptions(spark, options) // scalastyle:off deltahadoopconfiguration val hadoopConf = spark.sessionState.newHadoopConfWithOptions(fileSystemOptions) // scalastyle:on deltahadoopconfiguration @@ -1100,25 +1090,8 @@ object DeltaLog extends DeltaLogging { initialCatalogTable: Option[CatalogTable], clock: Clock ): DeltaLog = { - // Construct the filesystem options based on the DataFrameReader/Writer options, and if it's - // a catalog based table, we need combine both options and catalog-based table storage - // properties since all cloud credential information are stored in storage properties. - val catalogTableStorageProps = initialCatalogTable - .map(t => t.storage.properties.filter { case (k, _) => - DeltaTableUtils.validDeltaTableHadoopPrefixes.exists(k.startsWith) - }) - .getOrElse(Map.empty) - val fileSystemOptions: Map[String, String] = - if (spark.sessionState.conf.getConf( - DeltaSQLConf.LOAD_FILE_SYSTEM_CONFIGS_FROM_DATAFRAME_OPTIONS)) { - // We pick up only file system options so that we don't pass any parquet or json options to - // the code that reads Delta transaction logs. - catalogTableStorageProps ++ options.filterKeys { k => - DeltaTableUtils.validDeltaTableHadoopPrefixes.exists(k.startsWith) - }.toMap - } else { - catalogTableStorageProps - } + val fileSystemOptions = + DeltaFileSystemOptions.buildFsOptions(spark, options, initialCatalogTable) // scalastyle:off deltahadoopconfiguration val hadoopConf = spark.sessionState.newHadoopConfWithOptions(fileSystemOptions) @@ -1277,19 +1250,12 @@ object DeltaLog extends DeltaLogging { partitionFilters: Seq[Expression], partitionColumnPrefixes: Seq[String] = Nil, shouldRewritePartitionFilters: Boolean = true): DataFrame = { - - val rewrittenFilters = if (shouldRewritePartitionFilters) { - rewritePartitionFilters( - partitionSchema, - files.sparkSession.sessionState.conf.resolver, - partitionFilters, - partitionColumnPrefixes) - } else { - partitionFilters - } - val expr = rewrittenFilters.reduceLeftOption(And).getOrElse(Literal.TrueLiteral) - val columnFilter = Column(expr) - files.filter(columnFilter) + DeltaLogUtils.filterFileList( + partitionSchema, + files, + partitionFilters, + partitionColumnPrefixes, + shouldRewritePartitionFilters) } /** @@ -1306,26 +1272,8 @@ object DeltaLog extends DeltaLogging { resolver: Resolver, partitionFilters: Seq[Expression], partitionColumnPrefixes: Seq[String] = Nil): Seq[Expression] = { - partitionFilters - .map(_.transformUp { - case a: Attribute => - // If we have a special column name, e.g. `a.a`, then an UnresolvedAttribute returns - // the column name as '`a.a`' instead of 'a.a', therefore we need to strip the backticks. - val unquoted = a.name.stripPrefix("`").stripSuffix("`") - val partitionCol = partitionSchema.find { field => resolver(field.name, unquoted) } - partitionCol match { - case Some(f: StructField) => - val name = DeltaColumnMapping.getPhysicalName(f) - Cast( - UnresolvedAttribute(partitionColumnPrefixes ++ Seq("partitionValues", name)), - f.dataType) - case None => - // This should not be able to happen, but the case was present in the original code so - // we kept it to be safe. - log.error(s"Partition filter referenced column ${a.name} not in the partition schema") - UnresolvedAttribute(partitionColumnPrefixes ++ Seq("partitionValues", a.name)) - } - }) + DeltaLogUtils.rewritePartitionFilters( + partitionSchema, resolver, partitionFilters, partitionColumnPrefixes) } @@ -1375,3 +1323,72 @@ object DeltaLog extends DeltaLogging { } } } + +/** Partition-filtering helpers shared by DeltaLog and snapshot readers. */ +object DeltaLogUtils extends DeltaLogging { + + /** + * Filters the given [[Dataset]] by the given `partitionFilters`, returning those that match. + * @param files The active files in the log state, which contain partition value information + * @param partitionFilters Filters on the partition columns + * @param partitionColumnPrefixes The path to the `partitionValues` column, if it's nested + * @param shouldRewritePartitionFilters Whether to rewrite `partitionFilters` to be over the + * [[AddFile]] schema + */ + def filterFileList( + partitionSchema: StructType, + files: DataFrame, + partitionFilters: Seq[Expression], + partitionColumnPrefixes: Seq[String] = Nil, + shouldRewritePartitionFilters: Boolean = true): DataFrame = { + + val rewrittenFilters = if (shouldRewritePartitionFilters) { + rewritePartitionFilters( + partitionSchema, + files.sparkSession.sessionState.conf.resolver, + partitionFilters, + partitionColumnPrefixes) + } else { + partitionFilters + } + val expr = rewrittenFilters.reduceLeftOption(And).getOrElse(Literal.TrueLiteral) + val columnFilter = Column(expr) + files.filter(columnFilter) + } + + /** + * Rewrite the given `partitionFilters` to be used for filtering partition values. + * We need to explicitly resolve the partitioning columns here because the partition columns + * are stored as keys of a Map type instead of attributes in the AddFile schema and thus + * cannot be resolved automatically. + * + * @param partitionFilters Filters on the partition columns + * @param partitionColumnPrefixes The path to the `partitionValues` column, if it's nested + */ + def rewritePartitionFilters( + partitionSchema: StructType, + resolver: Resolver, + partitionFilters: Seq[Expression], + partitionColumnPrefixes: Seq[String] = Nil): Seq[Expression] = { + partitionFilters + .map(_.transformUp { + case a: Attribute => + // If we have a special column name, e.g. `a.a`, then an UnresolvedAttribute returns + // the column name as '`a.a`' instead of 'a.a', therefore we need to strip the backticks. + val unquoted = a.name.stripPrefix("`").stripSuffix("`") + val partitionCol = partitionSchema.find { field => resolver(field.name, unquoted) } + partitionCol match { + case Some(f: StructField) => + val name = DeltaColumnMapping.getPhysicalName(f) + Cast( + UnresolvedAttribute(partitionColumnPrefixes ++ Seq("partitionValues", name)), + f.dataType) + case None => + // This should not be able to happen, but the case was present in the original code so + // we kept it to be safe. + log.error(s"Partition filter referenced column ${a.name} not in the partition schema") + UnresolvedAttribute(partitionColumnPrefixes ++ Seq("partitionValues", a.name)) + } + }) + } +} diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaLogFileIndex.scala b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaLogFileIndex.scala index 08633d98421..14ca077f2c5 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaLogFileIndex.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaLogFileIndex.scala @@ -16,6 +16,7 @@ package org.apache.spark.sql.delta +import org.apache.spark.sql.delta.RowIndexFilterProvider import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.util.FileNames import org.apache.hadoop.fs._ @@ -23,7 +24,12 @@ import org.apache.hadoop.fs._ import org.apache.spark.internal.{Logging, MDC} import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.sql.execution.datasources.{FileFormat, FileIndex, PartitionDirectory} +import org.apache.spark.sql.execution.datasources.{ + FileFormat, + FileIndex, + FileStatusWithMetadata, + PartitionDirectory +} import org.apache.spark.sql.execution.datasources.json.JsonFileFormat import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat import org.apache.spark.sql.types.{LongType, StructField, StructType} @@ -38,24 +44,40 @@ import org.apache.spark.sql.types.{LongType, StructField, StructType} */ class DeltaLogFileIndex private[delta] ( val format: FileFormat, - val files: Array[FileStatus] - ) + val files: Array[FileStatus], + val perFileRowIndexFilters: Map[String, RowIndexFilterProvider] = Map.empty) extends FileIndex with Logging { import DeltaLogFileIndex._ - override lazy val rootPaths: Seq[Path] = files.map(_.getPath).toSeq def listAllFiles(): Seq[PartitionDirectory] = { files .groupBy(f => FileNames.getFileVersionOpt(f.getPath).getOrElse(-1L)) .map { case (version, versionFiles) => - PartitionDirectory(InternalRow(version), versionFiles) + fileStatusToPartitionDirectory(version, versionFiles.toSeq) } .toSeq } + private def fileStatusToPartitionDirectory( + version: Long, + versionFiles: Seq[FileStatus]): PartitionDirectory = { + if (perFileRowIndexFilters.nonEmpty) { + val statuses = versionFiles.map { file => + val path = file.getPath.toString + val metadata = perFileRowIndexFilters.get(path) + .map(ROW_INDEX_FILTER_PROVIDER_METADATA_KEY -> _) + .toMap + FileStatusWithMetadata(file, metadata) + }.toIndexedSeq + PartitionDirectory(InternalRow(version), statuses) + } else { + PartitionDirectory(InternalRow(version), versionFiles.toArray) + } + } + override def listFiles( partitionFilters: Seq[Expression], dataFilters: Seq[Expression]): Seq[PartitionDirectory] = { @@ -89,6 +111,9 @@ class DeltaLogFileIndex private[delta] ( } object DeltaLogFileIndex { + /** Per-file metadata key used to carry a portable row-index-filter provider to executors. */ + private[delta] val ROW_INDEX_FILTER_PROVIDER_METADATA_KEY = "row_index_filter_provider" + val COMMIT_VERSION_COLUMN = "version" lazy val COMMIT_FILE_FORMAT = new JsonFileFormat @@ -112,4 +137,13 @@ object DeltaLogFileIndex { filesOpt.flatMap(DeltaLogFileIndex(format, _)) } + /** + * Builds an index that carries per-file row-index filters. + */ + def apply( + format: FileFormat, + files: Array[FileStatus], + perFileRowIndexFilters: Map[String, RowIndexFilterProvider]): DeltaLogFileIndex = { + new DeltaLogFileIndex(format, files, perFileRowIndexFilters = perFileRowIndexFilters) + } } diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaOperations.scala b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaOperations.scala index 058ae644cd0..b9424217f1d 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaOperations.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaOperations.scala @@ -106,6 +106,12 @@ object DeltaOperations { */ def isInPlaceFileMetadataUpdate: Option[Boolean] + /** + * Whether we expect logical data changes from this operation. + * Validated during commit, gated by `DELTA_COMMIT_VALIDATE_EXPECTED_DATA_CHANGE_MODE`. + */ + def expectedFileDataChange: Option[Boolean] + /** * Whether this operation is allowed to change the set and order of partition columns. @@ -212,6 +218,9 @@ object DeltaOperations { override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) + // The dataChange write option allows WRITE to be used for data-rearrangement operations. + override def expectedFileDataChange: Option[Boolean] = None + def shouldCollectInsertReplaceMetrics: Boolean = (predicate.nonEmpty && replaceWhereMetricsEnabled) || replaceOnCond.nonEmpty || @@ -234,6 +243,8 @@ object DeltaOperations { override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) + override def expectedFileDataChange: Option[Boolean] = Some(false) + override def canChangePartitionColumns: Boolean = false } @@ -255,6 +266,8 @@ object DeltaOperations { override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) + override def expectedFileDataChange: Option[Boolean] = Some(true) + override def canChangePartitionColumns: Boolean = false } /** Recorded while deleting certain partitions. */ @@ -284,6 +297,8 @@ object DeltaOperations { override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) + override def expectedFileDataChange: Option[Boolean] = Some(true) + override def canChangePartitionColumns: Boolean = false } /** Recorded when truncating the table. */ @@ -297,6 +312,8 @@ object DeltaOperations { override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) + override def expectedFileDataChange: Option[Boolean] = Some(true) + override def canChangePartitionColumns: Boolean = false } @@ -321,6 +338,8 @@ object DeltaOperations { override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) + override def expectedFileDataChange: Option[Boolean] = Some(true) + override def canChangePartitionColumns: Boolean = false } @@ -397,6 +416,8 @@ object DeltaOperations { override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) + override def expectedFileDataChange: Option[Boolean] = Some(true) + override def canChangePartitionColumns: Boolean = false } @@ -436,6 +457,8 @@ object DeltaOperations { override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) + override def expectedFileDataChange: Option[Boolean] = Some(true) + override def canChangePartitionColumns: Boolean = false } /** Recorded when the table is created. */ @@ -464,6 +487,8 @@ object DeltaOperations { override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) + override def expectedFileDataChange: Option[Boolean] = Some(true) + override def canChangePartitionColumns: Boolean = true } /** Recorded when the table is replaced. */ @@ -515,6 +540,8 @@ object DeltaOperations { override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) + override def expectedFileDataChange: Option[Boolean] = Some(true) + // We allow ReplaceTable operations to change partition columns when they are // 1) creating/replacing a new table, 2) not invoked via saveAsTable or 3) invoked via // saveAsTable but with schema overwrite. @@ -534,6 +561,8 @@ object DeltaOperations { override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) + override def expectedFileDataChange: Option[Boolean] = Some(false) + override def canChangePartitionColumns: Boolean = false } /** Recorded when the table properties are unset. */ @@ -549,6 +578,8 @@ object DeltaOperations { override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) + override def expectedFileDataChange: Option[Boolean] = Some(false) + override def canChangePartitionColumns: Boolean = false } /** Recorded when dropping a table feature. */ @@ -567,6 +598,8 @@ object DeltaOperations { override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(true) + override def expectedFileDataChange: Option[Boolean] = Some(false) + override def canChangePartitionColumns: Boolean = false } @@ -583,6 +616,8 @@ object DeltaOperations { override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) + override def expectedFileDataChange: Option[Boolean] = Some(false) + override def canChangePartitionColumns: Boolean = false } @@ -603,6 +638,8 @@ object DeltaOperations { override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) + override def expectedFileDataChange: Option[Boolean] = Some(true) + override def canChangePartitionColumns: Boolean = false } @@ -619,6 +656,8 @@ object DeltaOperations { override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) + override def expectedFileDataChange: Option[Boolean] = Some(true) + override def canChangePartitionColumns: Boolean = false } @@ -636,6 +675,8 @@ object DeltaOperations { override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) + override def expectedFileDataChange: Option[Boolean] = Some(true) + override def canChangePartitionColumns: Boolean = true } @@ -655,6 +696,8 @@ object DeltaOperations { override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) + override def expectedFileDataChange: Option[Boolean] = Some(true) + override def canChangePartitionColumns: Boolean = false } @@ -673,6 +716,8 @@ object DeltaOperations { override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) + override def expectedFileDataChange: Option[Boolean] = Some(true) + override def canChangePartitionColumns: Boolean = false } @@ -688,6 +733,8 @@ object DeltaOperations { override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) + override def expectedFileDataChange: Option[Boolean] = Some(true) + override def canChangePartitionColumns: Boolean = false } @@ -704,6 +751,8 @@ object DeltaOperations { override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) + override def expectedFileDataChange: Option[Boolean] = Some(false) + override def canChangePartitionColumns: Boolean = false } @@ -717,6 +766,8 @@ object DeltaOperations { // adding a new Delta operation. For test-only code use TestOperation. override val isInPlaceFileMetadataUpdate: Option[Boolean] = None + override def expectedFileDataChange: Option[Boolean] = None + override def canChangePartitionColumns: Boolean = true } @@ -729,6 +780,8 @@ object DeltaOperations { override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) + override def expectedFileDataChange: Option[Boolean] = Some(false) + override def canChangePartitionColumns: Boolean = false } @@ -747,6 +800,8 @@ object DeltaOperations { override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) + override def expectedFileDataChange: Option[Boolean] = Some(false) + override def canChangePartitionColumns: Boolean = false } @@ -761,6 +816,8 @@ object DeltaOperations { override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) + override def expectedFileDataChange: Option[Boolean] = Some(false) + override def canChangePartitionColumns: Boolean = false } @@ -773,6 +830,8 @@ object DeltaOperations { override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) + override def expectedFileDataChange: Option[Boolean] = Some(false) + override def canChangePartitionColumns: Boolean = false } @@ -791,6 +850,8 @@ object DeltaOperations { override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) + override def expectedFileDataChange: Option[Boolean] = Some(false) + override def canChangePartitionColumns: Boolean = false } @@ -805,6 +866,8 @@ object DeltaOperations { // ComputeStats operation only updates statistics of existing files. override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(true) + override def expectedFileDataChange: Option[Boolean] = Some(false) + override def canChangePartitionColumns: Boolean = false } @@ -829,6 +892,9 @@ object DeltaOperations { // identifiers, meaning that metadata differences are ignored. override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) + // RESTORE can deliberately stamp file actions with either value via RESTORE_DATA_CHANGE. + override def expectedFileDataChange: Option[Boolean] = None + override def canChangePartitionColumns: Boolean = false } @@ -877,6 +943,8 @@ object DeltaOperations { override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) + override def expectedFileDataChange: Option[Boolean] = Some(false) + override def canChangePartitionColumns: Boolean = false } @@ -893,6 +961,8 @@ object DeltaOperations { override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) + override def expectedFileDataChange: Option[Boolean] = Some(false) + override def canChangePartitionColumns: Boolean = false } @@ -915,6 +985,8 @@ object DeltaOperations { override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) + override def expectedFileDataChange: Option[Boolean] = Some(true) + override def canChangePartitionColumns: Boolean = true } @@ -939,6 +1011,8 @@ object DeltaOperations { override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) + override def expectedFileDataChange: Option[Boolean] = Some(false) + override def canChangePartitionColumns: Boolean = false } @@ -961,6 +1035,8 @@ object DeltaOperations { override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) + override def expectedFileDataChange: Option[Boolean] = Some(false) + override def canChangePartitionColumns: Boolean = false } @@ -983,6 +1059,8 @@ object DeltaOperations { override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) + override def expectedFileDataChange: Option[Boolean] = Some(false) + override def canChangePartitionColumns: Boolean = false } @@ -999,9 +1077,35 @@ object DeltaOperations { override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) + override def expectedFileDataChange: Option[Boolean] = Some(false) + override def canChangePartitionColumns: Boolean = false } + /** + * Recorded when the partitioning of a table is replaced with clustering (or no layout) via + * `ALTER TABLE ... REPLACE PARTITIONED BY WITH CLUSTER BY`. + */ + case class ReplacePartitionedByWithClusterBy( + oldPartitioningColumns: String, + newClusteringColumns: String) extends Operation("REPLACE PARTITIONED BY WITH CLUSTER BY") { + override val parameters: Map[String, Any] = Map( + "oldPartitioningColumns" -> oldPartitioningColumns, + "newClusteringColumns" -> newClusteringColumns) + + // Existing AddFiles are re-added with their DVs and stats copied over (plus synthesized + // stats for the former partition columns). Stats may have been recomputed as tight bounds + // by ComputeStats before, so this check should be disabled. + override def checkAddFileWithDeletionVectorStatsAreNotTightBounds: Boolean = false + + // Only partition values and stats of existing files are updated in place. + override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(true) + + override def expectedFileDataChange: Option[Boolean] = Some(false) + + override def canChangePartitionColumns: Boolean = true + } + /** Recorded when we backfill a Delta table's existing AddFiles with row tracking data. */ case class RowTrackingBackfill( batchId: Int = 0) extends Operation(ROW_TRACKING_BACKFILL_OPERATION_NAME) { @@ -1016,6 +1120,8 @@ object DeltaOperations { // RowTrackingBackfill only updates tags of existing files. override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(true) + override def expectedFileDataChange: Option[Boolean] = Some(false) + override def canChangePartitionColumns: Boolean = false } @@ -1036,6 +1142,8 @@ object DeltaOperations { // RowTrackingUnBackfill only updates metadata of existing files. override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(true) + override def expectedFileDataChange: Option[Boolean] = Some(false) + override def canChangePartitionColumns: Boolean = false } @@ -1063,6 +1171,8 @@ object DeltaOperations { // Only removes domain metadata. override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) + override def expectedFileDataChange: Option[Boolean] = Some(false) + override def canChangePartitionColumns: Boolean = false } @@ -1081,6 +1191,7 @@ object DeltaOperations { override val isInPlaceFileMetadataUpdate: Option[Boolean] = None ) extends Operation(operationName) { override val parameters: Map[String, Any] = Map.empty + override def expectedFileDataChange: Option[Boolean] = None // Perform the check for testing. override def checkAddFileWithDeletionVectorStatsAreNotTightBounds: Boolean = true @@ -1112,6 +1223,8 @@ object DeltaOperations { override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) + override def expectedFileDataChange: Option[Boolean] = Some(false) + override def canChangePartitionColumns: Boolean = false } } diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaOptions.scala b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaOptions.scala index 8ffecdbeeba..c81d5b0155b 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaOptions.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaOptions.scala @@ -39,7 +39,7 @@ trait DeltaOptionParser { def toBoolean(input: String, name: String): Boolean = { Try(input.toBoolean).toOption.getOrElse { - throw DeltaErrors.illegalDeltaOptionException(name, input, "must be 'true' or 'false'") + throw DeltaErrors.illegalDeltaOptionMustBeBoolean(name, input) } } } @@ -68,10 +68,9 @@ trait DeltaWriteOptions cols.split(/* separator = */ ",", /* limit = */ -1).map(_.trim).toSeq.distinct // scalastyle:on if (parsed.exists(_.isEmpty)) { - throw DeltaErrors.illegalDeltaOptionException( + throw DeltaErrors.illegalDeltaOptionNoEmptyColumnNames( name = REPLACE_USING_OPTION, - input = cols, - explain = "must not contain empty column names") + input = cols) } parsed } @@ -141,10 +140,13 @@ trait DeltaWriteOptionsImpl extends DeltaOptionParser { } val txnVersion = options.get(TXN_VERSION).map { str => - Try(str.toLong).toOption.filter(_ >= 0).getOrElse { - throw DeltaErrors.illegalDeltaOptionException( - TXN_VERSION, str, "must be a non-negative integer") + val version = Try(str.toLong).toOption.getOrElse { + throw DeltaErrors.illegalDeltaOptionMustBeInteger(TXN_VERSION, str) } + if (version < 0) { + throw DeltaErrors.illegalDeltaOptionMustBeNonNegativeNumber(TXN_VERSION, str) + } + version } val txnAppId = options.get(TXN_APP_ID) @@ -154,8 +156,7 @@ trait DeltaWriteOptionsImpl extends DeltaOptionParser { // neither must be given. In all other cases, throw an exception. val numOptions = txnVersion.size + txnAppId.size if (numOptions != 0 && numOptions != 2) { - throw DeltaErrors.invalidIdempotentWritesOptionsException("Both txnVersion and txnAppId " + - "must be specified for idempotent data frame writes") + throw DeltaErrors.invalidIdempotentWritesMissingWriteOptionsException() } } @@ -181,11 +182,9 @@ trait DeltaWriteOptionsImpl extends DeltaOptionParser { } else { if (mode == null || !DeltaOptions.PARTITION_OVERWRITE_MODE_VALUES.exists(mode.equalsIgnoreCase(_))) { - val acceptableStr = - DeltaOptions.PARTITION_OVERWRITE_MODE_VALUES.map("'" + _ + "'").mkString(" or ") - throw DeltaErrors.illegalDeltaOptionException( - PARTITION_OVERWRITE_MODE_OPTION, mode, s"must be ${acceptableStr}" - ) + throw DeltaErrors.illegalDeltaOptionMustBeOneOf( + PARTITION_OVERWRITE_MODE_OPTION, mode, + DeltaOptions.PARTITION_OVERWRITE_MODE_VALUES.map("'" + _ + "'").toSeq) } modeIsDynamic } @@ -196,16 +195,19 @@ trait DeltaReadOptions extends DeltaOptionParser { import DeltaOptions._ val maxFilesPerTrigger = options.get(MAX_FILES_PER_TRIGGER_OPTION).map { str => - Try(str.toInt).toOption.filter(_ > 0).getOrElse { - throw DeltaErrors.illegalDeltaOptionException( - MAX_FILES_PER_TRIGGER_OPTION, str, "must be a positive integer") + val maxFiles = Try(str.toInt).toOption.getOrElse { + throw DeltaErrors.illegalDeltaOptionMustBeInteger(MAX_FILES_PER_TRIGGER_OPTION, str) + } + if (maxFiles <= 0) { + throw DeltaErrors.illegalDeltaOptionMustBePositiveNumber(MAX_FILES_PER_TRIGGER_OPTION, str) } + maxFiles } val maxBytesPerTrigger = options.get(MAX_BYTES_PER_TRIGGER_OPTION).map { str => Try(JavaUtils.byteStringAs(str, ByteUnit.BYTE)).toOption.filter(_ > 0).getOrElse { - throw DeltaErrors.illegalDeltaOptionException( - MAX_BYTES_PER_TRIGGER_OPTION, str, "must be a size configuration such as '10g'") + throw DeltaErrors.illegalDeltaOptionMustBeSizeConfiguration( + MAX_BYTES_PER_TRIGGER_OPTION, str) } } @@ -234,10 +236,13 @@ trait DeltaReadOptions extends DeltaOptionParser { val startingVersion: Option[DeltaStartingVersion] = options.get(STARTING_VERSION_OPTION).map { case "latest" => StartingVersionLatest case str => - Try(str.toLong).toOption.filter(_ >= 0).map(StartingVersion).getOrElse{ - throw DeltaErrors.illegalDeltaOptionException( - STARTING_VERSION_OPTION, str, "must be greater than or equal to zero") + val version = Try(str.toLong).toOption.getOrElse { + throw DeltaErrors.illegalDeltaOptionMustBeInteger(STARTING_VERSION_OPTION, str) + } + if (version < 0) { + throw DeltaErrors.illegalDeltaOptionMustBeNonNegativeNumber(STARTING_VERSION_OPTION, str) } + StartingVersion(version) } val startingTimestamp = options.get(STARTING_TIMESTAMP_OPTION) @@ -400,9 +405,11 @@ object DeltaOptions extends DeltaLogging { USE_NULL_INTOLERANT_EQUALITY_WITH_DPO, PARTITION_OVERWRITE_MODE_OPTION, MAX_FILES_PER_TRIGGER_OPTION, + MAX_BYTES_PER_TRIGGER_OPTION, IGNORE_FILE_DELETION_OPTION, IGNORE_CHANGES_OPTION, IGNORE_DELETES_OPTION, + SKIP_CHANGE_COMMITS_OPTION, FAIL_ON_DATA_LOSS_OPTION, OPTIMIZE_WRITE_OPTION, DATA_CHANGE_OPTION, @@ -423,6 +430,9 @@ object DeltaOptions extends DeltaLogging { SCHEMA_TRACKING_LOCATION, SCHEMA_TRACKING_LOCATION_ALIAS, STREAMING_SOURCE_TRACKING_ID, + ALLOW_SOURCE_COLUMN_RENAME, + ALLOW_SOURCE_COLUMN_DROP, + ALLOW_SOURCE_COLUMN_TYPE_CHANGE, "queryName", "checkpointLocation", "path", diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaParquetFileFormat.scala b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaParquetFileFormat.scala index b8d2b875a67..9b3c4ac3a49 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaParquetFileFormat.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaParquetFileFormat.scala @@ -16,11 +16,11 @@ package org.apache.spark.sql.delta -import scala.collection.mutable.ArrayBuffer import scala.util.control.NonFatal import org.apache.spark.sql.delta.RowIndexFilterType import org.apache.spark.sql.delta.DeltaParquetFileFormat._ +import org.apache.spark.sql.delta.RowIndexFilterMetadataColumnUtils.ColumnMetadata import org.apache.spark.sql.delta.actions.{DeletionVectorDescriptor, Metadata, Protocol} import org.apache.spark.sql.delta.commands.DeletionVectorUtils.deletionVectorsReadable import org.apache.spark.sql.delta.deletionvectors.{DropMarkedRowsFilter, KeepAllRowsFilter, KeepMarkedRowsFilter} @@ -41,11 +41,9 @@ import org.apache.spark.sql.catalyst.expressions.FileSourceConstantMetadataStruc import org.apache.spark.sql.execution.datasources.OutputWriterFactory import org.apache.spark.sql.execution.datasources.PartitionedFile import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat -import org.apache.spark.sql.execution.vectorized.{OffHeapColumnVector, OnHeapColumnVector, WritableColumnVector} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.sources._ import org.apache.spark.sql.types.{ByteType, LongType, MetadataBuilder, StringType, StructField, StructType} -import org.apache.spark.sql.vectorized.{ColumnarBatch, ColumnarBatchRow, ColumnVector} import org.apache.spark.util.SerializableConfiguration /** @@ -63,9 +61,9 @@ import org.apache.spark.util.SerializableConfiguration * @param optimizationsEnabled Whether to enable optimizations (file splitting, predicate pushdown) * @param tablePath Table path for deletion vector support; None disables DV processing * @param isCDCRead Whether this is a CDC (Change Data Capture) read - * @param useMetadataRowIndexOpt Controls row index source for DV filtering. When provided, - * must match optimizationsEnabled (true enables _metadata.row_index - * and file splitting; false uses internal counter, no splitting). + * @param useMetadataRowIndexOpt Controls row index source for DV filtering. When false, + * optimizationsEnabled must also be false because the internal + * counter does not support file splitting or predicate pushdown. * When None, reads from session config. */ abstract class DeltaParquetFileFormatBase( @@ -78,11 +76,11 @@ abstract class DeltaParquetFileFormatBase( protected val useMetadataRowIndexOpt: Option[Boolean] = None) extends ParquetFileFormat with Logging { - // Validate either we have all arguments for DV enabled read or none of them. + // An internal row-index counter requires unsplit files and no predicate pushdown. if (hasTablePath) { useMetadataRowIndexOpt.foreach { useMetadataRowIndex => - require(useMetadataRowIndex == optimizationsEnabled, - "Wrong arguments for Delta table scan with deletion vectors") + require(useMetadataRowIndex || !optimizationsEnabled, + "Cannot use internal row index with file splitting or predicate pushdown") } } @@ -238,7 +236,8 @@ abstract class DeltaParquetFileFormatBase( // the file and returns `RecordReaderIterator` (which implements `AutoCloseable` and // `Iterator`) instance as a `Iterator`. rowIteratorFromParquet match { - case resource: AutoCloseable => DeltaParquetFileFormat.closeQuietly(resource) + case resource: AutoCloseable => + RowIndexFilterMetadataColumnUtils.closeQuietly(resource) case _ => // do nothing } throw e @@ -363,109 +362,13 @@ abstract class DeltaParquetFileFormatBase( } } - // We only generate the row index column when predicate pushdown is not enabled. - val rowIndexColumnToWriteOpt = if (useMetadataRowIndex) None else rowIndexColumnOpt - val metadataColumnsToWrite = - Seq(isRowDeletedColumnOpt, rowIndexColumnToWriteOpt).filter(_.nonEmpty).map(_.get) - - // When metadata.row_index is not used there is no way to verify the Parquet index is - // starting from 0. We disable the splits, so the assumption is ParquetFileFormat respects - // that. - var rowIndex: Long = 0 - - // Used only when non-column row batches are received from the Parquet reader - val tempVector = new OnHeapColumnVector(1, ByteType) - - iterator.map { row => - row match { - case batch: ColumnarBatch => // When vectorized Parquet reader is enabled. - val size = batch.numRows() - // Create vectors for all needed metadata columns. - // We can't use the one from Parquet reader as it set the - // [[WritableColumnVector.isAllNulls]] to true and it can't be reset with using any - // public APIs. - DeltaParquetFileFormat.trySafely( - useOffHeapBuffers, size, metadataColumnsToWrite) { writableVectors => - val indexVectorTuples = new ArrayBuffer[(Int, ColumnVector)] - - // When predicate pushdown is enabled we use _metadata.row_index. Therefore, - // we only need to construct the isRowDeleted column. - var index = 0 - isRowDeletedColumnOpt.foreach { columnMetadata => - val isRowDeletedVector = writableVectors(index) - if (useMetadataRowIndex) { - rowIndexFilterOpt.get.materializeIntoVectorWithRowIndex( - size, batch.column(rowIndexColumnOpt.get.index), isRowDeletedVector) - } else { - rowIndexFilterOpt.get - .materializeIntoVector(rowIndex, rowIndex + size, isRowDeletedVector) - } - indexVectorTuples += (columnMetadata.index -> isRowDeletedVector) - index += 1 - } - - rowIndexColumnToWriteOpt.foreach { columnMetadata => - val rowIndexVector = writableVectors(index) - // populate the row index column value. - for (i <- 0 until size) { - rowIndexVector.putLong(i, rowIndex + i) - } - - indexVectorTuples += (columnMetadata.index -> rowIndexVector) - index += 1 - } - - val newBatch = DeltaParquetFileFormat.replaceVectors(batch, indexVectorTuples.toSeq: _*) - rowIndex += size - newBatch - } - - case columnarRow: ColumnarBatchRow => - // When vectorized reader is enabled but returns immutable rows instead of - // columnar batches [[ColumnarBatchRow]]. So we have to copy the row as a - // mutable [[InternalRow]] and set the `row_index` and `is_row_deleted` - // column values. This is not efficient. It should affect only the wide - // tables. https://github.com/delta-io/delta/issues/2246 - val newRow = columnarRow.copy(); - isRowDeletedColumnOpt.foreach { columnMetadata => - val rowIndexForFiltering = if (useMetadataRowIndex) { - columnarRow.getLong(rowIndexColumnOpt.get.index) - } else { - rowIndex - } - rowIndexFilterOpt.get.materializeSingleRowWithRowIndex(rowIndexForFiltering, tempVector) - newRow.setByte(columnMetadata.index, tempVector.getByte(0)) - } - - rowIndexColumnToWriteOpt - .foreach(columnMetadata => newRow.setLong(columnMetadata.index, rowIndex)) - rowIndex += 1 - - newRow - case rest: InternalRow => // When vectorized Parquet reader is disabled - // Temporary vector variable used to get DV values from RowIndexFilter - // Currently the RowIndexFilter only supports writing into a columnar vector - // and doesn't have methods to get DV value for a specific row index. - // TODO: This is not efficient, but it is ok given the default reader is vectorized - isRowDeletedColumnOpt.foreach { columnMetadata => - val rowIndexForFiltering = if (useMetadataRowIndex) { - rest.getLong(rowIndexColumnOpt.get.index) - } else { - rowIndex - } - rowIndexFilterOpt.get.materializeSingleRowWithRowIndex(rowIndexForFiltering, tempVector) - rest.setByte(columnMetadata.index, tempVector.getByte(0)) - } - - rowIndexColumnToWriteOpt - .foreach(columnMetadata => rest.setLong(columnMetadata.index, rowIndex)) - rowIndex += 1 - rest - case others => - throw new RuntimeException( - s"Parquet reader returned an unknown row type: ${others.getClass.getName}") - } - } + RowIndexFilterMetadataColumnUtils.iteratorWithAdditionalMetadataColumns( + iterator, + rowIndexFilterOpt, + isRowDeletedColumnOpt, + rowIndexColumnOpt, + useOffHeapBuffers, + useMetadataRowIndex) } /** @@ -552,7 +455,7 @@ case class DeltaParquetFileFormat( optimizationsEnabled = optimizationsEnabled, tablePath = tablePath, isCDCRead = isCDCRead, - // V1: capture config at construction, used in buildReaderWithPartitionValues + // V1: capture config at construction, used in buildReaderWithPartitionValues. useMetadataRowIndexOpt = SparkSession.getActiveSession.map( _.sessionState.conf.getConf(DeltaSQLConf.DELETION_VECTORS_USE_METADATA_ROW_INDEX))) { @@ -603,70 +506,4 @@ object DeltaParquetFileFormat { * [[PartitionedFile]]'s otherConstantMetadataColumnValues map. */ val FILE_ROW_INDEX_FILTER_TYPE = "row_index_filter_type" - /** Utility method to create a new writable vector */ - private[delta] def newVector( - useOffHeapBuffers: Boolean, size: Int, dataType: StructField): WritableColumnVector = { - if (useOffHeapBuffers) { - OffHeapColumnVector.allocateColumns(size, Seq(dataType).toArray)(0) - } else { - OnHeapColumnVector.allocateColumns(size, Seq(dataType).toArray)(0) - } - } - - /** Try the operation, if the operation fails release the created resource */ - private[delta] def trySafely[R <: WritableColumnVector, T]( - useOffHeapBuffers: Boolean, - size: Int, - columns: Seq[ColumnMetadata])(f: Seq[WritableColumnVector] => T): T = { - val resources = new ArrayBuffer[WritableColumnVector](columns.size) - try { - columns.foreach(col => resources.append(newVector(useOffHeapBuffers, size, col.structField))) - f(resources.toSeq) - } catch { - case NonFatal(e) => - resources.foreach(closeQuietly(_)) - throw e - } - } - - /** Utility method to quietly close an [[AutoCloseable]] */ - private[delta] def closeQuietly(closeable: AutoCloseable): Unit = { - if (closeable != null) { - try { - closeable.close() - } catch { - case NonFatal(_) => // ignore - } - } - } - - /** - * Helper method to replace the vectors in given [[ColumnarBatch]]. - * New vectors and its index in the batch are given as tuples. - */ - private[delta] def replaceVectors( - batch: ColumnarBatch, - indexVectorTuples: (Int, ColumnVector) *): ColumnarBatch = { - val vectors = ArrayBuffer[ColumnVector]() - for (i <- 0 until batch.numCols()) { - var replaced: Boolean = false - for (indexVectorTuple <- indexVectorTuples) { - val index = indexVectorTuple._1 - val vector = indexVectorTuple._2 - if (indexVectorTuple._1 == i) { - vectors += indexVectorTuple._2 - // Make sure to close the existing vector allocated in the Parquet - batch.column(i).close() - replaced = true - } - } - if (!replaced) { - vectors += batch.column(i) - } - } - new ColumnarBatch(vectors.toArray, batch.numRows()) - } - - /** Helper class to encapsulate column info */ - case class ColumnMetadata(index: Int, structField: StructField) } diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaThrowableHelper.scala b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaThrowableHelper.scala index 1ac4ab167c3..45615adc972 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaThrowableHelper.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaThrowableHelper.scala @@ -79,11 +79,24 @@ object DeltaThrowableHelper */ def getMessage(errorClass: String, messageParameters: Array[String]): String = { validateParameterValues(errorClass, errorSubClass = null, messageParameters) - val template = errorClassReader.getMessageTemplate(errorClass) + val template = getMessageTemplate(errorClass) val message = formatMessage(errorClass, messageParameters, template) s"[$errorClass] $message" } + /** + * This is a temporary workaround for SPARK-58999. It bypasses a limitation in Spark, and allows + * us to raise an error that uses only the main error class, even if the given error class has + * available subclasses. + */ + private[delta] def getMessageTemplate(errorClass: String): String = { + if (errorClass.contains(".")) { + errorClassReader.getMessageTemplate(errorClass) + } else { + getMainMessageTemplate(errorClass) + } + } + private def formatMessage( errorClass: String, messageParameters: Array[String], @@ -174,7 +187,7 @@ object DeltaThrowableHelper } else { errorClass + "." + errorSubClass } - val parameterizedMessage = errorClassReader.getMessageTemplate(wholeErrorClass) + val parameterizedMessage = getMessageTemplate(wholeErrorClass) parsePrameterNamesFromParameterizedMessage(parameterizedMessage) } diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaTimeTravelSpec.scala b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaTimeTravelSpec.scala index 4c4b0317f18..4aee147b296 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaTimeTravelSpec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaTimeTravelSpec.scala @@ -83,7 +83,7 @@ case class DeltaTimeTravelSpec( } } -object DeltaTimeTravelSpec { +object DeltaTimeTravelSpec extends DeltaLogging { /** A regex which looks for the pattern ...@v(some numbers) for extracting the version number */ private val VERSION_URI_FOR_TIME_TRAVEL = ".*@[vV](\\d+)$".r @@ -116,11 +116,14 @@ object DeltaTimeTravelSpec { val timestamp = parseTimestamp(ts, conf.sessionLocalTimeZone) // Drop the 18 characters in the right, which is the timestamp format and the @ character. val realIdentifier = identifier.dropRight(TIMESTAMP_FORMAT_LENGTH + 1) - + recordDeltaEvent(null, "delta.timeTravel.atSyntaxUsage", + data = Map("source" -> "atSyntax.path", "atTimestamp" -> ts)) DeltaTimeTravelSpec(Some(timestamp), None, Some("atSyntax.path")) -> realIdentifier case VERSION_URI_FOR_TIME_TRAVEL(v) => // Drop the version, and `@v` characters from the identifier val realIdentifier = identifier.dropRight(v.length + 2) + recordDeltaEvent(null, "delta.timeTravel.atSyntaxUsage", + data = Map("source" -> "atSyntax.path", "atVersion" -> v)) DeltaTimeTravelSpec(None, Some(v.toLong), Some("atSyntax.path")) -> realIdentifier } } diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/GeneratedColumn.scala b/spark/src/main/scala/org/apache/spark/sql/delta/GeneratedColumn.scala index cc7d77c746c..705e7d06e57 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/GeneratedColumn.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/GeneratedColumn.scala @@ -19,17 +19,22 @@ package org.apache.spark.sql.delta // scalastyle:off import.ordering.noEmptyLine import java.util.Locale +import scala.util.control.NonFatal + import org.apache.spark.sql.delta.DataFrameUtils import org.apache.spark.sql.delta.ClassicColumnConversions._ import org.apache.spark.sql.delta.actions.{Metadata, Protocol} import org.apache.spark.sql.delta.v2.interop.AbstractProtocol import org.apache.spark.sql.delta.files.{TahoeBatchFileIndex, TahoeFileIndex} +import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.schema.SchemaUtils.quoteIdentifier import org.apache.spark.sql.delta.sources.DeltaSourceUtils.GENERATION_EXPRESSION_METADATA_KEY import org.apache.spark.sql.delta.sources.DeltaSQLConf +import org.apache.spark.sql.delta.sources.DeltaSQLConf.GeneratedColumnPartitionFilterInferenceMode import org.apache.spark.sql.delta.util.AnalysisHelper +import org.apache.spark.internal.MDC import org.apache.spark.sql.{AnalysisException, Column, Dataset, SparkSession} import org.apache.spark.sql.catalyst.analysis.Analyzer import org.apache.spark.sql.catalyst.expressions._ @@ -38,12 +43,13 @@ import org.apache.spark.sql.catalyst.expressions.objects.StaticInvoke import org.apache.spark.sql.catalyst.optimizer.CollapseProject import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan, Project} import org.apache.spark.sql.catalyst.types.DataTypeUtils.toAttributes -import org.apache.spark.sql.catalyst.util.{quoteIfNeeded, CaseInsensitiveMap, CharVarcharCodegenUtils} +import org.apache.spark.sql.catalyst.util.{quoteIfNeeded, CaseInsensitiveMap, CharVarcharCodegenUtils, DateTimeUtils} import org.apache.spark.sql.execution.SQLExecution import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelation} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ import org.apache.spark.sql.types.{Metadata => FieldMetadata} +import org.apache.spark.unsafe.types.UTF8String /** * Provide utility methods to implement Generated Columns for Delta. Users can use the following * SQL syntax to create a table with generated columns. @@ -502,6 +508,136 @@ object GeneratedColumn extends DeltaLogging with AnalysisHelper { } } + private case class PartitionFilterCandidate( + expression: Expression, + descriptor: OptimizablePartitionExpression, + sourceDataType: DataType, + sourcePredicate: Expression) { + + val sourceLiteral: Option[Literal] = sourcePredicate match { + case comparison: BinaryComparison => comparison.right match { + case literal: Literal => Some(literal) + case _ => None + } + case _ => None + } + + val operatorName: String = sourcePredicate.getClass.getSimpleName + } + + private case class PartitionFilterCandidateSummary( + descriptor: String, + sourceDataType: String, + partitionDataTypes: Seq[String], + literalDataType: Option[String], + operator: String, + hasKnownRisk: Boolean, + count: Int = 1) + + private case class PartitionFilterInferenceEvent( + candidateSignatures: Seq[PartitionFilterCandidateSummary]) + + /** Returns the partition-column data types referenced by a candidate expression. */ + private def partitionDataTypes(candidate: PartitionFilterCandidate): Seq[String] = { + candidate.expression.references.toSeq + .map(_.dataType.catalogString) + .distinct + .sorted + } + + /** Returns whether a literal is compatible with string-to-date conversion. */ + private def stringLiteralCanCastToDate(literal: Literal): Boolean = { + literal.dataType match { + case _: StringType => + DateTimeUtils.stringToDate(literal.value.asInstanceOf[UTF8String]).isDefined + case _ => true + } + } + + /** + * Returns whether a candidate inference is unsafe. + * + * String `TRUNC` inference is unsafe when its generated date predicate does not preserve the + * source predicate. Date truncation changes range ordering, while an unparseable equality literal + * may fail under ANSI evaluation even though the original comparison is between strings. + */ + private def isUnsafeInference(candidate: PartitionFilterCandidate): Boolean = { + // Match the source predicates used by `OptimizablePartitionExpression` to generate candidates. + val isStringTrunc = candidate.descriptor.isInstanceOf[TruncDatePartitionExpr] && + candidate.sourceDataType == StringType + isStringTrunc && (candidate.sourcePredicate match { + case _: LessThan | _: LessThanOrEqual | _: GreaterThan | _: GreaterThanOrEqual => true + case _: EqualTo => + // Date parsing accepts single-digit month and day components, so values such as + // `2024-4-1` normalize to `2024-04-01` and remain safe. + candidate.sourceLiteral.exists(value => !stringLiteralCanCastToDate(value)) + case _ => false + }) + } + + /** Returns whether the candidate matches a known inference risk. */ + private def hasKnownInferenceRisk(candidate: PartitionFilterCandidate): Boolean = { + isUnsafeInference(candidate) + } + + /** Builds the telemetry signature for an inferred predicate. */ + private def candidateSignature( + candidate: PartitionFilterCandidate): PartitionFilterCandidateSummary = { + val partitionTypes = partitionDataTypes(candidate) + PartitionFilterCandidateSummary( + descriptor = candidate.descriptor.getClass.getSimpleName, + sourceDataType = candidate.sourceDataType.catalogString, + partitionDataTypes = partitionTypes, + literalDataType = candidate.sourceLiteral.map(_.dataType.catalogString), + operator = candidate.operatorName, + hasKnownRisk = hasKnownInferenceRisk(candidate)) + } + + /** Records predicate signatures and their occurrence counts in a Delta event. */ + private def recordPartitionFilterInferenceCandidates( + snapshot: SnapshotDescriptor, + candidates: Seq[PartitionFilterCandidate]): Unit = { + val signatureCounts = candidates.map(candidateSignature).groupBy(identity).toSeq + .map { case (signature, matching) => + signature.copy(count = matching.size) + } + .sortBy(_.toString) + recordDeltaEvent( + snapshot, + "delta.generatedColumns.partitionFilterInference", + data = PartitionFilterInferenceEvent(signatureCounts)) + } + + /** + * Applies the configured rollout mode to candidate inferences. + * + * LOG_ONLY / ASSERT modes record telemetry, and ASSERT also filters candidates classified by + * `isUnsafeInference`. + */ + private def applyPartitionFilterInferenceRollout( + spark: SparkSession, + snapshot: SnapshotDescriptor, + candidates: Seq[PartitionFilterCandidate]): Seq[PartitionFilterCandidate] = { + if (candidates.isEmpty) return candidates + + val conf = spark.sessionState.conf + val mode = GeneratedColumnPartitionFilterInferenceMode.fromConf(conf) + if (mode == GeneratedColumnPartitionFilterInferenceMode.OFF) { + return candidates + } + + val suppressUnsafeInference = mode == GeneratedColumnPartitionFilterInferenceMode.ASSERT + try { + recordPartitionFilterInferenceCandidates(snapshot, candidates) + if (suppressUnsafeInference) candidates.filterNot(isUnsafeInference) else candidates + } catch { + case NonFatal(e) => + logWarning(log"Failed to apply generated-column partition-filter inference rollout: " + + log"${MDC(DeltaLogKeys.EXCEPTION, e)}") + candidates + } + } + def partitionFilterOptimizationEnabled(spark: SparkSession): Boolean = { spark.sessionState.conf .getConf(DeltaSQLConf.GENERATED_COLUMN_PARTITION_FILTER_OPTIMIZATION_ENABLED) @@ -557,37 +693,63 @@ object GeneratedColumn extends DeltaLogging with AnalysisHelper { */ def toPartitionFilter( nameParts: Seq[String], - func: (OptimizablePartitionExpression) => Option[Expression]): Seq[Expression] = { + sourceDataType: DataType, + sourcePredicate: Expression, + func: (OptimizablePartitionExpression) => Option[Expression]) + : Seq[PartitionFilterCandidate] = { optimizablePartitionExpressions.get(createFieldPath(nameParts)).toSeq.flatMap { exprs => - exprs.flatMap(expr => func(expr)) + exprs.flatMap { descriptor => + func(descriptor).map { expression => + PartitionFilterCandidate( + expression, + descriptor, + sourceDataType, + sourcePredicate) + } + } } } - val partitionFilters = dataFilters.flatMap { filter => + val partitionFilterCandidates = dataFilters.flatMap { filter => preprocess(filter) match { - case LessThan(ExtractBaseColumn(nameParts, _), lit: Literal) => - toPartitionFilter(nameParts, _.lessThan(lit)) - case LessThanOrEqual(ExtractBaseColumn(nameParts, _), lit: Literal) => - toPartitionFilter(nameParts, _.lessThanOrEqual(lit)) - case EqualTo(ExtractBaseColumn(nameParts, _), lit: Literal) => - toPartitionFilter(nameParts, _.equalTo(lit)) - case GreaterThan(ExtractBaseColumn(nameParts, _), lit: Literal) => - toPartitionFilter(nameParts, _.greaterThan(lit)) - case GreaterThanOrEqual(ExtractBaseColumn(nameParts, _), lit: Literal) => - toPartitionFilter(nameParts, _.greaterThanOrEqual(lit)) - case IsNull(ExtractBaseColumn(nameParts, _)) => - toPartitionFilter(nameParts, _.isNull()) + case predicate @ LessThan(ExtractBaseColumn(nameParts, sourceType), lit: Literal) => + toPartitionFilter(nameParts, sourceType, predicate, + _.lessThan(lit)) + case predicate @ LessThanOrEqual( + ExtractBaseColumn(nameParts, sourceType), lit: Literal) => + toPartitionFilter(nameParts, sourceType, predicate, + _.lessThanOrEqual(lit)) + case predicate @ EqualTo(ExtractBaseColumn(nameParts, sourceType), lit: Literal) => + toPartitionFilter(nameParts, sourceType, predicate, + _.equalTo(lit)) + case predicate @ GreaterThan(ExtractBaseColumn(nameParts, sourceType), lit: Literal) => + toPartitionFilter(nameParts, sourceType, predicate, + _.greaterThan(lit)) + case predicate @ GreaterThanOrEqual( + ExtractBaseColumn(nameParts, sourceType), lit: Literal) => + toPartitionFilter(nameParts, sourceType, predicate, + _.greaterThanOrEqual(lit)) + case predicate @ IsNull(ExtractBaseColumn(nameParts, sourceType)) => + toPartitionFilter(nameParts, sourceType, predicate, _.isNull()) case _ => Nil } } - val resolvedPartitionFilters = resolveReferencesForExpressions(spark, partitionFilters, delta) + val resolvedExpressions = resolveReferencesForExpressions( + spark, partitionFilterCandidates.map(_.expression), delta) + val resolvedCandidates = partitionFilterCandidates.zip(resolvedExpressions) + .map { case (candidate, resolvedExpression) => + candidate.copy(expression = resolvedExpression) + } + val retainedCandidates = applyPartitionFilterInferenceRollout( + spark, snapshot, resolvedCandidates) + val resolvedPartitionFilters = retainedCandidates.map(_.expression) if (log.isDebugEnabled) { logDebug("User provided data filters:") dataFilters.foreach(f => logDebug(f.sql)) logDebug("Auto generated partition filters:") - partitionFilters.foreach(f => logDebug(f.sql)) + partitionFilterCandidates.foreach(candidate => logDebug(candidate.expression.sql)) logDebug("Resolved generated partition filters:") resolvedPartitionFilters.foreach(f => logDebug(f.sql)) } diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/IcebergCompat.scala b/spark/src/main/scala/org/apache/spark/sql/delta/IcebergCompat.scala index b171f3f31ae..adaca1ba7b7 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/IcebergCompat.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/IcebergCompat.scala @@ -637,7 +637,9 @@ object CheckTypeInV2AllowList extends CheckTypeInAllowList { object CheckPartitionDataTypeInV2AllowList extends IcebergCompatCheck { private val allowedTypes = Set[Class[_]] ( ByteType.getClass, ShortType.getClass, IntegerType.getClass, LongType.getClass, - FloatType.getClass, DoubleType.getClass, DecimalType.getClass, + // DecimalType is parameterized by precision and scale, so schema fields contain + // DecimalType instances rather than the companion object. + FloatType.getClass, DoubleType.getClass, classOf[DecimalType], StringType.getClass, BinaryType.getClass, BooleanType.getClass, TimestampType.getClass, TimestampNTZType.getClass, DateType.getClass diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/LastCheckpointInfo.scala b/spark/src/main/scala/org/apache/spark/sql/delta/LastCheckpointInfo.scala index 3d23153af36..486588b258f 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/LastCheckpointInfo.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/LastCheckpointInfo.scala @@ -18,12 +18,14 @@ package org.apache.spark.sql.delta import scala.collection.mutable.ArrayBuffer -import org.apache.spark.sql.delta.actions.{CheckpointMetadata, SidecarFile, SingleAction} +import org.apache.spark.sql.delta.actions.{Checkpoint, CheckpointMetadata, SidecarFile, SingleAction} +import org.apache.spark.sql.delta.amt.DataManifestEntry import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util.FileNames.{checkpointVersion, numCheckpointParts} import org.apache.spark.sql.delta.util.JsonUtils -import com.fasterxml.jackson.annotation.{JsonIgnore, JsonIgnoreProperties, JsonPropertyOrder} -import com.fasterxml.jackson.databind.{DeserializationFeature, JsonNode} +import com.fasterxml.jackson.annotation.{JsonIgnore, JsonPropertyOrder, JsonValue} +import com.fasterxml.jackson.core.JsonParser +import com.fasterxml.jackson.databind.{DeserializationContext, DeserializationFeature, JsonDeserializer, JsonNode} import com.fasterxml.jackson.databind.annotation.JsonDeserialize import com.fasterxml.jackson.databind.node.ObjectNode import org.apache.commons.codec.digest.DigestUtils @@ -32,6 +34,12 @@ import org.apache.hadoop.fs.FileStatus import org.apache.spark.sql.SparkSession import org.apache.spark.sql.types.StructType +/** Information about the AMT checkpoint in the LAST_CHECKPOINT file. */ +case class LastAMTCheckpoint( + manifestCommitVersion: Long, + checkpoint: Option[Checkpoint], + leaves: Option[Seq[DataManifestEntry]]) + /** * Information about the V2 Checkpoint in the LAST_CHECKPOINT file * @param path file name corresponding to the uuid-named v2 checkpoint @@ -89,13 +97,16 @@ object LastCheckpointV2 { * from one read request. For this reason, we use `JsonPropertyOrder` to force them in the beginning * together. * - * @param version the version of this checkpoint - * @param size the number of actions in the checkpoint, -1 if the information is unavailable. - * @param parts the number of parts when the checkpoint has multiple parts. None if this is a - * singular checkpoint + * @param version the version of this checkpoint. Represents the content root version if this is an + * AMT checkpoint. + * @param size the number of actions in the checkpoint. -1 if this is an AMT checkpoint or the + * information is unavailable. + * @param parts the number of parts when the checkpoint has multiple parts. Represents the number of + * leaves if this is an AMT checkpoint. None if this is a singular checkpoint. * @param sizeInBytes the number of bytes of the checkpoint - * @param numOfAddFiles the number of AddFile actions in the checkpoint + * @param numOfAddFiles the number of AddFile actions in the checkpoint. None for AMT checkpoints. * @param checkpointSchema the schema of the underlying checkpoint files + * @param amtCheckpoint information about the AMT checkpoint, present only for AMT tables * @param checksum the checksum of the [[LastCheckpointInfo]]. */ @JsonPropertyOrder(Array("version", "size", "parts")) @@ -108,11 +119,20 @@ case class LastCheckpointInfo( @JsonDeserialize(contentAs = classOf[java.lang.Long]) numOfAddFiles: Option[Long], checkpointSchema: Option[StructType], + @JsonDeserialize(contentUsing = classOf[LastCheckpointInfo.CheckpointTypeDeserializer]) + checkpointType: Option[LastCheckpointInfo.CheckpointType] = None, + amtCheckpoint: Option[LastAMTCheckpoint] = None, v2Checkpoint: Option[LastCheckpointV2] = None, checksum: Option[String] = None) { + // Format-specific fields are mutually exclusive. + require(!(amtCheckpoint.nonEmpty && v2Checkpoint.nonEmpty)) + require(Seq(LastCheckpointInfo.CheckpointType.AMT, LastCheckpointInfo.CheckpointType.UNKNOWN) + .exists(checkpointType.contains) == amtCheckpoint.nonEmpty) + @JsonIgnore def getFormatEnum(): CheckpointInstance.Format = parts match { + case _ if amtCheckpoint.nonEmpty => CheckpointInstance.Format.AMT case _ if v2Checkpoint.nonEmpty => CheckpointInstance.Format.V2 case Some(_) => CheckpointInstance.Format.WITH_PARTS case None => CheckpointInstance.Format.SINGLE @@ -125,6 +145,30 @@ case class LastCheckpointInfo( } object LastCheckpointInfo { + /** The checkpoint type described by a [[LastCheckpointInfo]]. */ + sealed abstract class CheckpointType(val name: String) { + @JsonValue + override def toString: String = name + } + + /** Reads a [[CheckpointType]] back from its bare `name` string. */ + class CheckpointTypeDeserializer extends JsonDeserializer[CheckpointType] { + override def deserialize(p: JsonParser, ctxt: DeserializationContext): CheckpointType = + CheckpointType.fromName(p.readValueAs(classOf[String])) + } + + object CheckpointType { + object AMT extends CheckpointType("AdaptiveMetadataTree") + object UNKNOWN extends CheckpointType("Unknown") + + /** All known checkpoint types. Extend this when adding a new [[CheckpointType]]. */ + val ALL: Seq[CheckpointType] = Seq(AMT) + + def unapply(name: String): Option[CheckpointType] = ALL.find(_.name == name) + + /** Used for deserialization. */ + def fromName(name: String): CheckpointType = unapply(name).getOrElse(UNKNOWN) + } val STORED_CHECKSUM_KEY = "checksum" @@ -150,6 +194,8 @@ object LastCheckpointInfo { lastCheckpointInfo.parts, sizeInBytes = None, numOfAddFiles = None, + checkpointType = None, + amtCheckpoint = None, v2Checkpoint = None, checkpointSchema = None)) } diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/MetadataCleanup.scala b/spark/src/main/scala/org/apache/spark/sql/delta/MetadataCleanup.scala index 6c608073030..94d8d82da7d 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/MetadataCleanup.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/MetadataCleanup.scala @@ -67,6 +67,14 @@ trait MetadataCleanup extends DeltaLogging { snapshotToCleanup: Snapshot, catalogTableOpt: Option[CatalogTable]): Unit = { if (enableExpiredLogCleanup(unsafeVolatileSnapshot.metadata)) { + try { + DeltaErrors.checkCatalogManagedTableOperationAllowed( + CatalogManagedTableMaintenanceOperation.METADATA_CLEANUP, + snapshotToCleanup, + catalogTableOpt) + } catch { + case _: DeltaUnsupportedOperationException => return + } cleanUpExpiredLogs(snapshotToCleanup, catalogTableOpt) } } @@ -77,6 +85,11 @@ trait MetadataCleanup extends DeltaLogging { catalogTableOpt: Option[CatalogTable] = None, deltaRetentionMillisOpt: Option[Long] = None, cutoffTruncationGranularity: TruncationGranularity = DAY): Unit = { + DeltaErrors.checkCatalogManagedTableOperationAllowed( + CatalogManagedTableMaintenanceOperation.METADATA_CLEANUP, + snapshotToCleanup, + catalogTableOpt) + recordDeltaOperation(this, "delta.log.cleanup") { val retentionMillis = deltaRetentionMillisOpt.getOrElse(deltaRetentionMillis(unsafeVolatileSnapshot.metadata)) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/OptimisticTransaction.scala b/spark/src/main/scala/org/apache/spark/sql/delta/OptimisticTransaction.scala index 90959261ab0..8138f74bc2a 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/OptimisticTransaction.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/OptimisticTransaction.scala @@ -37,7 +37,7 @@ import org.apache.spark.sql.delta.DeltaGeoSpatial import org.apache.spark.sql.delta.DeltaOperations.{ChangeColumn, ChangeColumns, CreateTable, Operation, ReplaceColumns, ReplaceTable, UpdateSchema} import org.apache.spark.sql.delta.RowId.RowTrackingMetadataDomain import org.apache.spark.sql.delta.actions._ -import org.apache.spark.sql.delta.amt.{AMTCheckpointProvider, AMTWriteMetrics, AMTWriteResult, AMTWriterManager} +import org.apache.spark.sql.delta.amt.{AMTCheckpointProvider, AMTCommitStats, AMTUtils, AMTWriteResult, AMTWriterManager} import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.commands.DeletionVectorUtils import org.apache.spark.sql.delta.commands.cdc.CDCReader @@ -91,6 +91,37 @@ case class CoordinatedCommitsStats( commitCoordinatorName: String, commitCoordinatorConf: Map[String, String]) +/** + * Metrics gathered while preparing a commit attempt, carried through to the + * final [[CommitStats]]. + */ +case class CommitPrepMetrics( + icebergMetadataGenerationDurationMsOpt: Option[Long] = None) + +/** + * The prepared state of one commit attempt, ready for the write step. Produced by + * [[OptimisticTransaction.prepareCommit]] on the first attempt and by + * [[OptimisticTransaction.rebaseCurrentTransactionInfo]] on a conflict retry -- the only two places + * that edit the [[CurrentTransactionInfo]] once a commit attempt begins. + * + * @param commitVersion the version this attempt targets (the rebased version on a retry) + * @param currentTransactionInfo the fully prepared txn info (Uniform + AMT checkpoint ref applied) + * @param currentTransactionInfoBeforePreparedResult the txn info before this attempt's preparation + * (conflict-resolved, but without the Uniform/AMT edits). The retry loop feeds this -- not + * the prepared `currentTransactionInfo` -- into the next `checkForConflicts`, so prepared + * state never leaks into a later conflict check. + * @param actionsToWriteInLogFile the final actions to persist, incl. the inline AMT checkpoint + * @param amtWriteResultForLastCheckpointOpt the AMT write result, for the _last_checkpoint write + * @param prepMetrics metrics gathered during preparation + */ +case class PrepareCommitResult( + commitVersion: Long, + currentTransactionInfo: CurrentTransactionInfo, + currentTransactionInfoBeforePreparedResult: CurrentTransactionInfo, + actionsToWriteInLogFile: Seq[Action], + amtWriteResultForLastCheckpointOpt: Option[AMTWriteResult], + prepMetrics: CommitPrepMetrics) + /** Record metrics about a successful commit. */ case class CommitStats( /** The version read by the txn when it starts. */ @@ -138,8 +169,8 @@ case class CommitStats( isIdempotentRetry: Boolean = false, numOfDomainMetadatas: Long = 0, txnId: Option[String] = None, - /** Metrics for the inline AMT (Adaptive Metadata Tree) write, if this commit emitted one. */ - amtWriteMetrics: Option[AMTWriteMetrics] = None + /** Metrics for the AMT (Adaptive Metadata Tree) write used by this commit, if any. */ + amtCommitStats: Option[AMTCommitStats] = None ) /** @@ -655,7 +686,7 @@ trait OptimisticTransactionImpl extends TransactionHelper /** * Do the actual checks and works to update the metadata and save it into the `newMetadata` - * field, which will be added to the actions to commit in [[prepareCommit]]. + * field, which will be added to the actions to commit in [[prepareInitialActions]]. */ protected def updateMetadataInternal( proposedNewMetadata: Metadata, @@ -1093,7 +1124,7 @@ trait OptimisticTransactionImpl extends TransactionHelper */ private def enableAdaptiveMetadataDependentFeatures(metadata: Metadata): Metadata = { val adaptiveMetadataEnabled = - protocol.isFeatureSupported(AdaptiveMetadataTableFeature) || + AMTUtils.amtEnabled(metadata, protocol) || TableFeatureProtocolUtils.isFeatureSupportedInTableConfigs( metadata.configuration, AdaptiveMetadataTableFeature) if (!adaptiveMetadataEnabled) { @@ -1101,7 +1132,7 @@ trait OptimisticTransactionImpl extends TransactionHelper } val existingTableHasAdaptiveMetadata = - snapshot.protocol.isFeatureSupported(AdaptiveMetadataTableFeature) + AMTUtils.amtEnabled(snapshot) if (!isCreatingNewTable && !existingTableHasAdaptiveMetadata) { throw DeltaErrors.adaptiveMetadataUpgradeNotSupported(AdaptiveMetadataTableFeature.name) } @@ -1580,9 +1611,16 @@ trait OptimisticTransactionImpl extends TransactionHelper } } - /** Ensure that actions do not contain duplicates for the same path. */ + /** + * Ensure that actions do not contain duplicates for the same path. + * This check is performed using AMT-aware object identity mode. + */ protected def checkNoDuplicateActions(actions: Seq[Action]): Unit = { - ConflictChecker.checkNoDuplicateActions(spark, actions.iterator).foreach(_ => ()) + val useDVObjectIdentity = + FileAction.useDeletionVectorObjectIdentity(metadata, protocol, spark) + ConflictChecker.checkNoDuplicateActions( + spark, actions.iterator, dataPath, useDVObjectIdentity) + .foreach(_ => ()) } /** @@ -1854,22 +1892,19 @@ trait OptimisticTransactionImpl extends TransactionHelper val identityOnlyMetadataUpdate = isIdentityOnlyMetadataUpdate() // Update schema for IDENTITY column writes if necessary. This has to be called before - // `prepareCommit` because it might change metadata and `prepareCommit` is responsible for - // converting updated metadata into a `Metadata` action. + // `prepareInitialActions` because it might change metadata and `prepareInitialActions` is + // responsible for converting updated metadata into a `Metadata` action. precommitUpdateSchemaWithIdentityHighWaterMarks() // Try to commit at the next version. var preparedActions = executionObserver.preparingCommit { - prepareCommit(finalActions, op) + prepareInitialActions(finalActions, op) } validateActionsAddFileInvariants(preparedActions, metadata) checkNoDuplicateActions(preparedActions) - ConflictChecker.trackConsistentDataChange( - spark, preparedActions.iterator, deltaLog, op, callerContext = "commit") - .foreach(_ => ()) // Find the isolation level to use for this commit val isolationLevelToUse = getIsolationLevelToUse(preparedActions, op) @@ -1906,10 +1941,14 @@ trait OptimisticTransactionImpl extends TransactionHelper readVersion = Some(readVersion).filter(_ >= 0), isolationLevel = Option(isolationLevelToUse.toString), isBlindAppend = Some(isBlindAppend), + dataChange = Some(CommitInfo.dataChangeFromActions(preparedActions)), operationMetrics = getOperationMetrics(op), userMetadata = getUserMetadata(op), tags = if (allTags.nonEmpty) Some(allTags) else None, - txnId = Some(txnId)) + txnId = Some(txnId), + // We initialize lastManifestCommit by carrying forward the previous one. If this commit + // writes a new AMT checkpoint, the commitInfo will be updated afterwards. + lastManifestCommit = snapshot.lastManifestCommitOpt) val firstAttemptVersion = getFirstAttemptVersion val metadataUpdatedWithCoordinatedCommitsInfo = updateMetadataWithCoordinatedCommitsConfs() @@ -1920,10 +1959,7 @@ trait OptimisticTransactionImpl extends TransactionHelper case other => other } } - val preCommitLatestAMTCheckpointOpt = snapshot.checkpointProvider match { - case amt: AMTCheckpointProvider => Some(amt.checkpointAction) - case _ => None - } + val preCommitLatestAMTCheckpointOpt = amtCheckpointProviderOpt.map(_.checkpointAction) val currentTransactionInfo = CurrentTransactionInfo( txnId = txnId, readPredicates = readPredicates.asScala.toVector, @@ -2080,7 +2116,8 @@ trait OptimisticTransactionImpl extends TransactionHelper newProtocolOpt: Option[Protocol], op: DeltaOperations.Operation, context: Map[String, String], - metrics: Map[String, String] + metrics: Map[String, String], + dataChange: Option[Boolean] ): (Long, Snapshot) = recordDeltaOperation(deltaLog, "delta.commit.large") { assert(committed.isEmpty, "Transaction already committed.") commitStartNano = System.nanoTime() @@ -2125,10 +2162,14 @@ trait OptimisticTransactionImpl extends TransactionHelper readVersion = Some(readVersion), isolationLevel = Some(Serializable.toString), isBlindAppend = Some(false), - Some(metrics), + dataChange = dataChange, + operationMetrics = Some(metrics), userMetadata = getUserMetadata(op), tags = if (tags.nonEmpty) Some(tags) else None, - txnId = Some(txnId)) + txnId = Some(txnId), + // `commitLarge` never emits an inline AMT checkpoint, so it only carries the read + // snapshot's lastManifestCommitOpt forward. + lastManifestCommit = snapshot.lastManifestCommitOpt) val assertDeletionVectorWellFormed = getAssertDeletionVectorWellFormedFunc(spark, op) updateMetadataWithCoordinatedCommitsConfs() @@ -2152,20 +2193,18 @@ trait OptimisticTransactionImpl extends TransactionHelper toProtocol = p, isCreatingNewTable) DeltaTableV2.withEnrichedUnsupportedTableException(catalogTable) { - deltaLog.protocolWrite(p) + validateProtocolWrite(p) } case _ => } action } - allActions = ConflictChecker.trackConsistentDataChange( - spark, allActions, deltaLog, op, callerContext = "commitLarge") val (allActions2, acStatsCollector) = collectAutoOptimizeStats(allActions) allActions = allActions2 // Validate protocol support, specifically writer features. DeltaTableV2.withEnrichedUnsupportedTableException(catalogTable) { - deltaLog.protocolWrite(snapshot.protocol) + validateProtocolWrite(snapshot.protocol) } allActions = RowId.assignFreshRowIds(spark, protocol, snapshot, allActions, op) @@ -2190,6 +2229,8 @@ trait OptimisticTransactionImpl extends TransactionHelper val commitStatsComputer = new CommitStatsComputer() allActions = commitStatsComputer.addToCommitStats(allActions) + allActions = ConflictChecker.trackDataChange( + spark, allActions, deltaLog, op, callerContext = "commitLarge") executionObserver.beginDoCommit() if (readVersion < 0) { deltaLog.createLogDirectoriesIfNotExists() @@ -2488,7 +2529,7 @@ trait OptimisticTransactionImpl extends TransactionHelper * Prepare for a commit by doing all necessary pre-commit checks and modifications to the actions. * @return The finalized set of actions. */ - protected def prepareCommit( + protected def prepareInitialActions( actions: Seq[Action], op: DeltaOperations.Operation): Seq[Action] = { @@ -2542,7 +2583,7 @@ trait OptimisticTransactionImpl extends TransactionHelper isCreatingNewTable, operationNameOpt = Some(op.name)) DeltaTableV2.withEnrichedUnsupportedTableException(catalogTable) { - deltaLog.protocolWrite(p) + validateProtocolWrite(p) } } @@ -2641,8 +2682,8 @@ trait OptimisticTransactionImpl extends TransactionHelper } DeltaTableV2.withEnrichedUnsupportedTableException(catalogTable) { - newProtocol.foreach(deltaLog.protocolWrite) - deltaLog.protocolWrite(snapshot.protocol) + newProtocol.foreach(validateProtocolWrite) + validateProtocolWrite(snapshot.protocol) } finalActions = RowId.assignFreshRowIds( @@ -2669,19 +2710,21 @@ trait OptimisticTransactionImpl extends TransactionHelper checkColumnDefaults(op) } - verifyAmtBackReferences(finalActions) + verifyAmtBackReferences(finalActions, amtCheckpointProviderOpt) finalActions } /** * Test-only invariant check for AMT back references, run on every commit to an AMT-backed table. */ - private def verifyAmtBackReferences(finalActions: Seq[Action]): Unit = { + private def verifyAmtBackReferences( + finalActions: Seq[Action], + amtProviderOpt: => Option[AMTCheckpointProvider]): Unit = { if (!DeltaUtils.isTesting) return - snapshot.checkpointProvider match { - case amt: AMTCheckpointProvider => - amt.verifyCommitBackReferences(spark, deltaLog, finalActions) - case _ => + amtProviderOpt match { + case Some(amt) => + amt.verifyCommitBackReferences(spark, deltaLog, catalogTable, finalActions) + case None => // Not an AMT-backed table: no file action may carry a back reference. finalActions.foreach { case a: AddFile if a.backReference.isDefined => @@ -2760,8 +2803,12 @@ trait OptimisticTransactionImpl extends TransactionHelper deltaLog, "delta.commit.allAttempts") { lockCommitIfEnabled { var commitVersion = attemptVersion - var updatedCurrentTransactionInfo = currentTransactionInfo - val amtWriterManager = new AMTWriterManager(snapshot, currentTransactionInfo.op) + // The [[CurrentTransactionInfo]] with everything except the prepareCommit (Uniform/AMT) + // edits. This is the logical set of actions we want to commit, and it is what gets fed to + // the ConflictChecker in case of a conflict. + var updatedUnpreparedCurrentTransactionInfo = currentTransactionInfo + val amtWriterManager = + new AMTWriterManager(currentTransactionInfo.txnId, snapshot, currentTransactionInfo.op) val isFsToCcCommit = snapshot.metadata.coordinatedCommitsCoordinatorName.isEmpty && metadata.coordinatedCommitsCoordinatorName.nonEmpty @@ -2770,24 +2817,41 @@ trait OptimisticTransactionImpl extends TransactionHelper spark.conf.get(DeltaSQLConf.DELTA_MAX_NON_CONFLICT_RETRY_COMMIT_ATTEMPTS) var nonConflictAttemptNumber = 0 var shouldCheckForConflicts = false + // Idempotent handling: sometimes the commit RPC fails but the commit actually landed. We + // discover that in the next iteration during rebase. Once we establish that our commit from + // the previous attempt really succeeded, we finalize it by invoking performPostCommitActions + // with this lastPreparedCommitResult (the exact actions, metrics, and AMT write result of the + // attempt that landed) rather than re-reading them back. + var lastPreparedCommitResult: Option[PrepareCommitResult] = None for (attemptNumber <- 0 to maxRetryAttempts) { try { val (postCommitSnapshot, committedTransactionInfo) = if (!shouldCheckForConflicts) { - doCommit( - commitVersion, updatedCurrentTransactionInfo, attemptNumber, isolationLevel, + val prepareCommitResult = prepareCommit( + commitVersion, + updatedUnpreparedCurrentTransactionInfo, amtWriterManager) + lastPreparedCommitResult = Some(prepareCommitResult) + doCommit( + prepareCommitResult.commitVersion, + prepareCommitResult.currentTransactionInfo, + prepareCommitResult.actionsToWriteInLogFile, + prepareCommitResult.amtWriteResultForLastCheckpointOpt, + prepareCommitResult.prepMetrics, + attemptNumber, + isolationLevel) } else recordDeltaOperation(deltaLog, "delta.commit.retry") { - // The [[CurrentTransactionInfo]] might keep on getting updated while resolving - // conflicts against the already committed concurrent transactions. This can happen - // when we resolve conflicts against no-data-change transaction - we might map our - // readFiles or we might rollback the no-data-change transaction and update our actions - // that we want to commit. - val (newCommitVersion, newCurrentTransactionInfo) = checkForConflicts( - commitVersion, updatedCurrentTransactionInfo, attemptNumber, isolationLevel) + val rebaseResult = rebaseCurrentTransactionInfo( + commitVersion, + updatedUnpreparedCurrentTransactionInfo, + attemptNumber, + isolationLevel, + lastPreparedCommitResult, + amtWriterManager + ) // Check if we may have already committed this version but retried due to // a transient error. If that's the case, just return success. - newCurrentTransactionInfo.idempotentCommitAlreadyLandedAt match { + rebaseResult.currentTransactionInfo.idempotentCommitAlreadyLandedAt match { case Some(committedVersion) => // The idempotent self-commit is expected to land at exactly the version we were // attempting. If it landed at a different version, surface it. @@ -2796,35 +2860,48 @@ trait OptimisticTransactionImpl extends TransactionHelper s"Idempotent self-commit landed at version $committedVersion but the " + s"transaction was attempting version $commitVersion.") } - // newCurrentTransactionInfo carries the actions read back from the commit file - // that actually landed (adopted in resolveConflicts on the idempotent path), so - // post-commit stats and the resulting CommittedTransaction reflect exactly what - // was committed rather than the pre-doCommit action set - val actions = newCurrentTransactionInfo.finalActionsToCommit + // rebaseResult is the prepared result from the attempt that already landed (the + // commit RPC failed but the write succeeded); it was validated against the landed + // commit inside rebaseCurrentTransactionInfo. Finalize it directly -- its actions, + // metrics, and AMT write result are exactly what was committed. + val actions = rebaseResult.actionsToWriteInLogFile val jsonActions = actions.map(_.json) val (postCommitSnapshot, txnInfo) = performPostCommitActions( committedVersion, actions, jsonActions, - newCurrentTransactionInfo, + rebaseResult.currentTransactionInfo, isolationLevel, fsWriteStartNanoOpt = None, - amtWriterManager = amtWriterManager, + prepMetrics = rebaseResult.prepMetrics, newChecksumOpt = None, - amtWriteResultOpt = None, + amtWriteResultOpt = rebaseResult.amtWriteResultForLastCheckpointOpt, commitOpt = None, isIdempotentRetry = true ) return (committedVersion, postCommitSnapshot, txnInfo) case None => // fall through to the normal retry commit below } - commitVersion = newCommitVersion - updatedCurrentTransactionInfo = newCurrentTransactionInfo + commitVersion = rebaseResult.commitVersion + // Feed the unprepared (conflict-resolved) txn info -- not this attempt's Uniform/AMT + // edits -- into any later retry's conflict check. + updatedUnpreparedCurrentTransactionInfo = + rebaseResult.currentTransactionInfoBeforePreparedResult + lastPreparedCommitResult = Some(rebaseResult) + // Re-check the AMT back references on the rebased actions after conflict resolution. + verifyAmtBackReferences( + rebaseResult.currentTransactionInfo.finalActionsToCommit, + amtWriterManager.preCommitLatestAMTCheckpointProviderOpt) doCommit( - commitVersion, updatedCurrentTransactionInfo, attemptNumber, isolationLevel, - amtWriterManager) + rebaseResult.commitVersion, + rebaseResult.currentTransactionInfo, + rebaseResult.actionsToWriteInLogFile, + rebaseResult.amtWriteResultForLastCheckpointOpt, + rebaseResult.prepMetrics, + attemptNumber, + isolationLevel) } - return (commitVersion, postCommitSnapshot, updatedCurrentTransactionInfo) + return (commitVersion, postCommitSnapshot, updatedUnpreparedCurrentTransactionInfo) } catch { case _: FileAlreadyExistsException if isFsToCcCommit => // Don't retry if this commit tries to upgrade the table from filesystem to managed @@ -2861,31 +2938,74 @@ trait OptimisticTransactionImpl extends TransactionHelper maxRetryAttempts + 1, commitVersion, attemptVersion, - updatedCurrentTransactionInfo.finalActionsToCommit.length, + updatedUnpreparedCurrentTransactionInfo.finalActionsToCommit.length, totalCommitAttemptTime) } } /** - * Commit `actions` using `attemptVersion` version number. Throws a FileAlreadyExistsException - * if any conflicts are detected. + * If we believe that this is an idempotent self-commit, the actions that we read from the + * winning commit should match exactly with what we were trying to write. Note that the + * conflict checking logic injects a `version` in the winning commit's CommitInfo (see + * [[WinningCommitSummary]]), which we explicitly ignore during the comparison. + */ + private def validateIdempotentCommitInvariants( + prepared: PrepareCommitResult, + landedActions: Seq[Action]): Unit = { + // Comparing the full action lists has a performance cost, so this validation is gated behind a + // flag and is intended to be removed once the idempotent self-commit finalization path is + // proven. + if (!spark.conf.get( + DeltaSQLConf.DELTA_COMMIT_IDEMPOTENCY_CHECK_VALIDATE_PREPARED_ACTIONS_ENABLED)) { + return + } + val clearCommitInfoVersion: Action => Action = { + case ci: CommitInfo => ci.copy(version = None) + case other => other + } + val msg = "Prepared commit actions do not match the actions read back from the landed " + + "idempotent self-commit (ignoring CommitInfo.version)." + deltaAssertAndThrow( + prepared.actionsToWriteInLogFile.map(clearCommitInfoVersion) == + landedActions.map(clearCommitInfoVersion), + name = "idempotentCommitPreparedActionsMismatch", + msg = msg, + throwable = new IllegalStateException(msg), + deltaLog = deltaLog) + } + + /** + * Prepares one commit attempt: + * - handles writing Adaptive metadata tree + * - generates Uniform/Iceberg metadata + * Invoked directly on the first attempt; + * On a conflict retry, [[rebaseCurrentTransactionInfo]] invokes it after conflict resolution. * - * @param amtWriterManager the AMT writer for this commit, shared across all attempts. - * @return the post-commit snapshot and transaction info used by the successful commit. */ - protected def doCommit( + protected def prepareCommit( attemptVersion: Long, currentTransactionInfo: CurrentTransactionInfo, - attemptNumber: Int, - isolationLevel: IsolationLevel, - amtWriterManager: AMTWriterManager): (Snapshot, CurrentTransactionInfo) = { + amtWriterManager: AMTWriterManager, + winningCommitMetrics: Seq[WinningCommitMetrics] = Seq.empty): PrepareCommitResult = { val targetCatalogTable = catalogTable + // Validate resolved file actions before writing AMT. Iceberg conversion does not change them. + ConflictChecker.trackDataChange( + spark, + currentTransactionInfo.finalActionsToCommit.iterator, + deltaLog, + currentTransactionInfo.op, + callerContext = "doCommit").foreach(_ => ()) + val amtWriteResultOpt = amtWriterManager.writeAMT( + nextAttemptVersion = attemptVersion, + currentTransactionInfo = currentTransactionInfo, + preCommitLogSegment = preCommitLogSegment, + winningCommitMetricsForConflictedRange = winningCommitMetrics) + // If the table requires atomic Iceberg metadata generation // , generate iceberg metadata and update the transaction info. var icebergMetadataGenerationDurationMsOpt: Option[Long] = None - val updatedCurrentTransactionInfo = - targetCatalogTable - .map { table => + var updatedCurrentTransactionInfo = + targetCatalogTable.map { table => val startNanos = System.nanoTime() // Following call generates Iceberg metadata and updates CurrentTransactionInfo val (updatedInfo, isConversionPerformed) = @@ -2902,18 +3022,132 @@ trait OptimisticTransactionImpl extends TransactionHelper } updatedInfo }.getOrElse(currentTransactionInfo) + val baseActions = updatedCurrentTransactionInfo.finalActionsToCommit - val amtWriteResultOpt = amtWriterManager.writeAMT( - commitVersion = attemptVersion, - currentTransactionInfo = updatedCurrentTransactionInfo, - preCommitLogSegment = preCommitLogSegment) val actions = amtWriteResultOpt match { case Some(result) if !result.includeActionsInCommitJson => throw new UnsupportedOperationException( "Omitting file actions from the commit JSON is not yet supported.") - case Some(result) => baseActions :+ result.checkpoint + case Some(result) => + // When we have become a manifest commit, update the reference in the current commitInfo so + // the persisted commit (and its derived CRC) records where the AMT checkpoint lives. + updatedCurrentTransactionInfo = AMTUtils.updateCurrentTransactionInfo( + updatedCurrentTransactionInfo, + newLastManifestCommit = LastManifestCommit( + version = attemptVersion, + contentRootVersion = result.contentRootVersion) + ) + // CatalogOwned tables assume that before a checkpoint is written, all the commits till the + // checkpoint version (inclusive) have been backfilled (in [[Checkpoints.writeCheckpoint]]). + // For AMT though, since the checkpoint itself is a (manifest) commit, we lose the inclusive + // guarantee: the manifest commit itself cannot be backfilled as it hasn't been written yet. + // We still backfill up to the previous version (attemptVersion - 1), and we store the + // unbackfilled manifest commit file status in an extra field in LogSegment. + // We assume the readSnapshot's commit-coordinator is unchanged, otherwise the conflict + // checker would have detected a conflict earlier. + CatalogOwnedTableUtils + .populateTableCommitCoordinatorFromCatalog(spark, targetCatalogTable, snapshot) + .foreach { readSnapshotTableCommitCoordinatorClient => + CoordinatedCommitsUtils.ensureCommitFilesBackfilled( + version = attemptVersion - 1, + deltaLog = deltaLog, + tableCommitCoordinatorClient = readSnapshotTableCommitCoordinatorClient, + deltaCommitFileProvider = DeltaCommitFileProvider(logPath, preCommitLogSegment), + catalogTableOpt = targetCatalogTable) + } + // Recompute the actions from the patched txn info so the committed CommitInfo carries the + // reference, then append the inline checkpoint action. + updatedCurrentTransactionInfo.finalActionsToCommit :+ result.checkpoint case None => baseActions } + // Surface the AMT checkpoint that this commit emits inline (at `attemptVersion`) so the + // incremental CRC can tell whether the resulting tree is root-only. This is the current + // attempt's tree, not `preCommitLatestAMTCheckpointOpt` (which tracks [0, attemptVersion-1]). + val txnInfoForCommit = amtWriteResultOpt.map(_.checkpoint) match { + case Some(amtCheckpoint) => + updatedCurrentTransactionInfo.copy( + currentCommitAttemptAMTCheckpointOpt = Some(amtCheckpoint)) + case None => updatedCurrentTransactionInfo + } + PrepareCommitResult( + commitVersion = attemptVersion, + currentTransactionInfo = txnInfoForCommit, + currentTransactionInfoBeforePreparedResult = currentTransactionInfo, + actionsToWriteInLogFile = actions, + amtWriteResultForLastCheckpointOpt = amtWriteResultOpt, + prepMetrics = CommitPrepMetrics( + icebergMetadataGenerationDurationMsOpt = icebergMetadataGenerationDurationMsOpt)) + } + + /** + * Resolves conflicts against the winning commits and re-prepares this transaction on top of them. + * Runs on a conflict retry: resolves conflicts, updates the post-conflict metrics, then re-runs + * the Uniform + AMT preparation via [[prepareCommit]]. Together with [[prepareCommit]] + * (used on the first attempt) it is the only place that edits the [[CurrentTransactionInfo]] once + * a commit attempt has begun. + * + * On the idempotent self-commit path (our commit already landed under a transient-error retry) + * returns the last prepared result directly. + * + */ + protected def rebaseCurrentTransactionInfo( + commitVersion: Long, + currentTransactionInfo: CurrentTransactionInfo, + attemptNumber: Int, + isolationLevel: IsolationLevel, + lastPreparedCommitResult: Option[PrepareCommitResult], + amtWriterManager: AMTWriterManager): PrepareCommitResult = { + // The [[CurrentTransactionInfo]] might keep on getting updated while resolving + // conflicts against the already committed concurrent transactions. This can happen + // when we resolve conflicts against no-data-change transaction - we might map our + // readFiles or we might rollback the no-data-change transaction and update our actions + // that we want to commit. + val (newCommitVersion, newCurrentTransactionInfo, winningCommitMetrics) = checkForConflicts( + commitVersion, + currentTransactionInfo, + attemptNumber, + isolationLevel, + amtWriterManager) + newCurrentTransactionInfo.idempotentCommitAlreadyLandedAt match { + case Some(landedVersion) => + // Our commit from a previous attempt already landed (the commit RPC failed but the write + // succeeded), so there is nothing new to prepare. + val prepared = lastPreparedCommitResult.getOrElse( + throw new IllegalStateException( + "Idempotent self-commit detected but no prior prepared commit result is " + + "available to finalize it.")) + validateIdempotentCommitInvariants( + prepared, newCurrentTransactionInfo.finalActionsToCommit) + return prepared.copy( + currentTransactionInfo = prepared.currentTransactionInfo.copy( + idempotentCommitAlreadyLandedAt = Some(landedVersion))) + case None => // nothing landed yet; prepare this attempt normally below + } + var rebasedTransactionInfo = newCurrentTransactionInfo + prepareCommit( + newCommitVersion, + rebasedTransactionInfo, + amtWriterManager, + winningCommitMetrics) + } + + /** + * Persists a prepared commit attempt: writes the commit file and runs the post-commit actions. + * Throws a FileAlreadyExistsException if any conflicts are detected. The [[PrepareCommitResult]] + * carries everything to persist -- all [[CurrentTransactionInfo]] preparation happened in + * [[prepareCommit]] / [[rebaseCurrentTransactionInfo]] before this is called. + * + * @return the post-commit snapshot and transaction info used by the successful commit. + */ + protected def doCommit( + attemptVersion: Long, + txnInfoForCommit: CurrentTransactionInfo, + actions: Seq[Action], + amtWriteResultForLastCheckpointOpt: Option[AMTWriteResult], + prepMetrics: CommitPrepMetrics, + attemptNumber: Int, + isolationLevel: IsolationLevel): (Snapshot, CurrentTransactionInfo) = { + val targetCatalogTable = catalogTable logInfo( log"Attempting to commit version ${MDC(DeltaLogKeys.VERSION, attemptVersion)} with " + log"${MDC(DeltaLogKeys.NUM_ACTIONS, actions.size.toLong)} actions with " + @@ -2936,7 +3170,7 @@ trait OptimisticTransactionImpl extends TransactionHelper val jsonActions = actions.map(_.json) val (newChecksumOpt, commit, committedTransactionInfo) = - writeCommitFile(attemptVersion, jsonActions.toIterator, updatedCurrentTransactionInfo) + writeCommitFile(attemptVersion, jsonActions.toIterator, txnInfoForCommit) performPostCommitActions( attemptVersion, @@ -2945,9 +3179,9 @@ trait OptimisticTransactionImpl extends TransactionHelper committedTransactionInfo, isolationLevel, Some(fsWriteStartNano), - amtWriterManager, + prepMetrics, newChecksumOpt, - amtWriteResultOpt, + amtWriteResultForLastCheckpointOpt, commitOpt = Some(commit) ) } @@ -2960,7 +3194,7 @@ trait OptimisticTransactionImpl extends TransactionHelper committedTransactionInfo: CurrentTransactionInfo, isolationLevel: IsolationLevel, fsWriteStartNanoOpt: Option[Long], - amtWriterManager: AMTWriterManager, + prepMetrics: CommitPrepMetrics, newChecksumOpt: Option[VersionChecksum], amtWriteResultOpt: Option[AMTWriteResult], commitOpt: Option[Commit], @@ -2973,25 +3207,30 @@ trait OptimisticTransactionImpl extends TransactionHelper executionObserver.beginPostCommit() val catalogTableForPostCommitSnapshot = catalogTable - val postCommitSnapshot = deltaLog.updateAfterCommit( + val postCommitSnapshot = resolvePostCommitSnapshot( attemptVersion, commitOpt, newChecksumOpt, - preCommitLogSegment, catalogTableForPostCommitSnapshot, - amtCheckpointOpt = amtWriteResultOpt.map(_.checkpoint), + amtCheckpointWrittenInCommitOpt = amtWriteResultOpt.map(_.checkpoint), isIdempotentRetry = isIdempotentRetry) val postCommitReconstructionTime = System.nanoTime() maintenanceOperation = if ( - !postCommitSnapshot.protocol.isFeatureSupported(AdaptiveMetadataTableFeature)) { + !AMTUtils.amtEnabled(postCommitSnapshot)) { MaintenanceOperation( shouldCheckpoint = isCheckpointNeeded(attemptVersion, postCommitSnapshot)) } else if (amtWriteResultOpt.isEmpty) { // The AMT writer did not write inline, so schedule a follow-up OPTIMIZE CHECKPOINT commit // (issued by the post-commit checkpoint hook) // if needed. - amtWriterManager.planMaintenance(attemptVersion, postCommitSnapshot) + AMTWriterManager.planMaintenance( + spark, snapshot, committedTransactionInfo.op, attemptVersion, postCommitSnapshot) } else { + // The _last_checkpoint file must be updated here rather than the checkpoint hook, because the + // hook does not know about the new AMT checkpoint emitted in this commit. + amtWriteResultOpt.foreach { writeResult => + deltaLog.writeLastCheckpointFileForAMT(manifestCommitVersion = attemptVersion, writeResult) + } // The AMT was written inline with this commit (always incremental). A full rewrite is never // inlined, so schedule a follow-up full OPTIMIZE CHECKPOINT commit when the full-rewrite // cadence is due -- otherwise a table whose commits consistently inline would never get a @@ -3002,13 +3241,14 @@ trait OptimisticTransactionImpl extends TransactionHelper // the commit JSON, and skipping it would push those actions back inline and bloat the commit // JSON -- exactly what the inline path avoids. The rare double materialization is the // accepted cost of never writing a very large commit JSON. - amtWriterManager.planMaintenanceAfterInlineWrite(attemptVersion, postCommitSnapshot) + AMTWriterManager.planMaintenanceAfterInlineWrite( + spark, snapshot, committedTransactionInfo.op, attemptVersion, postCommitSnapshot) } val commitStatsComputer = new CommitStatsComputer() // Add to commit stats and consume the returned iterator. commitStatsComputer.addToCommitStats(actions.toIterator).foreach(_ => ()) partitionsAddedToOpt = Some(commitStatsComputer.getPartitionsAddedByTransaction) - collectAutoOptimizeStatsAndFinalize(actions, deltaLog.unsafeVolatileTableId) + collectAutoOptimizeStatsAndFinalize(actions, commitTableId) val commitSizeBytes: Long = jsonActions.map(_.length.toLong).sum commitStatsComputer.finalizeAndEmitCommitStats( spark, @@ -3028,8 +3268,12 @@ trait OptimisticTransactionImpl extends TransactionHelper fileSizeHistogramOpt = postCommitSnapshot.checksumOpt.flatMap(_.fileSizeHistogram), commitInfoOpt = committedTransactionInfo.commitInfo, commitSizeBytes = commitSizeBytes, - amtWriteMetricsOpt = Option.when(amtWriterManager.metrics.attempts.nonEmpty)( - amtWriterManager.metrics), + amtCommitStatsOpt = amtWriteResultOpt.map { result => + AMTCommitStats( + contentRootVersion = result.contentRootVersion, + lastAMTWriteMetrics = result.amtWriteMetrics, + includeActionsInCommitJson = result.includeActionsInCommitJson) + }, isIdempotentRetry = isIdempotentRetry ) @@ -3189,6 +3433,9 @@ trait OptimisticTransactionImpl extends TransactionHelper protected def incrementallyDeriveChecksum( attemptVersion: Long, currentTransactionInfo: CurrentTransactionInfo): Option[VersionChecksum] = { + val effectiveLatestAMTCheckpointAtCommitVersion = + currentTransactionInfo.currentCommitAttemptAMTCheckpointOpt + .orElse(currentTransactionInfo.preCommitLatestAMTCheckpointOpt) incrementallyDeriveChecksum( spark, deltaLog, @@ -3201,7 +3448,8 @@ trait OptimisticTransactionImpl extends TransactionHelper previousVersionState = scala.Left(snapshot), mustIncludeFileSizeHistogram = spark.conf.get(DeltaSQLConf.DELTA_FILE_SIZE_HISTOGRAM_ENABLED), - includeAddFilesInCrc = Snapshot.shouldIncludeAddFilesInCrc(spark, snapshot, metadata) + includeAddFilesInCrc = Snapshot.shouldIncludeAddFilesInCrc( + spark, snapshot, metadata, effectiveLatestAMTCheckpointAtCommitVersion) ).toOption } @@ -3214,11 +3462,12 @@ trait OptimisticTransactionImpl extends TransactionHelper checkVersion: Long, currentTransactionInfo: CurrentTransactionInfo, attemptNumber: Int, - commitIsolationLevel: IsolationLevel) - : (Long, CurrentTransactionInfo) = recordDeltaOperation( + commitIsolationLevel: IsolationLevel, + amtWriterManager: AMTWriterManager) + : (Long, CurrentTransactionInfo, Seq[WinningCommitMetrics]) = recordDeltaOperation( deltaLog, "delta.commit.retry.conflictCheck", - tags = Map(TAG_LOG_STORE_CLASS -> deltaLog.store.getClass.getName)) { + tags = Map(TAG_LOG_STORE_CLASS -> commitLogStoreClassNameForTag)) { DeltaTableV2.withEnrichedUnsupportedTableException(catalogTable) { val fileStatuses = getConflictingVersions(checkVersion) @@ -3253,17 +3502,24 @@ trait OptimisticTransactionImpl extends TransactionHelper log"${MDC(DeltaLogKeys.VERSION2, nextAttemptVersion)}) " + log"with current txn having " + txnDetailsLog) - val updatedCurrentTransactionInfo = { + val (updatedCurrentTransactionInfo, winningCommitMetrics) = { if (expected.isEmpty) { - currentTransactionInfo + (currentTransactionInfo, Seq.empty[WinningCommitMetrics]) } else { - resolveConflicts( + val (currentTransactionInfoAfterResolvingConflicts, metrics) = resolveConflicts( currentTransactionInfo = currentTransactionInfo, firstWinningVersion = expected.head, lastWinningVersion = expected.last, conflictingCommitFiles = fileStatuses, commitIsolationLevel = commitIsolationLevel) + // On an AMT rebase where a winning commit installed a new tree, re-point the writer's + // cached AMT provider at the folded tree, then re-derive the committed file actions' back + // references against it (a no-op otherwise). + amtWriterManager.updatePreCommitLatestAMTCheckpointProvider( + currentTransactionInfoAfterResolvingConflicts) + (amtWriterManager.rebaseBackReferences(currentTransactionInfoAfterResolvingConflicts), + metrics) } } @@ -3275,7 +3531,7 @@ trait OptimisticTransactionImpl extends TransactionHelper log"with current txn having " + txnDetailsLog + log"${MDC(DeltaLogKeys.TIME_MS, clock.getTimeMillis() - commitAttemptStartTimeMillis)} " + log"ms since start") - (nextAttemptVersion, updatedCurrentTransactionInfo) + (nextAttemptVersion, updatedCurrentTransactionInfo, winningCommitMetrics) } } @@ -3295,14 +3551,16 @@ trait OptimisticTransactionImpl extends TransactionHelper firstWinningVersion: Long, lastWinningVersion: Long, conflictingCommitFiles: Seq[FileStatus], - commitIsolationLevel: IsolationLevel) : CurrentTransactionInfo = { + commitIsolationLevel: IsolationLevel) + : (CurrentTransactionInfo, Seq[WinningCommitMetrics]) = { var updatedCurrentTransactionInfo = currentTransactionInfo + val winningCommitMetrics = Seq.newBuilder[WinningCommitMetrics] (firstWinningVersion to lastWinningVersion) .zip(conflictingCommitFiles) .foreach { case (otherCommitVersion, otherCommitFileStatus) => - val winningCommitSummary = WinningCommitSummary.createFromFileStatus( - deltaLog, otherCommitFileStatus) + val winningCommitSummary = readWinningCommitSummary(otherCommitFileStatus) + winningCommitMetrics += WinningCommitMetrics.fromWinningCommitSummary(winningCommitSummary) val conflictChecker = new ConflictChecker( spark, @@ -3320,7 +3578,7 @@ trait OptimisticTransactionImpl extends TransactionHelper log"${MDC(DeltaLogKeys.DURATION, clock.getTimeMillis() - commitAttemptStartTimeMillis)} ms since start") } - updatedCurrentTransactionInfo + (updatedCurrentTransactionInfo, winningCommitMetrics.result()) } /** @@ -3370,9 +3628,51 @@ trait OptimisticTransactionImpl extends TransactionHelper } } + /** + * Reads the actions of a winning commit into a [[WinningCommitSummary]] for conflict resolution. + */ + protected def readWinningCommitSummary(fileStatus: FileStatus): WinningCommitSummary = + WinningCommitSummary.createFromFileStatus(deltaLog, fileStatus) + /** Returns the version that the first attempt will try to commit at. */ private[delta] def getFirstAttemptVersion: Long = readVersion + 1L + /** + * Validates that this client may write with the given protocol. + */ + protected def validateProtocolWrite(protocol: Protocol): Unit = + deltaLog.protocolWrite(protocol) + + /** + * Resolves the post-commit snapshot after a successful commit-file write. It resolves it and + * installs it as the DeltaLog's current snapshot. + */ + protected def resolvePostCommitSnapshot( + committedVersion: Long, + commitOpt: Option[Commit], + newChecksumOpt: Option[VersionChecksum], + catalogTableOpt: Option[CatalogTable], + amtCheckpointWrittenInCommitOpt: Option[Checkpoint] = None, + isIdempotentRetry: Boolean = false): Snapshot = + deltaLog.updateAfterCommit( + committedVersion, commitOpt, newChecksumOpt, preCommitLogSegment, catalogTableOpt, + amtCheckpointWrittenInCommitOpt = amtCheckpointWrittenInCommitOpt, + isIdempotentRetry = isIdempotentRetry) + + /** + * The table id used for auto-optimize commit stats. V1 uses `deltaLog.unsafeVolatileTableId` + */ + protected def commitTableId: String = deltaLog.unsafeVolatileTableId + + /** + * The read snapshot's [[AMTCheckpointProvider]], or None if the table is not AMT-backed. + */ + protected def amtCheckpointProviderOpt: Option[AMTCheckpointProvider] = + snapshot.checkpointProvider match { + case amt: AMTCheckpointProvider => Some(amt) + case _ => None + } + /** Returns the conflicting commit information */ protected def getConflictingVersions(previousAttemptVersion: Long): Seq[FileStatus] = { assert(previousAttemptVersion == preCommitLogSegment.version + 1) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/PreDowngradeTableFeatureCommand.scala b/spark/src/main/scala/org/apache/spark/sql/delta/PreDowngradeTableFeatureCommand.scala index 84ad43dee38..efe9de0191d 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/PreDowngradeTableFeatureCommand.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/PreDowngradeTableFeatureCommand.scala @@ -271,7 +271,11 @@ case class DeletionVectorsPreDowngradeCommand(table: DeltaTableV2) op = DeltaOperations.AddDeletionVectorsTombstones, newProtocolOpt = None, context = Map.empty, - metrics = Map("dvTombstonesWithinRetentionPeriod" -> tombstonesToAddCount.toString)) + metrics = Map("dvTombstonesWithinRetentionPeriod" -> tombstonesToAddCount.toString), + // The commit is only DV tombstones: RemoveFiles whose path is a deletion vector file + // rather than a data file, written so that VACUUM can delete those DVs. They drop no + // rows so dataChange is false. + dataChange = Some(false)) } else { table.startTransaction(Some(snapshotToUse)) .commit(actionsToCommit.toList, DeltaOperations.AddDeletionVectorsTombstones) @@ -339,6 +343,15 @@ case class DeletionVectorsPreDowngradeCommand(table: DeltaTableV2) } } +case class AdaptiveMetadataPreDowngradeCommand(table: DeltaTableV2) + extends PreDowngradeTableFeatureCommand { + + override def removeFeatureTracesIfNeeded(spark: SparkSession): PreDowngradeStatus = { + throw new UnsupportedOperationException( + s"Dropping the ${AdaptiveMetadataTableFeature.name} table feature is not yet supported.") + } +} + case class V2CheckpointPreDowngradeCommand(table: DeltaTableV2) extends PreDowngradeTableFeatureCommand with DeltaLogging { @@ -611,6 +624,19 @@ case class GeospatialPreDowngradeCommand(table: DeltaTableV2) } } +case class FileTypePreDowngradeCommand(table: DeltaTableV2) + extends PreDowngradeTableFeatureCommand { + + /** + * There is no `file` type in the schema yet, so a table can never contain `file` columns and + * there are never any traces of the feature to remove. Once `file` column support lands, this + * must remove or reject remaining `file` usages before the feature can be dropped (mirroring + * [[GeospatialPreDowngradeCommand]]). + */ + override def removeFeatureTracesIfNeeded(spark: SparkSession): PreDowngradeStatus = + PreDowngradeStatus.DID_NOT_PERFORM_CHANGES +} + case class ColumnMappingPreDowngradeCommand(table: DeltaTableV2) extends PreDowngradeTableFeatureCommand with DeltaLogging { diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/RowId.scala b/spark/src/main/scala/org/apache/spark/sql/delta/RowId.scala index b2e9e472e64..39651f0faea 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/RowId.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/RowId.scala @@ -141,7 +141,7 @@ object RowId { */ private[delta] def extractHighWatermark(snapshot: Snapshot): Option[Long] = if (isSupported(snapshot.protocol)) { - RowTrackingMetadataDomain.fromSnapshot(snapshot).map(_.rowIdHighWaterMark) + snapshot.getRowTrackingHighWaterMark() } else { None } diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/RowIndexFilterMetadataColumnUtils.scala b/spark/src/main/scala/org/apache/spark/sql/delta/RowIndexFilterMetadataColumnUtils.scala new file mode 100644 index 00000000000..aa5166cc217 --- /dev/null +++ b/spark/src/main/scala/org/apache/spark/sql/delta/RowIndexFilterMetadataColumnUtils.scala @@ -0,0 +1,215 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.spark.sql.delta + +import scala.collection.mutable.ArrayBuffer +import scala.util.control.NonFatal + +import org.apache.spark.sql.delta.RowIndexFilter + +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.execution.vectorized.{OffHeapColumnVector, OnHeapColumnVector, WritableColumnVector} +import org.apache.spark.sql.types.{ByteType, StructField} +import org.apache.spark.sql.vectorized.{ColumnarBatch, ColumnarBatchRow, ColumnVector} + +/** Materializes row-index-filter and row-index metadata columns in Parquet reader output. */ +private[delta] object RowIndexFilterMetadataColumnUtils { + + /** + * Populates the row-index-filter marker and optional generated row-index column after the caller + * has resolved the file's [[RowIndexFilter]]. + */ + private[delta] def iteratorWithAdditionalMetadataColumns( + iterator: Iterator[Object], + rowIndexFilterOpt: Option[RowIndexFilter], + isRowDeletedColumnOpt: Option[ColumnMetadata], + rowIndexColumnOpt: Option[ColumnMetadata], + useOffHeapBuffers: Boolean, + useMetadataRowIndex: Boolean): Iterator[Object] = { + // We only generate the row index column when predicate pushdown is not enabled. + val rowIndexColumnToWriteOpt = if (useMetadataRowIndex) None else rowIndexColumnOpt + val metadataColumnsToWrite = + Seq(isRowDeletedColumnOpt, rowIndexColumnToWriteOpt).filter(_.nonEmpty).map(_.get) + + // When metadata.row_index is not used there is no way to verify the Parquet index is + // starting from 0. We disable the splits, so the assumption is ParquetFileFormat respects + // that. + var rowIndex: Long = 0 + + // Used only when non-column row batches are received from the Parquet reader + val tempVector = new OnHeapColumnVector(1, ByteType) + + iterator.map { row => + row match { + case batch: ColumnarBatch => // When vectorized Parquet reader is enabled. + val size = batch.numRows() + // Create vectors for all needed metadata columns. + // We can't use the one from Parquet reader as it set the + // [[WritableColumnVector.isAllNulls]] to true and it can't be reset with using any + // public APIs. + trySafely( + useOffHeapBuffers, size, metadataColumnsToWrite) { writableVectors => + val indexVectorTuples = new ArrayBuffer[(Int, ColumnVector)] + + // When predicate pushdown is enabled we use _metadata.row_index. Therefore, + // we only need to construct the isRowDeleted column. + var index = 0 + isRowDeletedColumnOpt.foreach { columnMetadata => + val isRowDeletedVector = writableVectors(index) + if (useMetadataRowIndex) { + rowIndexFilterOpt.get.materializeIntoVectorWithRowIndex( + size, batch.column(rowIndexColumnOpt.get.index), isRowDeletedVector) + } else { + rowIndexFilterOpt.get + .materializeIntoVector(rowIndex, rowIndex + size, isRowDeletedVector) + } + indexVectorTuples += (columnMetadata.index -> isRowDeletedVector) + index += 1 + } + + rowIndexColumnToWriteOpt.foreach { columnMetadata => + val rowIndexVector = writableVectors(index) + // populate the row index column value. + for (i <- 0 until size) { + rowIndexVector.putLong(i, rowIndex + i) + } + + indexVectorTuples += (columnMetadata.index -> rowIndexVector) + index += 1 + } + + val newBatch = replaceVectors(batch, indexVectorTuples.toSeq: _*) + rowIndex += size + newBatch + } + + case columnarRow: ColumnarBatchRow => + // When vectorized reader is enabled but returns immutable rows instead of + // columnar batches [[ColumnarBatchRow]]. So we have to copy the row as a + // mutable [[InternalRow]] and set the `row_index` and `is_row_deleted` + // column values. This is not efficient. It should affect only the wide + // tables. https://github.com/delta-io/delta/issues/2246 + val newRow = columnarRow.copy(); + isRowDeletedColumnOpt.foreach { columnMetadata => + val rowIndexForFiltering = if (useMetadataRowIndex) { + columnarRow.getLong(rowIndexColumnOpt.get.index) + } else { + rowIndex + } + rowIndexFilterOpt.get.materializeSingleRowWithRowIndex(rowIndexForFiltering, tempVector) + newRow.setByte(columnMetadata.index, tempVector.getByte(0)) + } + + rowIndexColumnToWriteOpt + .foreach(columnMetadata => newRow.setLong(columnMetadata.index, rowIndex)) + rowIndex += 1 + + newRow + case rest: InternalRow => // When vectorized Parquet reader is disabled + // Temporary vector variable used to get DV values from RowIndexFilter + // Currently the RowIndexFilter only supports writing into a columnar vector + // and doesn't have methods to get DV value for a specific row index. + // TODO: This is not efficient, but it is ok given the default reader is vectorized + isRowDeletedColumnOpt.foreach { columnMetadata => + val rowIndexForFiltering = if (useMetadataRowIndex) { + rest.getLong(rowIndexColumnOpt.get.index) + } else { + rowIndex + } + rowIndexFilterOpt.get.materializeSingleRowWithRowIndex(rowIndexForFiltering, tempVector) + rest.setByte(columnMetadata.index, tempVector.getByte(0)) + } + + rowIndexColumnToWriteOpt + .foreach(columnMetadata => rest.setLong(columnMetadata.index, rowIndex)) + rowIndex += 1 + rest + case others => + throw new RuntimeException( + s"Parquet reader returned an unknown row type: ${others.getClass.getName}") + } + } + } + + /** Utility method to create a new writable vector */ + private[delta] def newVector( + useOffHeapBuffers: Boolean, size: Int, dataType: StructField): WritableColumnVector = { + if (useOffHeapBuffers) { + OffHeapColumnVector.allocateColumns(size, Seq(dataType).toArray)(0) + } else { + OnHeapColumnVector.allocateColumns(size, Seq(dataType).toArray)(0) + } + } + + /** Try the operation, if the operation fails release the created resource */ + private[delta] def trySafely[R <: WritableColumnVector, T]( + useOffHeapBuffers: Boolean, + size: Int, + columns: Seq[ColumnMetadata])(f: Seq[WritableColumnVector] => T): T = { + val resources = new ArrayBuffer[WritableColumnVector](columns.size) + try { + columns.foreach(col => resources.append(newVector(useOffHeapBuffers, size, col.structField))) + f(resources.toSeq) + } catch { + case NonFatal(e) => + resources.foreach(closeQuietly(_)) + throw e + } + } + + /** Utility method to quietly close an [[AutoCloseable]] */ + private[delta] def closeQuietly(closeable: AutoCloseable): Unit = { + if (closeable != null) { + try { + closeable.close() + } catch { + case NonFatal(_) => // ignore + } + } + } + + /** + * Helper method to replace the vectors in given [[ColumnarBatch]]. + * New vectors and its index in the batch are given as tuples. + */ + private[delta] def replaceVectors( + batch: ColumnarBatch, + indexVectorTuples: (Int, ColumnVector) *): ColumnarBatch = { + val vectors = ArrayBuffer[ColumnVector]() + for (i <- 0 until batch.numCols()) { + var replaced: Boolean = false + for (indexVectorTuple <- indexVectorTuples) { + val index = indexVectorTuple._1 + val vector = indexVectorTuple._2 + if (indexVectorTuple._1 == i) { + vectors += indexVectorTuple._2 + // Make sure to close the existing vector allocated in the Parquet + batch.column(i).close() + replaced = true + } + } + if (!replaced) { + vectors += batch.column(i) + } + } + new ColumnarBatch(vectors.toArray, batch.numRows()) + } + + /** Helper class to encapsulate column info */ + case class ColumnMetadata(index: Int, structField: StructField) +} diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/SingleCommit.scala b/spark/src/main/scala/org/apache/spark/sql/delta/SingleCommit.scala index f9a445ad4ca..94267df6a4c 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/SingleCommit.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/SingleCommit.scala @@ -16,10 +16,10 @@ package org.apache.spark.sql.delta -import org.apache.spark.sql.delta.actions.Action +import org.apache.spark.sql.delta.actions.{Action, Checkpoint} import org.apache.spark.sql.delta.storage.{ClosableIterator, SupportsRewinding} import org.apache.spark.sql.delta.storage.ClosableIterator._ -import org.apache.hadoop.fs.{FileStatus, Path} +import org.apache.hadoop.fs.FileStatus /** * A handle to a single commit in the Delta log, standing in for the raw commit log file that @@ -44,19 +44,37 @@ class SingleCommit private ( SingleCommit.createRewindableActionIterator(deltaLog, fileStatus, maxInMemoryFileSizeBytes) /** - * The modification time of this commit's log file. Delta internal only. + * Like [[getActionsIterator]], but asserts this commit is a log commit. + * + * @param maxInMemoryFileSizeBytes see [[getActionsIterator]]. */ - private[delta] def fileModificationTimestamp: Long = fileStatus.getModificationTime + private[delta] def getLogCommitActionsIteratorUnsafe( + maxInMemoryFileSizeBytes: Long = 0) + : ClosableIterator[Action] with SupportsRewinding[Action] = { + val underlying = + SingleCommit.createRewindableActionIterator(deltaLog, fileStatus, maxInMemoryFileSizeBytes) + new ClosableIterator[Action] with SupportsRewinding[Action] { + override def hasNext: Boolean = underlying.hasNext + override def next(): Action = underlying.next() match { + case _: Checkpoint => + throw new IllegalStateException( + s"commit $version is a manifest commit and cannot be read as a log commit") + case action => action + } + override def close(): Unit = underlying.close() + override def rewind(): Unit = underlying.rewind() + } + } /** - * The size of this commit's log file, in bytes. Delta internal only. + * The modification time of this commit's log file. */ - private[delta] def sizeInBytes: Long = fileStatus.getLen + private[delta] def fileModificationTimestamp: Long = fileStatus.getModificationTime /** - * The commit log file's path. Should used by delta internal only. + * The size of this commit's log file, in bytes. */ - private[delta] def path: Path = fileStatus.getPath + private[delta] def commitFileSizeInBytes: Long = fileStatus.getLen } object SingleCommit { diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/Snapshot.scala b/spark/src/main/scala/org/apache/spark/sql/delta/Snapshot.scala index 379a1b0fbd6..aeff9868129 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/Snapshot.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/Snapshot.scala @@ -26,7 +26,7 @@ import scala.util.Try import com.databricks.spark.util.TagDefinition import org.apache.spark.sql.delta.actions._ import org.apache.spark.sql.delta.actions.Action.logSchema -import org.apache.spark.sql.delta.amt.{AMTCheckpointProvider, AMTUsageLogs} +import org.apache.spark.sql.delta.amt.{AMTCheckpointProvider, AMTUsageLogs, AMTUtils} import org.apache.spark.sql.delta.ClassicColumnConversions._ import org.apache.spark.sql.delta.coordinatedcommits.{CatalogOwnedTableUtils, CommitCoordinatorClient, CommitCoordinatorProvider, CoordinatedCommitsUsageLogs, CoordinatedCommitsUtils, TableCommitCoordinatorClient} import org.apache.spark.sql.delta.expressions.EncodeNestedVariantAsZ85String @@ -36,8 +36,10 @@ import org.apache.spark.sql.delta.schema.SchemaUtils import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.stats.DataSkippingReader import org.apache.spark.sql.delta.stats.DataSkippingReaderConf +import org.apache.spark.sql.delta.stats.DeletedRecordCountsHistogram import org.apache.spark.sql.delta.stats.DeltaStatsColumnSpec import org.apache.spark.sql.delta.stats.StatisticsCollection +import org.apache.spark.sql.delta.util.{Utils => DeltaUtils} import org.apache.spark.sql.delta.util.DeltaCommitFileProvider import org.apache.spark.sql.delta.util.FileNames import org.apache.spark.sql.delta.util.StateCache @@ -236,7 +238,7 @@ class Snapshot( /** Returns the lastManifestCommit reliably for this snapshot's version. */ lazy val lastManifestCommitOpt: Option[LastManifestCommit] = - Option.when(protocol.isFeatureSupported(AdaptiveMetadataTableFeature)) { + Option.when(AMTUtils.amtEnabled(this)) { _reconstructedProtocolMetadataICTAndLMC.bestEffortLastManifestCommit.orElse { // Only fallback to direct CommitInfo read if there are no trailing deltas. // This avoids unnecessary I/O when there isn't any AMT checkpoint yet. @@ -314,11 +316,12 @@ class Snapshot( * been backfilled. */ private[delta] def allCommitsBackfilled: Boolean = { - lastKnownBackfilledVersion >= FileNames.getFileVersion(logSegment.deltas.last) && - // This should always be true because we synchronously backfill during checkpoint - // creation and always create a new snapshot after that, which will force the - // latest LogSegment to be used. - lastKnownBackfilledVersion >= logSegment.checkpointProvider.version + // This should never happen, but we check anyway to be defensive. + if (lastKnownBackfilledVersion > this.version) { + throw new IllegalStateException("`lastKnownBackfilledVersion` should never be greater " + + "than the snapshot's version.") + } + lastKnownBackfilledVersion == this.version } /** @@ -473,6 +476,79 @@ class Snapshot( deltaLog.protocolRead(protocol) deltaLog.assertTableFeaturesMatchMetadata(protocol, metadata) SchemaUtils.recordUndefinedTypes(deltaLog, metadata.schema) + assertAMTInvariantsAtInit() + } + + /** + * Asserts AMT invariants at snapshot initialization. Only effective in testing. + */ + private def assertAMTInvariantsAtInit(): Unit = { + def toStrLMC(lastManifestCommit: LastManifestCommit): String = { + s"LastManifestCommit[version=${lastManifestCommit.version}, " + + s"contentRootVersion=${lastManifestCommit.contentRootVersion}]" + } + def toStrCP(amtCheckpointProvider: AMTCheckpointProvider): String = { + s"AMTCheckpointProvider[version=${amtCheckpointProvider.version}]" + } + + if (!DeltaUtils.isTesting) { + return + } + + val amtCheckpointProviderOpt = logSegment.checkpointProvider match { + case amtCheckpointProvider: AMTCheckpointProvider => + if (amtCheckpointProvider.version > version) { + throw new IllegalStateException( + s"${toStrCP(amtCheckpointProvider)} exceeds snapshot version $version.") + } + Some(amtCheckpointProvider) + case _ => None + } + + if (!AMTUtils.amtEnabled(this)) { + amtCheckpointProviderOpt.foreach { cp => + throw new IllegalStateException(s"${toStrCP(cp)} is present but AMT is disabled.") + } + return + } + + if (logSegment.nonCompactedDeltasOpt.isEmpty) { + throw new IllegalStateException( + s"An AMT-enabled snapshot must define nonCompactedDeltasOpt, got None.\n" + + s"${logSegment.toPrettyString}") + } + + if (amtCheckpointProviderOpt.isDefined && logSegment.deltaAtCheckpointVersionOpt.isEmpty) { + throw new IllegalStateException( + s"An AMT-enabled snapshot must define deltaAtCheckpointVersionOpt, got None.\n" + + s"${logSegment.toPrettyString}") + } + + (amtCheckpointProviderOpt, lastManifestCommitOpt) match { + // Normally, the AMT checkpoint provider's version matches the lastManifestCommit exactly. + // However, during time travel it may instead describe something newer, i.e. a later manifest + // commit not yet discoverable at the target version, carried forward from lastCheckpointInfo. + // Thus, we only throw when the AMT checkpoint provider is stale. + // For example, if a table has the following latest state: + // DeltaLog[commits v0~9, manifest-commit v10 (content root @v5), commits v11~20] + // Then `deltaLog.getSnapshotAt(version = 8)` could construct a LogSegment with: + // LogSegment[AMTCheckpointProvider(version = 5), deltas = commits v6~8] + case (Some(amtCp), Some(lmc)) => + if (lmc.contentRootVersion > amtCp.version) { + throw new IllegalStateException( + s"LastManifestCommit.contentRootVersion and AMTCheckpointProvider.version mismatch: " + + s"${toStrLMC(lmc)} vs ${toStrCP(amtCp)}") + } + // When that undiscoverable manifest commit is the first manifest commit, lastManifestCommit + // does not exist, so we will not throw here. + case (Some(amtCp), None) => () + // But when the lastManifestCommit is present but the AMT checkpoint provider is missing, + // that's a mismatch. + case (None, Some(lmc)) => + throw new IllegalStateException( + s"LastManifestCommit is present but AMTCheckpointProvider is missing. ${toStrLMC(lmc)}") + case (None, None) => () + } } /** The current set of actions in this [[Snapshot]] as plain Rows */ @@ -508,6 +584,10 @@ class Snapshot( override def protocol: Protocol = _reconstructedProtocolMetadataICTAndLMC.protocol + /** The row tracking high watermark assigned in this snapshot. */ + def getRowTrackingHighWaterMark(): Option[Long] = + RowId.RowTrackingMetadataDomain.fromSnapshot(this).map(_.rowIdHighWaterMark) + /** * Tries to retrieve the protocol, metadata, and in-commit-timestamp (if needed) from the * checksum file. If the checksum file is not present or if the protocol or metadata is missing @@ -600,6 +680,9 @@ class Snapshot( // for serializability val localMinFileRetentionTimestamp = minFileRetentionTimestamp val localMinSetTransactionRetentionTimestamp = minSetTransactionRetentionTimestamp + val localTableRoot = deltaLog.dataPath.toString + val localUseDeletionVectorObjectIdentity = + FileAction.useDeletionVectorObjectIdentity(metadata, protocol, spark) val canonicalPath = deltaLog.getCanonicalPathUdf() @@ -635,7 +718,8 @@ class Snapshot( col("add.baseRowId"), col("add.defaultRowCommitVersion"), col("add.clusteringProvider"), - col("add.backReference") + col("add.backReference"), + col("add.amtPassthrough") ))) .withColumn("remove", when( col("remove.path").isNotNull, @@ -645,7 +729,9 @@ class Snapshot( val state: LogReplay = new InMemoryLogReplay( Some(localMinFileRetentionTimestamp), - localMinSetTransactionRetentionTimestamp) + localMinSetTransactionRetentionTimestamp, + tableRoot = new Path(localTableRoot), + useDeletionVectorObjectIdentity = localUseDeletionVectorObjectIdentity) state.append(0, iter.map(_.unwrap)) state.checkpoint.map(_.wrap) } @@ -742,7 +828,7 @@ class Snapshot( checksumOpt.flatMap(_.fileSizeHistogram) .orElse(Option.when(_computedStateTriggered)(fileSizeHistogram).flatten) }.flatten, - lastManifestCommit = None + lastManifestCommit = lastManifestCommitOpt ) /** Returns the data schema of the table, used for reading stats */ @@ -771,16 +857,7 @@ class Snapshot( def redactedPath: String = Utils.redact(spark.sessionState.conf.stringRedactionPattern, path.toUri.toString) - /** - * Ensures that commit files are backfilled up to the current version in the snapshot. - * - * This method checks if there are any un-backfilled versions up to the current version and - * triggers the backfilling process using the commit-coordinator. It verifies that the delta file - * for the current version exists after the backfilling process. - * - * @throws IllegalStateException - * if the delta file for the current version is not found after backfilling. - */ + /** Ensures that commit files are backfilled up to the current version in the snapshot. */ def ensureCommitFilesBackfilled(catalogTableOpt: Option[CatalogTable]): Unit = { val tableCommitCoordinatorClientOpt = if (isCatalogOwned) { CatalogOwnedTableUtils.populateTableCommitCoordinatorFromCatalog(spark, catalogTableOpt, this) @@ -790,20 +867,12 @@ class Snapshot( val tableCommitCoordinatorClient = tableCommitCoordinatorClientOpt.getOrElse { return } - val minUnbackfilledVersion = DeltaCommitFileProvider(this).minUnbackfilledVersion - if (minUnbackfilledVersion <= version) { - val hadoopConf = deltaLog.newDeltaHadoopConf() - tableCommitCoordinatorClient.backfillToVersion( - catalogTableOpt.map(_.identifier), - version, - lastKnownBackfilledVersion = Some(minUnbackfilledVersion - 1)) - val fs = deltaLog.logPath.getFileSystem(hadoopConf) - val expectedBackfilledDeltaFile = FileNames.unsafeDeltaFile(deltaLog.logPath, version) - if (!fs.exists(expectedBackfilledDeltaFile)) { - throw new IllegalStateException("Backfilling of commit files failed. " + - s"Expected delta file $expectedBackfilledDeltaFile not found.") - } - } + CoordinatedCommitsUtils.ensureCommitFilesBackfilled( + version, + deltaLog, + tableCommitCoordinatorClient, + DeltaCommitFileProvider(this), + catalogTableOpt) } @@ -954,7 +1023,20 @@ object Snapshot extends DeltaLogging { * If the oldSnapshot itself is missing, we don't incrementally compute the checksum. */ private[delta] def shouldIncludeAddFilesInCrc( - spark: SparkSession, snapshot: Snapshot, metadata: Metadata): Boolean = { + spark: SparkSession, + snapshot: Snapshot, + metadata: Metadata, + effectiveLatestAMTCheckpointAtCommitVersion: Option[Checkpoint] = None): Boolean = { + // An AMT table may carry AddFiles in the incremental CRC only when the manifest tree that will + // back the resulting snapshot is ROOT-ONLY (no leaf manifests). This is because the incremental + // AddFile-generation logic cannot handle back references (yet): it writes allFiles into the CRC + // without them. Only leaf-resident files carry a back reference, so a root-only tree yields a + // back-reference-free list that matches state reconstruction. + if (AMTUtils.amtEnabled(snapshot)) { + val rootOnly = + effectiveLatestAMTCheckpointAtCommitVersion.exists(_.contentRoot.numLeaves.contains(0L)) + if (!rootOnly) return false + } allFilesInCrcWritePathEnabled(spark, snapshot) && (snapshot.version == -1 || snapshot.metadata.schema == metadata.schema) } @@ -1055,6 +1137,14 @@ class DummySnapshot( override def domainMetadata: Seq[DomainMetadata] = domainMetadataOpt.getOrElse(Seq.empty) override protected lazy val computedState: SnapshotState = initialState(metadata, protocol) + // The base DV accessors gate on DV readability of this snapshot's protocol, but a dummy snapshot + // seeds the incremental checksum of the first commit, which may enable DVs. Serve the + // already-computed initial state directly so that checksum starts from zeroed DV metrics instead + // of None, which would force a state reconstruction on the next commit. + override def numDeletedRecordsOpt: Option[Long] = computedState.numDeletedRecordsOpt + override def numDeletionVectorsOpt: Option[Long] = computedState.numDeletionVectorsOpt + override def deletedRecordCountsHistogramOpt: Option[DeletedRecordCountsHistogram] = + computedState.deletedRecordCountsHistogramOpt override protected[delta] lazy val getInCommitTimestampOpt: Option[Long] = None /* A dummy snapshot never has a manifest commit. */ override lazy val lastManifestCommitOpt: Option[LastManifestCommit] = None diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/SnapshotManagement.scala b/spark/src/main/scala/org/apache/spark/sql/delta/SnapshotManagement.scala index 61579722083..5fd80d99436 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/SnapshotManagement.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/SnapshotManagement.scala @@ -124,10 +124,18 @@ trait SnapshotManagement { self: DeltaLog => } catch { case _: FileNotFoundException => None } + val filterStagedCommits = spark.conf.get( + DeltaSQLConf.DELTA_SNAPSHOT_FILESYSTEM_LISTING_FILTER_STAGED_COMMITS_ENABLED) val files = filesOpt.map { _.flatMap { - case DeltaFile(f, fileVersion) => + // A recursive listing can return files from `_staged_commits`. A file in this directory is + // not a valid commit until the commit coordinator approves it. By default, accept only + // backfilled commit files from the listing. If filtering is disabled, accept all Delta + // commit file names as before. + case DeltaFile(f, fileVersion) if !filterStagedCommits => + Some((f, FileType.DELTA, fileVersion)) + case BackfilledDeltaFile(f, fileVersion) => Some((f, FileType.DELTA, fileVersion)) case CompactedDeltaFile(f, startVersion, endVersion) if includeMinorCompactions && versionToLoad.forall(endVersion <= _) => @@ -409,11 +417,13 @@ trait SnapshotManagement { self: DeltaLog => private def createLogSegment( previousSnapshot: Snapshot, catalogTableOpt: Option[CatalogTable], - commitCoordinatorOpt: Option[TableCommitCoordinatorClient]): Option[LogSegment] = { + commitCoordinatorOpt: Option[TableCommitCoordinatorClient], + lastCheckpointInfo: Option[LastCheckpointInfo]): Option[LogSegment] = { createLogSegment( oldCheckpointProviderOpt = Some(previousSnapshot.checkpointProvider), tableCommitCoordinatorClientOpt = commitCoordinatorOpt, - catalogTableOpt = catalogTableOpt) + catalogTableOpt = catalogTableOpt, + lastCheckpointInfo = lastCheckpointInfo) } /** @@ -437,7 +447,7 @@ trait SnapshotManagement { self: DeltaLog => * no checkpoint is selected. * @param versionToLoad - version for which we want to create the Snapshot. */ - private def validateDeltaVersions( + protected def validateDeltaVersions( selectedDeltas: Array[FileStatus], checkpointVersion: Long, versionToLoad: Option[Long]): Unit = { @@ -514,8 +524,12 @@ trait SnapshotManagement { self: DeltaLog => val (checkpoints, deltasAndCompactedDeltas) = newFiles.partition(isCheckpointFile) val (deltas, compactedDeltas) = deltasAndCompactedDeltas.partition(isDeltaFile) // Find the latest checkpoint in the listing that is not older than the versionToLoad - val checkpointFiles = checkpoints.map(f => CheckpointInstance(f.getPath)) - val newCheckpoint = getLatestCompleteCheckpointFromList(checkpointFiles, versionToLoad) + val fileBasedCheckpointInstances = checkpoints.map(f => CheckpointInstance(f.getPath)) + // An AMT checkpoint has no dedicated file in the listing; it is resolved from hint instead. + val amtCheckpointInstances = resolveAMTCheckpointInstances( + lastCheckpointInfo, oldCheckpointProviderOpt) + val checkpointInstances = fileBasedCheckpointInstances ++ amtCheckpointInstances + val newCheckpoint = getLatestCompleteCheckpointFromList(checkpointInstances, versionToLoad) val newCheckpointVersion = newCheckpoint.map(_.version).getOrElse { // If we do not have any checkpoint, pass new checkpoint version as -1 so that first // delta version can be 0. @@ -549,6 +563,9 @@ trait SnapshotManagement { self: DeltaLog => val deltasAfterCheckpoint = deltas.filter { file => deltaVersion(file) > newCheckpointVersion } + val deltaAtCheckpointVersionOpt = deltas.find { file => + deltaVersion(file) == newCheckpointVersion + } // Here we validate that we are able to create a valid LogSegment by just using commit deltas // and without considering minor-compacted deltas. We want to fail early if log is messed up @@ -596,17 +613,26 @@ trait SnapshotManagement { self: DeltaLog => checkpointVersion = newCheckpointVersion, versionToLoad = versionToLoad) + // Supply the non-compacted deltas only when the validation flag is true, which indicates that + // the deltas are continuous without gaps. + val nonCompactedDeltasOpt = Option.when(validateLogSegmentWithoutCompactedDeltas)( + deltasAfterCheckpoint.toSeq) + Some(LogSegment( logPath, newVersion, - deltasAndCompactedDeltasForLogSegment, - checkpointProviderOpt, + deltas = deltasAndCompactedDeltasForLogSegment, + nonCompactedDeltasOpt = nonCompactedDeltasOpt, + deltaAtCheckpointVersionOpt = deltaAtCheckpointVersionOpt, + checkpointProviderOpt = checkpointProviderOpt, lastCommitTimestamp)) } } /** - * @param deltasAndCompactedDeltas - all deltas or compacted deltas which could be used + * @param deltasAndCompactedDeltas - all deltas or compacted deltas which could be used, assumed + * to be sorted lexicographically by filename, e.g.: + * [v4.json, v5.v10.compacted.json, v5.json, v6.json, ...] * @param deltasAfterCheckpoint - deltas after the last checkpoint file * @param latestCommitVersion - commit version for which we are trying to create Snapshot for * @param checkpointVersionToUse - underlying checkpoint version to use in Snapshot, -1 if no @@ -625,7 +651,15 @@ trait SnapshotManagement { self: DeltaLog => val commitRangeCovered = mutable.ArrayBuffer.empty[Long] // track if there is at least 1 compacted delta in `deltasAndCompactedDeltas` var hasCompactedDeltas = false + // track if `deltasAndCompactedDeltas` is already sorted + var lastFileNameSeen = "" + var isDeltasAndCompactedDeltasSorted = true for (file <- deltasAndCompactedDeltas) { + if (file.getPath.getName < lastFileNameSeen) { + isDeltasAndCompactedDeltasSorted = false + } + lastFileNameSeen = file.getPath.getName + val (startVersion, endVersion) = file match { case CompactedDeltaFile(_, startVersion, endVersion) => hasCompactedDeltas = true @@ -636,6 +670,8 @@ trait SnapshotManagement { self: DeltaLog => // select the compacted delta if the startVersion doesn't straddle `highestVersionSeen` and // the endVersion doesn't cross the latestCommitVersion. + // When a compacted delta and a non-compacted delta have the same start version, we need the + // compacted delta to be evaluated first to prioritize selecting the compacted delta. if (highestVersionSeen < startVersion && endVersion <= latestCommitVersion) { commitRangeCovered.appendAll(startVersion to endVersion) selectedDeltas += file @@ -654,7 +690,9 @@ trait SnapshotManagement { self: DeltaLog => // either represented by compacted delta or by the delta. val requiredCommits = (checkpointVersionToUse + 1) to latestCommitVersion val missingCommits = requiredCommits.toSet -- coveredCommits - if (!hasDuplicates && missingCommits.isEmpty) return selectedDeltas.toArray + if (!hasDuplicates && missingCommits.isEmpty && isDeltasAndCompactedDeltasSorted) { + return selectedDeltas.toArray + } // If the above check failed, that means the compacted delta validation failed. // Just record that event and return just the deltas (deltasAfterCheckpoint). @@ -664,8 +702,15 @@ trait SnapshotManagement { self: DeltaLog => "latestCommitVersion" -> latestCommitVersion, "checkpointVersionToUse" -> checkpointVersionToUse, "hasDuplicates" -> hasDuplicates, - "missingCommits" -> missingCommits + "missingCommits" -> missingCommits, + "isDeltasAndCompactedDeltasSorted" -> isDeltasAndCompactedDeltasSorted ) + deltaAssert( + check = isDeltasAndCompactedDeltasSorted, + name = "v4amt.useCompactedDeltasForLogSegment.isDeltasAndCompactedDeltasSorted", + msg = s"Deltas and compacted deltas are not sorted. [${JsonUtils.toJson(eventData)}]", + deltaLog = this, + data = eventData) recordDeltaEvent( provider = this, opType = "delta.getLogSegmentForVersion.compactedDeltaValidationFailed", @@ -701,6 +746,7 @@ trait SnapshotManagement { self: DeltaLog => initialSegmentForNewSnapshot = initialSegmentForNewSnapshot, initialTableCommitCoordinatorClient = None, catalogTableOpt = initialCatalogTable, + lastCheckpointInfo = lastCheckpointOpt, isAsync = false) currentSnapshot = CapturedSnapshot(snapshot, snapshotInitWallclockTime) } @@ -740,7 +786,8 @@ trait SnapshotManagement { self: DeltaLog => initSegment: LogSegment, tableCommitCoordinatorClientOpt: Option[TableCommitCoordinatorClient], catalogTableOpt: Option[CatalogTable], - checksumOpt: Option[VersionChecksum]): Snapshot = { + checksumOpt: Option[VersionChecksum], + shouldReconcileAMTCheckpointProvider: Boolean = true): Snapshot = { val startingFrom = if (!initSegment.checkpointProvider.isEmpty) { log" starting from checkpoint version " + log"${MDC(DeltaLogKeys.START_VERSION, initSegment.checkpointProvider.version)}." @@ -749,13 +796,23 @@ trait SnapshotManagement { self: DeltaLog => log"Loading version ${MDC(DeltaLogKeys.VERSION, initSegment.version)}" + startingFrom) createSnapshotFromGivenOrEquivalentLogSegment( initSegment, tableCommitCoordinatorClientOpt, catalogTableOpt) { segment => + // Start reading the latest commit's CommitInfo (the fallback source of the last manifest + // commit reference, used when no CRC is available) in parallel with the CRC read below. + val lastCommitInfoFutureOpt = maybeReadLastCommitInfoAsync( + segment, shouldReconcileAMTCheckpointProvider) + val checksumOptAfterRead = checksumOpt.orElse( + readChecksum(segment.version, lastSeenChecksumFileStatusOpt)) + val finalLogSegment = if (shouldReconcileAMTCheckpointProvider) { + reconcileAMTCheckpointProvider(segment, checksumOptAfterRead, lastCommitInfoFutureOpt) + } else { + segment + } new Snapshot( path = logPath, version = segment.version, - logSegment = segment, + logSegment = finalLogSegment, deltaLog = this, - checksumOpt = checksumOpt.orElse( - readChecksum(segment.version, lastSeenChecksumFileStatusOpt)) + checksumOpt = checksumOptAfterRead ) } } @@ -826,9 +883,13 @@ trait SnapshotManagement { self: DeltaLog => Some(LogSegment( logPath, snapshotVersion, - deltas, - Some(checkpointProvider), - deltas.last.getModificationTime)) + deltas = deltasAfterCheckpoint, + // `deltasAfterCheckpoint` comes from a listing with `includeMinorCompactions = false`, + // so it must not contain compacted deltas. + nonCompactedDeltasOpt = Some(deltasAfterCheckpoint), + deltaAtCheckpointVersionOpt = deltas.find(deltaVersion(_) == cp.version), + checkpointProviderOpt = Some(checkpointProvider), + deltasAfterCheckpoint.last.getModificationTime)) case None => val listFromResult = listDeltaCompactedDeltaAndCheckpointFiles( @@ -859,6 +920,11 @@ trait SnapshotManagement { self: DeltaLog => logPath = logPath, version = snapshotVersion, deltas = deltas, + // `deltas` came from a listing with `includeMinorCompactions = false`, so it must not + // contain compacted deltas. + nonCompactedDeltasOpt = Some(deltas), + // No checkpoint here, so there is no delta at a checkpoint version to track. + deltaAtCheckpointVersionOpt = None, checkpointProviderOpt = None, lastCommitTimestamp = deltas.last.getModificationTime)) } @@ -876,7 +942,7 @@ trait SnapshotManagement { self: DeltaLog => tableCommitCoordinatorClientOpt: Option[TableCommitCoordinatorClient], catalogTableOpt: Option[CatalogTable], oldCheckpointProvider: CheckpointProvider, - amtCheckpointProviderOpt: Option[CheckpointProvider] = None + amtCheckpointProviderOpt: Option[AMTCheckpointProvider] = None ): LogSegment = recordFrameProfile( "Delta", "SnapshotManagement.getLogSegmentAfterCommit") { // If the table doesn't have any competing updates, then go ahead and use the optimized @@ -893,11 +959,7 @@ trait SnapshotManagement { self: DeltaLog => preCommitLogSegment, commit.getFileStatus, committedVersion) // The AMT manifest tree is authoritative for state up to its checkpoint version, so install // the provider and trim the segment's deltas to versions after it. - amtCheckpointProviderOpt.map { cp => - segment.copy( - checkpointProvider = cp, - deltas = segment.deltas.filter(f => deltaVersion(f) > cp.version)) - }.getOrElse(segment) + amtCheckpointProviderOpt.map(trimLogSegmentToAMTCheckpoint(segment, _)).getOrElse(segment) } else { val latestCheckpointProvider = Seq(preCommitLogSegment.checkpointProvider, oldCheckpointProvider).maxBy(_.version) @@ -1015,16 +1077,34 @@ trait SnapshotManagement { self: DeltaLog => // the Delta log. return (oldLogSegment, Nil) } + // Pass only checkpoint files that follow the deterministic checkpoint file naming pattern to + // getLogSegmentForVersion: every file in its `files` argument must be a FileNames + // isCheckpointFile / isCompactedDeltaFile / isDeltaFile, otherwise it fails classifying it. + val checkpointTopLevelFiles = + oldLogSegment.checkpointProvider.topLevelFiles.filter(isCheckpointFile) + // Collect the compacted and non-compacted deltas from the old log segment. The result should be + // identical to a listing within this window, which is sorted by the file name. + // In `getLogSegmentForVersion`, `oldLogSegment.deltaAtCheckpointVersionOpt` will be filtered + // out from the final field `deltas` of the returned segment, so the LogSegment structure is + // still maintained (checkpoint + deltas **after** checkpoint). It still needs to be supplied as + // part of `allFiles` though, because it needs to be reused as the `deltaAtCheckpointVersionOpt` + // of the returned segment when the checkpoint provider remains unchanged. + val deltasAndCompactedDeltas = oldLogSegment.deltaAtCheckpointVersionOpt.toSeq ++ + (oldLogSegment.deltas ++ oldLogSegment.nonCompactedDeltasOpt.getOrElse(Seq.empty)) + .distinct + .sortBy(_.getPath.getName) val allFiles = ( - oldLogSegment.checkpointProvider.topLevelFiles ++ - oldLogSegment.deltas ++ + checkpointTopLevelFiles ++ + deltasAndCompactedDeltas ++ newFiles ).toArray val lastCheckpointInfo = Option.empty[LastCheckpointInfo] val newLogSegment = getLogSegmentForVersion( versionToLoad = None, files = Some(allFiles), - validateLogSegmentWithoutCompactedDeltas = false, + // If the old segment's non-compacted deltas are defined, the deltas in `allFiles` will be + // continuous without gaps, and hence validatable. + validateLogSegmentWithoutCompactedDeltas = oldLogSegment.nonCompactedDeltasOpt.isDefined, tableCommitCoordinatorClientOpt = tableCommitCoordinatorClientOpt, catalogTableOpt = catalogTableOpt, lastCheckpointInfo = lastCheckpointInfo, @@ -1079,10 +1159,6 @@ trait SnapshotManagement { self: DeltaLog => // that there's no chance of a race condition changing the snapshot partway through the update. val capturedSnapshot = currentSnapshot val oldVersion = capturedSnapshot.snapshot.version - // TODO: implement deltaLog.update() rediscovery for AMT tables. - if (capturedSnapshot.snapshot.protocol.isFeatureSupported(AdaptiveMetadataTableFeature)) { - return capturedSnapshot.snapshot - } def sendEvent( newSnapshot: Snapshot, snapshotAlreadyUpdatedAfterRequiredTimestamp: Boolean = false @@ -1177,12 +1253,14 @@ trait SnapshotManagement { self: DeltaLog => val segmentOpt = createLogSegment( previousSnapshot, catalogTableOpt, - commitCoordinatorOpt) + commitCoordinatorOpt, + lastCheckpointInfo = None) val newSnapshot = getUpdatedSnapshot( oldSnapshotOpt = Some(previousSnapshot), initialSegmentForNewSnapshot = segmentOpt, initialTableCommitCoordinatorClient = commitCoordinatorOpt, catalogTableOpt = catalogTableOpt, + lastCheckpointInfo = None, isAsync = isAsync) installSnapshot(newSnapshot, updateStartTimeMs) } @@ -1204,6 +1282,7 @@ trait SnapshotManagement { self: DeltaLog => initialSegmentForNewSnapshot: Option[LogSegment], initialTableCommitCoordinatorClient: Option[TableCommitCoordinatorClient], catalogTableOpt: Option[CatalogTable], + lastCheckpointInfo: Option[LastCheckpointInfo], isAsync: Boolean): Snapshot = { var newSnapshot = getSnapshotForLogSegment( oldSnapshotOpt, @@ -1232,7 +1311,8 @@ trait SnapshotManagement { self: DeltaLog => val segmentOpt = createLogSegment( newSnapshot, catalogTableOpt, - commitCoordinatorOpt) + commitCoordinatorOpt, + lastCheckpointInfo) newSnapshot = getSnapshotForLogSegment( Some(newSnapshot), segmentOpt, @@ -1379,7 +1459,8 @@ trait SnapshotManagement { self: DeltaLog => newChecksumOpt: Option[VersionChecksum], tableCommitCoordinatorClientOpt: Option[TableCommitCoordinatorClient], catalogTableOpt: Option[CatalogTable], - committedVersion: Long): Snapshot = { + committedVersion: Long, + shouldReconcileAMTCheckpointProvider: Boolean): Snapshot = { logInfo( log"Creating a new snapshot v${MDC(DeltaLogKeys.VERSION, initSegment.version)} " + log"for commit version ${MDC(DeltaLogKeys.VERSION2, committedVersion)}") @@ -1399,7 +1480,9 @@ trait SnapshotManagement { self: DeltaLog => initSegment, tableCommitCoordinatorClientOpt, catalogTableOpt, - checksumOpt) + checksumOpt, + // The AMT checkpoint provider installed here comes from the commit and is authoritative. + shouldReconcileAMTCheckpointProvider) } var newSnapshot = createSnapshotWithCrc(snapChecksumOpt) @@ -1469,7 +1552,7 @@ trait SnapshotManagement { self: DeltaLog => newChecksumOpt: Option[VersionChecksum], preCommitLogSegment: LogSegment, catalogTableOpt: Option[CatalogTable], - amtCheckpointOpt: Option[Checkpoint] = None, + amtCheckpointWrittenInCommitOpt: Option[Checkpoint] = None, isIdempotentRetry: Boolean = false): Snapshot = { var previousSnapshot: Snapshot = null recordDeltaOperation(this, "delta.log.updateAfterCommit") { @@ -1481,8 +1564,8 @@ trait SnapshotManagement { self: DeltaLog => val commitCoordinatorOpt = populateCommitCoordinator( spark, catalogTableOpt, previousSnapshot ) - val amtCheckpointProviderOpt = - amtCheckpointOpt.map(cp => AMTCheckpointProvider.fromCheckpoint(spark, this, cp)) + val amtCheckpointProviderOpt = amtCheckpointWrittenInCommitOpt.map( + AMTCheckpointProvider.fromCheckpoint(this, _, committedVersion)) val segment = if (isIdempotentRetry) { // The commit already landed and the preCommitLogSegment has been advanced to a // segment at >= committedVersion by conflict checking, so it is already the @@ -1493,7 +1576,12 @@ trait SnapshotManagement { self: DeltaLog => throw new IllegalStateException( "A Commit is required to build the post-commit log segment.") } - getLogSegmentAfterCommit( + // The listing can be stale for a few seconds after a commit due to an + // incident on the cloud provider side. Retry the operation a few times. + val maxRetries = + spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_COMMIT_INCONSISTENT_LIST_MAX_RETRIES) + var attempt = 0 + def fetchSegment(): LogSegment = getLogSegmentAfterCommit( committedVersion, newChecksumOpt, preCommitLogSegment, @@ -1502,23 +1590,45 @@ trait SnapshotManagement { self: DeltaLog => catalogTableOpt, previousSnapshot.checkpointProvider, amtCheckpointProviderOpt = amtCheckpointProviderOpt) + var fetched = fetchSegment() + while (attempt < maxRetries && fetched.version < committedVersion) { + val backoffMs = math.min(30.seconds.toMillis, 1000L << attempt) + logWarning(log"Stale log segment after commit: listed version " + + log"${MDC(DeltaLogKeys.VERSION, fetched.version)} < committed version " + + log"${MDC(DeltaLogKeys.VERSION2, committedVersion)}.") + Thread.sleep(backoffMs) + attempt += 1 + fetched = fetchSegment() + } + if (attempt > 0 && fetched.version >= committedVersion) { + recordDeltaEvent(this, "delta.commit.mitigatedInconsistentList", data = Map( + "numRetries" -> attempt, + "committedVersion" -> committedVersion, + "currentVersion" -> fetched.version + )) + } + fetched } // This likely implies a list-after-write inconsistency if (segment.version < committedVersion) { - recordDeltaEvent(this, "delta.commit.inconsistentList", data = Map( + recordDeltaEvent(this, "delta.assertions.commit.inconsistentList", data = Map( "committedVersion" -> committedVersion, "currentVersion" -> segment.version )) throw DeltaErrors.invalidCommittedVersion(committedVersion, segment.version) } + // We only trust the post-commit segment if we've written the AMT checkpoint in this commit. + val shouldReconcileAMTCheckpointProvider = amtCheckpointWrittenInCommitOpt.isEmpty && + previousSnapshot.protocol.isFeatureSupported(AdaptiveMetadataTableFeature) val newSnapshot = createSnapshotAfterCommit( segment, newChecksumOpt, commitCoordinatorOpt, catalogTableOpt, - committedVersion) + committedVersion, + shouldReconcileAMTCheckpointProvider) installSnapshot(newSnapshot, updateTimestamp) } logMetadataTableIdChange(previousSnapshot, updatedSnapshot) @@ -1679,9 +1789,12 @@ trait SnapshotManagement { self: DeltaLog => } object SnapshotManagement extends DeltaLogging { - // A thread pool for reading checkpoint files and collecting checkpoint v2 actions like + // A thread pool for checkpoint resolution. + // For v2 checkpoints, we use it to read the checkpoint files and collect the v2 actions like // checkpointMetadata, sidecarFiles. - private[delta] lazy val checkpointV2ThreadPool = { + // For AMT checkpoints, we use it to read the latest commit's CommitInfo during snapshot + // construction, as a fallback source when CRC is absent. + private[delta] lazy val checkpointThreadPool = { val numThreads = SparkSession.active.sessionState.conf.getConf( DeltaSQLConf.CHECKPOINT_V2_DRIVER_THREADPOOL_PARALLELISM) DeltaThreadPool("checkpointV2-threadpool", numThreads) @@ -1762,6 +1875,7 @@ object SnapshotManagement extends DeltaLogging { oldLogSegment.copy( version = committedVersion, deltas = oldLogSegment.deltas :+ commitFileStatus, + nonCompactedDeltasOpt = oldLogSegment.nonCompactedDeltasOpt.map(_ :+ commitFileStatus), lastCommitFileModificationTimestamp = commitFileStatus.getModificationTime) } } @@ -1810,7 +1924,13 @@ object SerializableFileStatus { * * @param logPath The path to the _delta_log directory * @param version The Snapshot version to generate - * @param deltas The delta commit files (.json) to read + * @param deltas The delta commit files (.json) to read; may include compacted deltas. + * @param nonCompactedDeltasOpt The full list of non-compacted delta commit files, if known. When + * provided, it must represent all the individual delta files in range + * [checkpointProvider.version + 1, version]. + * @param deltaAtCheckpointVersionOpt The commit file at checkpoint version, maybe unbackfilled + * for AMT tables. Tracked separately for filename resolution. + * Absent when the checkpoint provider is empty with version -1. * @param checkpointProvider provider to give information about Checkpoint files. * @param lastCommitFileModificationTimestamp The "unadjusted" file modification timestamp of the * last commit within this segment. By unadjusted, we mean that the commit timestamps may @@ -1820,9 +1940,42 @@ case class LogSegment( logPath: Path, version: Long, deltas: Seq[FileStatus], + nonCompactedDeltasOpt: Option[Seq[FileStatus]], + deltaAtCheckpointVersionOpt: Option[FileStatus], checkpointProvider: UninitializedCheckpointProvider, lastCommitFileModificationTimestamp: Long) { + if (DeltaUtils.isTesting) { + nonCompactedDeltasOpt.foreach { nonCompactedDeltas => + if (nonCompactedDeltas.exists(!isDeltaFile(_))) { + throw new IllegalArgumentException( + "nonCompactedDeltasOpt must contain only non-compacted delta files, but got " + + s"${nonCompactedDeltas.map(_.getPath.getName)}.") + } + val actualVersions = nonCompactedDeltas.map(deltaVersion).toList + val expectedVersions = ((checkpointProvider.version + 1) to version).toList + if (actualVersions != expectedVersions) { + throw new IllegalArgumentException( + s"Expected nonCompactedDeltasOpt to be $expectedVersions, but got $actualVersions.") + } + } + deltaAtCheckpointVersionOpt.foreach { delta => + if (checkpointProvider.version == -1) { + throw new IllegalArgumentException( + s"deltaAtCheckpointVersionOpt must be None for empty checkpoint, but got $delta.") + } + if (!isDeltaFile(delta)) { + throw new IllegalArgumentException( + s"deltaAtCheckpointVersionOpt must be a delta file, but got ${delta}.") + } + if (deltaVersion(delta) != checkpointProvider.version) { + throw new IllegalArgumentException( + s"deltaAtCheckpointVersionOpt must correspond to the checkpoint version " + + s"${checkpointProvider.version}, but got $delta.") + } + } + } + override def hashCode(): Int = logPath.hashCode() * 31 + (lastCommitFileModificationTimestamp % 10000).toInt @@ -1861,12 +2014,86 @@ case class LogSegment( } } - private[delta] lazy val lastBackfilledVersionInSegment = + private[delta] lazy val lastBackfilledVersionInSegment: Long = // This works if the last backfilled file is a minor-compaction, because // FileNames.getFileVersion returns the minor-compaction end version, // which correctly initializes the lastBackfilledVersionInSegment. CoordinatedCommitsUtils.getLastBackfilledFile(deltas).map(getFileVersion) - .getOrElse(checkpointProvider.version) + // All files are unbackfilled after the checkpoint version. For AMT tables, the file at the + // checkpoint version may or may not have been backfilled, hence the following match cases. + .getOrElse { + (checkpointProvider.version, deltaAtCheckpointVersionOpt) match { + // `deltaAtCheckpointVersionOpt` can only be None when the checkpoint provider is empty + // with version -1, and no corresponding delta file exists. + // The last backfilled version is still the checkpoint version. + case (cpVersion, None) => cpVersion + + case (-1, Some(delta)) => + throw new IllegalStateException(s"Unexpected delta file for empty checkpoint: $delta.") + + case (0, Some(delta)) if isUnbackfilledDeltaFile(delta) => + // Commit-0 must go through filesystem and can never be unbackfilled. + throw new IllegalStateException(s"Unexpected unbackfilled delta file at v0: $delta.") + + case (cpVersion, Some(delta)) if isBackfilledDeltaFile(delta) => cpVersion + case (cpVersion, Some(delta)) if isUnbackfilledDeltaFile(delta) => cpVersion - 1 + + case (cpVersion, Some(file)) => + throw new IllegalStateException( + s"Unexpected deltaAtCheckpointVersionOpt for checkpoint version $cpVersion: $file." + ) + } + } + + /** Returns the unbackfilled commits in the [[LogSegment]] */ + lazy val unbackfilledDeltas: Seq[(FileStatus, Long, String)] = + // If `deltas` contains a compacted delta [X...Y], then every commit up to Y must be backfilled. + // (guaranteed by minor-compaction hook), so collecting from `deltas` is sufficient. + // If `deltas` contains no compacted deltas, then for AMT tables, `deltaAtCheckpointVersionOpt` + // (must be defined for AMT tables) might still be unbackfilled, so we collect from it as well. + (deltaAtCheckpointVersionOpt.toSeq ++ deltas).collect { + case UnbackfilledDeltaFile(file, version, uuid) => (file, version, uuid) + } + + // Override the case-class toString so logging a LogSegment cannot OOM the driver: the generated + // toString expands the full deltas / nonCompactedDeltasOpt collections, which can be very large. + // The per-element rendering matches the generated toString (each file's FileStatus); only the + // number of files is capped. See [[LogSegment.truncatedDeltaString]]. + override def toString: String = { + def renderDeltas(files: Seq[FileStatus]): String = + LogSegment.truncatedDeltaString(files)(_.toString) + val nonCompactedDeltasStr = + nonCompactedDeltasOpt.map(files => s"Some(${renderDeltas(files)})").getOrElse("None") + s"LogSegment($logPath, $version, ${renderDeltas(deltas)}, $nonCompactedDeltasStr, " + + s"$checkpointProvider, $lastCommitFileModificationTimestamp)" + } + + def toPrettyString: String = { + def renderDelta(file: FileStatus): String = { + file match { + case CompactedDeltaFile(_, startVersion, endVersion) => + s"<${(startVersion to endVersion).mkString(", ")}>" + case delta => + deltaVersion(delta).toString + } + } + // E.g., "[1, 2, <3, 4, 5>, 6]". + def renderDeltas(files: Seq[FileStatus]): String = { + files.map(renderDelta).mkString("[", ", ", "]") + } + val deltaAtCpStr = deltaAtCheckpointVersionOpt.map(renderDelta).getOrElse("None") + s""" + |LogSegment( + | logPath = $logPath, + | version = $version, + | deltas = ${renderDeltas(deltas)}, + | nonCompactedDeltasOpt = ${nonCompactedDeltasOpt.map(renderDeltas).getOrElse("None")}, + | deltaAtCheckpointVersionOpt = $deltaAtCpStr, + | checkpointProvider = ${checkpointProvider.version}, + | lastCommitFileModificationTimestamp = $lastCommitFileModificationTimestamp + |) + """.stripMargin + } } /** Exception thrown When [[TableCommitCoordinatorClient.getCommits]] fails due to any reason. */ @@ -1878,10 +2105,19 @@ object LogSegment { logPath: Path, version: Long, deltas: Seq[FileStatus], + nonCompactedDeltasOpt: Option[Seq[FileStatus]], + deltaAtCheckpointVersionOpt: Option[FileStatus], checkpointProviderOpt: Option[UninitializedCheckpointProvider], lastCommitTimestamp: Long): LogSegment = { val checkpointProvider = checkpointProviderOpt.getOrElse(EmptyCheckpointProvider) - LogSegment(logPath, version, deltas, checkpointProvider, lastCommitTimestamp) + LogSegment( + logPath, + version, + deltas, + nonCompactedDeltasOpt, + deltaAtCheckpointVersionOpt, + checkpointProvider, + lastCommitTimestamp) } /** The LogSegment for an empty transaction log directory. */ @@ -1889,6 +2125,30 @@ object LogSegment { logPath = path, version = -1L, deltas = Nil, + nonCompactedDeltasOpt = Some(Nil), + deltaAtCheckpointVersionOpt = None, checkpointProviderOpt = None, lastCommitTimestamp = -1L) + + /** + * Renders `elems` for a log/error message, keeping at most + * [[DeltaSQLConf.DELTA_LOG_SEGMENT_DELTAS_TO_STRING_LIMIT]] entries. Rendering every element of a + * very large collection can materialize a huge string and OOM the driver, so only the retained + * entries are stringified; this is safe to call on an arbitrarily large collection. + */ + private[delta] def truncatedDeltaString[T](elems: Seq[T])(render: T => String): String = { + val truncationThresholdKey = DeltaSQLConf.DELTA_LOG_SEGMENT_DELTAS_TO_STRING_LIMIT + val truncationThreshold = SparkSession.getActiveSession + .map(_.sessionState.conf.getConf(truncationThresholdKey)) + .getOrElse(truncationThresholdKey.defaultValue.get) + val size = elems.size + // -1 (or any negative value) disables truncation; otherwise the threshold is below `size` + // here, so it fits in an Int for `take`. + if (truncationThreshold >= 0 && size > truncationThreshold) { + elems.iterator.take(truncationThreshold.toInt).map(render) + .mkString("[", ", ", s", ... (${size - truncationThreshold} of $size deltas omitted)]") + } else { + elems.iterator.map(render).mkString("[", ", ", "]") + } + } } diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/SnapshotState.scala b/spark/src/main/scala/org/apache/spark/sql/delta/SnapshotState.scala index e113762e251..cd8d567a1c4 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/SnapshotState.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/SnapshotState.scala @@ -83,9 +83,36 @@ trait SnapshotStateManager extends DeltaLogging { self: Snapshot => protected def fileSizeHistogramEnabled: Boolean = spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_FILE_SIZE_HISTOGRAM_ENABLED) + protected def deletedRecordCountsHistogramEnabled: Boolean = + spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_DELETED_RECORD_COUNTS_HISTOGRAM_ENABLED) + + /** + * Whether [[aggregationsToComputeState]] computes the deletion vector metrics + * (`numDeletedRecordsOpt` / `numDeletionVectorsOpt`, and with + * [[deletedRecordCountsHistogramEnabled]] also `deletedRecordCountsHistogramOpt`); when it does + * not, those aggregates are `null` and the corresponding fields end up as [[None]]. This is the + * *writable* predicate; the DV accessors instead gate their reads on the *readable* predicate + * ([[deletionVectorsReadableAndMetricsEnabled]]). + */ + protected def checksumDVMetricsComputed: Boolean = + spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_CHECKSUM_DV_METRICS_ENABLED) && + DeletionVectorUtils.deletionVectorsWritable(this) + /** Whether computedState is already computed or not */ @volatile protected var _computedStateTriggered: Boolean = false + /** Set when accessing computedState, recording which accessor forced the reconstruction. */ + @volatile protected var lastComputedStateAccessor: String = "" + + /** + * Runs `thunk`, the access to [[computedState]] that `accessor` is forcing, recording + * `lastComputedStateAccessor` for logging. All callers that reach [[computedState]] should go + * through this method. + */ + protected def recordComputedStateAccess[T](accessor: String)(thunk: => T): T = { + lastComputedStateAccessor = accessor + thunk + } /** A map to look up transaction version by appId. */ lazy val transactions: Map[String, Long] = setTransactions.map(t => t.appId -> t.version).toMap @@ -157,15 +184,7 @@ trait SnapshotStateManager extends DeltaLogging { self: Snapshot => * A Map of alias to aggregations which needs to be done to calculate the `computedState` */ protected def aggregationsToComputeState: Map[String, Column] = { - val checksumDVMetricsEnabled = - spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_CHECKSUM_DV_METRICS_ENABLED) - val deletedRecordCountsHistogramEnabled = - spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_DELETED_RECORD_COUNTS_HISTOGRAM_ENABLED) - lazy val persistentDVsOnTableSupported = - DeletionVectorUtils.deletionVectorsWritable(this) - - val computeChecksumDVMetrics = checksumDVMetricsEnabled && - persistentDVsOnTableSupported + val computeChecksumDVMetrics = checksumDVMetricsComputed val persistentDVsAggs = if (computeChecksumDVMetrics) { Map( @@ -206,26 +225,154 @@ trait SnapshotStateManager extends DeltaLogging { self: Snapshot => ) ++ persistentDVsAggs ++ histogramDVsAgg } + /** + * The checksum this snapshot serves its state fields from, when present. This is the snapshot's + * own checksum, gated on [[DeltaSQLConf.FAST_QUERY_PATH_ENABLED]]. + */ + protected def checksumOptForState: Option[VersionChecksum] = + if (spark.sessionState.conf.getConf(DeltaSQLConf.FAST_QUERY_PATH_ENABLED)) checksumOpt + else None + /** * The following is a list of convenience methods for accessing the computedState. + * + * Each of them first tries to answer from [[checksumOptForState]] and only falls back to + * [[computedState]] -- which triggers the expensive aggregation over the state reconstruction -- + * when the checksum cannot serve the value. [[recordComputedStateAccess]] records which accessor + * forced the fallback. + */ + def sizeInBytes: Long = + checksumOptForState.map(_.tableSizeBytes).getOrElse { + recordComputedStateAccess("sizeInBytes") { computedState.sizeInBytes } + } + def numOfSetTransactions: Long = + setTransactionsIfKnown.map(_.size.toLong).getOrElse { + recordComputedStateAccess("numOfSetTransactions") { computedState.numOfSetTransactions } + } + def numOfFiles: Long = + checksumOptForState.map(_.numFiles).getOrElse { + recordComputedStateAccess("numOfFiles") { computedState.numOfFiles } + } + // The number of tombstones is not tracked by the checksum, so this always needs the state + // reconstruction. + def numOfRemoves: Long = + recordComputedStateAccess("numOfRemoves") { computedState.numOfRemoves } + def numOfMetadata: Long = + checksumOptForState.map(_.numMetadata).getOrElse { + recordComputedStateAccess("numOfMetadata") { computedState.numOfMetadata } + } + def numOfProtocol: Long = + checksumOptForState.map(_.numProtocol).getOrElse { + recordComputedStateAccess("numOfProtocol") { computedState.numOfProtocol } + } + def setTransactions: Seq[SetTransaction] = + setTransactionsIfKnown.getOrElse { + recordComputedStateAccess("setTransactions") { computedState.setTransactions } + } + def fileSizeHistogram: Option[FileSizeHistogram] = + if (fileSizeHistogramEnabled) { + checksumOptForState.flatMap(_.fileSizeHistogram).orElse { + recordComputedStateAccess("fileSizeHistogram") { computedState.fileSizeHistogram } + } + } else None + def domainMetadata: Seq[DomainMetadata] = + domainMetadatasIfKnown.getOrElse { + recordComputedStateAccess("domainMetadata") { computedState.domainMetadata } + } + + /** + * Returns the table size in bytes if it is cheaply available from the checksum or from + * already-computed state, and [[None]] if answering would require state reconstruction. */ - def sizeInBytes: Long = computedState.sizeInBytes - def numOfSetTransactions: Long = computedState.numOfSetTransactions - def numOfFiles: Long = computedState.numOfFiles - def numOfRemoves: Long = computedState.numOfRemoves - def numOfMetadata: Long = computedState.numOfMetadata - def numOfProtocol: Long = computedState.numOfProtocol - def setTransactions: Seq[SetTransaction] = computedState.setTransactions - def fileSizeHistogram: Option[FileSizeHistogram] = computedState.fileSizeHistogram - def domainMetadata: Seq[DomainMetadata] = computedState.domainMetadata - protected[delta] def sizeInBytesIfKnown: Option[Long] = Some(sizeInBytes) - protected[delta] def setTransactionsIfKnown: Option[Seq[SetTransaction]] = Some(setTransactions) - protected[delta] def numOfFilesIfKnown: Option[Long] = Some(numOfFiles) - protected[delta] def domainMetadatasIfKnown: Option[Seq[DomainMetadata]] = Some(domainMetadata) - def numDeletedRecordsOpt: Option[Long] = computedState.numDeletedRecordsOpt - def numDeletionVectorsOpt: Option[Long] = computedState.numDeletionVectorsOpt - def deletedRecordCountsHistogramOpt: Option[DeletedRecordCountsHistogram] = - computedState.deletedRecordCountsHistogramOpt + protected[delta] def sizeInBytesIfKnown: Option[Long] = + getFieldFromVersionChecksumIfKnown(c => Some(c.tableSizeBytes), sizeInBytes) + + /** + * Returns the number of files if it is cheaply available from the checksum or from + * already-computed state, and [[None]] if answering would require state reconstruction. + */ + protected[delta] def numOfFilesIfKnown: Option[Long] = + getFieldFromVersionChecksumIfKnown(c => Some(c.numFiles), numOfFiles) + + /** + * Returns the [[SetTransaction]]s if they are already pre-computed or available via the + * checksum, and [[None]] if answering would require state reconstruction. + */ + protected[delta] def setTransactionsIfKnown: Option[Seq[SetTransaction]] = + checksumOptForState + .filter(_ => spark.conf.get(DeltaSQLConf.DELTA_READ_SET_TRANSACTIONS_FROM_CRC)) + .flatMap(_.setTransactions) + .map { setTransactionActions => + recordDeltaEvent(deltaLog, "delta.snapshot.setTransactions.viaCRC") + setTransactionActions + } + .orElse { if (_computedStateTriggered) Some(computedState.setTransactions) else None } + + /** + * Returns the [[DomainMetadata]]s if they are already pre-computed or available via the + * checksum, and [[None]] if answering would require state reconstruction. + */ + protected[delta] def domainMetadatasIfKnown: Option[Seq[DomainMetadata]] = + metadataDomainFromChecksumOpt.orElse { + if (_computedStateTriggered) Some(computedState.domainMetadata) else None + } + + /** The [[DomainMetadata]]s carried by [[checksumOptForState]], if any. */ + private lazy val metadataDomainFromChecksumOpt: Option[Seq[DomainMetadata]] = + checksumOptForState + .flatMap(_.domainMetadata) + .map { domainMetadata => + recordDeltaEvent(deltaLog, "delta.snapshot.domainMetadata.viaCRC") + domainMetadata + } + + // For DV metrics (numDeletedRecordsOpt / numDeletionVectorsOpt / deletedRecordCountsHistogramOpt) + // we only return a value for tables where DVs and DV metrics are readable, and fall back to a + // state recomputation when the checksum cannot serve them. The "IfKnown" getters only return a + // value when the answer is available without a fresh state reconstruction. + def numDeletedRecordsOpt: Option[Long] = { + if (!deletionVectorsReadableAndMetricsEnabled) return None + checksumOptForState.flatMap(_.numDeletedRecordsOpt).orElse { + recordComputedStateAccess("numDeletedRecordsOpt") { computedState.numDeletedRecordsOpt } + } + } + def numDeletionVectorsOpt: Option[Long] = { + if (!deletionVectorsReadableAndMetricsEnabled) return None + checksumOptForState.flatMap(_.numDeletionVectorsOpt).orElse { + recordComputedStateAccess("numDeletionVectorsOpt") { computedState.numDeletionVectorsOpt } + } + } + def deletedRecordCountsHistogramOpt: Option[DeletedRecordCountsHistogram] = { + if (!deletionVectorsReadableAndHistogramEnabled) return None + checksumOptForState.flatMap(_.deletedRecordCountsHistogramOpt).orElse { + recordComputedStateAccess("deletedRecordCountsHistogramOpt") { + computedState.deletedRecordCountsHistogramOpt + } + } + } + + /** + * Returns `Some(fromState)` when the field is known without a fresh state reconstruction, else + * `None`. It is known when [[checksumOptForState]] carries the field (`fromChecksum` is defined) + * or the state is already computed; only the presence of `fromChecksum` is used -- the value, and + * any of its own gating (e.g. DV readability), come from `fromState`. + */ + protected def getFieldFromVersionChecksumIfKnown[T]( + fromChecksum: VersionChecksum => Option[Any], + fromState: => T): Option[T] = + if (checksumOptForState.flatMap(fromChecksum).isEmpty && !_computedStateTriggered) None + else Some(fromState) + + protected[delta] def numDeletedRecordsOptIfKnown: Option[Option[Long]] = + getFieldFromVersionChecksumIfKnown(_.numDeletedRecordsOpt, numDeletedRecordsOpt) + + protected[delta] def numDeletionVectorsOptIfKnown: Option[Option[Long]] = + getFieldFromVersionChecksumIfKnown(_.numDeletionVectorsOpt, numDeletionVectorsOpt) + + protected[delta] def deletedRecordCountsHistogramOptIfKnown: + Option[Option[DeletedRecordCountsHistogram]] = + getFieldFromVersionChecksumIfKnown( + _.deletedRecordCountsHistogramOpt, deletedRecordCountsHistogramOpt) protected def deletionVectorsReadableAndMetricsEnabled: Boolean = { val checksumDVMetricsEnabled = @@ -235,8 +382,6 @@ trait SnapshotStateManager extends DeltaLogging { self: Snapshot => } protected def deletionVectorsReadableAndHistogramEnabled: Boolean = { - val deletedRecordCountsHistogramEnabled = - spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_DELETED_RECORD_COUNTS_HISTOGRAM_ENABLED) deletionVectorsReadableAndMetricsEnabled && deletedRecordCountsHistogramEnabled } diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/TableFeature.scala b/spark/src/main/scala/org/apache/spark/sql/delta/TableFeature.scala index f28b4e42e76..39ed8bee571 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/TableFeature.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/TableFeature.scala @@ -398,6 +398,7 @@ object TableFeature { VariantTypeTableFeature, VariantShreddingPreviewTableFeature, VariantShreddingTableFeature, + FileTypePreviewTableFeature, CatalogOwnedTableFeature, CoordinatedCommitsTableFeature, CheckpointProtectionTableFeature) @@ -874,6 +875,38 @@ object VariantShreddingTableFeature override def requiredFeatures: Set[TableFeature] = Set(VariantTypeTableFeature) } +/** + * Preview feature for the `file` data type (a reference to a range of bytes, inline or in an + * external file; maps to the Parquet `FILE` logical type). See the RFC at + * `protocol_rfcs/file-type.md` (discussion issue delta-io/delta#7147). + * + * This registers the `fileType-preview` table feature so it is recognized and can be gated while + * the type is developed behind the preview name (no forward-compatibility guarantee). Automatic + * enablement when a `file` column is present in the schema is added together with the type + * implementation (schema serialization / read / write / stats). + * + * The feature is removable so that it can be dropped once a table no longer contains `file` + * columns (for example after a `REPLACE` with a schema that has none). There is no `file` type in + * the schema yet, so nothing can currently reference the feature: the "feature in use" checks + * below return "not used" and must be tightened to detect `file` columns when the type support + * lands, otherwise `DROP FEATURE` would be allowed to remove the feature while `file` data exists. + */ +object FileTypePreviewTableFeature extends ReaderWriterFeature(name = "fileType-preview") + with RemovableFeature { + + // TODO(file-type impl): return false once a `file` column is present in the schema, so the + // feature cannot be dropped while it is still in use. + override def validateDropInvariants(table: DeltaTableV2, snapshot: Snapshot): Boolean = true + + // TODO(file-type impl): return true for actions that reference a `file` column, so historical + // uses of the feature are detected during removal (this is a reader-writer feature, so removal + // scans history via this method). + override def actionUsesFeature(action: Action): Boolean = false + + override def preDowngradeCommand(table: DeltaTableV2): PreDowngradeTableFeatureCommand = + FileTypePreDowngradeCommand(table) +} + object DeletionVectorsTableFeature extends ReaderWriterFeature(name = "deletionVectors") with RemovableFeature @@ -920,7 +953,8 @@ object DeletionVectorsTableFeature } object AdaptiveMetadataTableFeature - extends ReaderWriterFeature(name = "adaptiveMetadata-preview") { + extends ReaderWriterFeature(name = "adaptiveMetadata-preview") + with RemovableFeature { // The [[AdaptiveMetadataTableFeature]] relies on the following features: // - catalogManaged: adaptive metadata tables are catalog managed (CCv2) only. @@ -936,6 +970,13 @@ object AdaptiveMetadataTableFeature DomainMetadataTableFeature, DeletionVectorsTableFeature, ColumnMappingTableFeature) + + override def preDowngradeCommand(table: DeltaTableV2): PreDowngradeTableFeatureCommand = + AdaptiveMetadataPreDowngradeCommand(table) + + override def validateDropInvariants(table: DeltaTableV2, snapshot: Snapshot): Boolean = true + + override def actionUsesFeature(action: Action): Boolean = false } object RowTrackingFeature extends WriterFeature(name = "rowTracking") diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/TransactionExecutionObserver.scala b/spark/src/main/scala/org/apache/spark/sql/delta/TransactionExecutionObserver.scala index 89f0101dd67..01bf979d253 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/TransactionExecutionObserver.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/TransactionExecutionObserver.scala @@ -65,7 +65,7 @@ trait TransactionExecutionObserver * These are called from within the transaction object. */ - /** Wraps `prepareCommit`. */ + /** Wraps `prepareInitialActions`. */ def preparingCommit[T](f: => T): T /* diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/actions/DeletionVectorDescriptor.scala b/spark/src/main/scala/org/apache/spark/sql/delta/actions/DeletionVectorDescriptor.scala index 10091db82c5..d00eec4331a 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/actions/DeletionVectorDescriptor.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/actions/DeletionVectorDescriptor.scala @@ -20,10 +20,11 @@ import java.io.{ByteArrayInputStream, ByteArrayOutputStream, DataInputStream, Da import java.net.URI import java.util.{Base64, UUID} -import org.apache.spark.sql.delta.DeltaErrors +import org.apache.spark.sql.delta.{DeltaErrors, DeltaIllegalArgumentException} import org.apache.spark.sql.delta.DeltaUDF +import org.apache.spark.sql.delta.amt.AMTUtils import org.apache.spark.sql.delta.sources.DeltaSQLConf -import org.apache.spark.sql.delta.util.{Codec, DeltaEncoder, JsonUtils} +import org.apache.spark.sql.delta.util.{Codec, DeltaEncoder} import com.fasterxml.jackson.annotation.JsonIgnore import com.fasterxml.jackson.databind.annotation.JsonDeserialize import org.apache.hadoop.fs.Path @@ -33,6 +34,7 @@ import org.apache.spark.sql.{Column, Encoder} import org.apache.spark.sql.functions.{concat, lit, when} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ +import org.apache.spark.util.Utils /** Information about a deletion vector attached to a file action. */ case class DeletionVectorDescriptor( @@ -45,7 +47,7 @@ case class DeletionVectorDescriptor( /** * Contains the actual data that allows accessing the DV. * - * Three options are currently supported: + * Four options are currently supported: * - `storageType="u"` format: `` * The deletion vector is stored in a file with a path relative to * the data directory of this Delta Table, and the file name can be @@ -57,6 +59,12 @@ case class DeletionVectorDescriptor( * - `storageType="p"` format: `` * The DV is stored in a file with an absolute path given by this * url. Special characters in this path must be escaped. + * - `storageType="r"` format: `` + * The DV is stored in a file at this path, relative to the root of + * this Delta Table. Unlike `u`, the file name is arbitrary rather + * than derived from a UUID; unlike `p`, the path is neither + * absolute nor URL-encoded. Only permitted when the + * `adaptiveMetadata` table feature is enabled. */ pathOrInlineDv: String, /** @@ -80,9 +88,13 @@ case class DeletionVectorDescriptor( import DeletionVectorDescriptor._ + /** + * The legacy unique id of the DV. + * Prefer [[uniqueId]] if possible. See more details in the [[uniqueId]] documentation. + */ @JsonIgnore @transient - lazy val uniqueId: String = { + lazy val legacyUniqueId: String = { offset match { case Some(offset) => s"$uniqueFileId@$offset" case None => uniqueFileId @@ -93,6 +105,21 @@ case class DeletionVectorDescriptor( @transient lazy val uniqueFileId: String = s"$storageType$pathOrInlineDv" + /** + * Unique identifier for this DV. + * + * When `useObjectIdentity` is false, this returns the legacy descriptor identity: + * `storageType + pathOrInlineDv + optional offset`. That identity is stable for one serialized + * descriptor, but it treats equivalent `u`, `r`, and in-table `p` descriptors as different. + * + * When `useObjectIdentity` is true, this returns a normalized object identity. That identity + * represents the physical DV object relative to the table root when possible, plus the offset, so + * equivalent `u`, `r`, and in-table `p` descriptors compare equal. + */ + def uniqueId(tableRoot: Path, useObjectIdentity: Boolean): String = { + if (useObjectIdentity) normalizedTableRelativeObjectId(tableRoot) else legacyUniqueId + } + @JsonIgnore protected[delta] def isOnDisk: Boolean = !isInline @@ -100,7 +127,10 @@ case class DeletionVectorDescriptor( protected[delta] def isInline: Boolean = storageType == INLINE_DV_MARKER @JsonIgnore - protected[delta] def isRelative: Boolean = storageType == UUID_DV_MARKER + protected[delta] def isUuidRelative: Boolean = storageType == UUID_DV_MARKER + + @JsonIgnore + protected[delta] def isUnencodedRelative: Boolean = storageType == RELATIVE_DV_MARKER @JsonIgnore protected[delta] def isAbsolute: Boolean = storageType == PATH_DV_MARKER @@ -115,11 +145,12 @@ case class DeletionVectorDescriptor( val (randomPrefix, uuid) = getRandomPrefixAndUuid.get assembleDeletionVectorPath(tableLocation, uuid, randomPrefix) case PATH_DV_MARKER => - // Since there is no need for legacy support for relative paths for DVs, - // relative DVs should *always* use the UUID variant. val parsedUri = new URI(pathOrInlineDv) assert(parsedUri.isAbsolute, "Relative URIs are not supported for DVs") new Path(parsedUri) + case RELATIVE_DV_MARKER => + val path = new Path(pathOrInlineDv) + new Path(tableLocation, path) case _ => throw DeltaErrors.cannotReconstructPathFromURI(pathOrInlineDv) } } @@ -133,7 +164,7 @@ case class DeletionVectorDescriptor( * If the DV path is outside the table directory, returns None. */ def urlEncodedRelativePathIfExists(tablePath: Path): Option[String] = { - if (isRelative) { + if (isUuidRelative) { return Some(SparkPath.fromPath(absolutePath(new Path("."))).urlEncoded) } @@ -149,19 +180,66 @@ case class DeletionVectorDescriptor( } /** - * Parse the prefix and UUID of a relative DV. Returns None if the DV is not relative. + * Parse the prefix and UUID of a u DV. Returns None if the DV is not of type u. */ @JsonIgnore - def getRandomPrefixAndUuid: Option[(String, UUID)] = storageType match { - case UUID_DV_MARKER => - // If the file was written with a random prefix, we have to extract that, - // before decoding the UUID. - val randomPrefixLength = pathOrInlineDv.length - Codec.Base85Codec.ENCODED_UUID_LENGTH - val (randomPrefix, encodedUuid) = pathOrInlineDv.splitAt(randomPrefixLength) - Some((randomPrefix, Codec.Base85Codec.decodeUUID(encodedUuid))) - case _ => - None + def getRandomPrefixAndUuid: Option[(String, UUID)] = + DeletionVectorDescriptor.getRandomPrefixAndUuid(storageType, pathOrInlineDv) + + /** + * Computes a normalized object identity for this descriptor. Use this when the caller wants + * to identify the underlying DV object rather than this descriptor's storage encoding. + */ + def normalizedTableRelativeObjectId(tableRoot: Path): String = { + DeletionVectorDescriptor.normalizedTableRelativeObjectId( + storageType, + pathOrInlineDv, + offset, + tableRoot) + } + + /** + * Like [[normalizedTableRelativeObjectId]] but identifies the physical DV file rather than the + * individual DV within it and returns the normalized (storageType, path) with no offset. + */ + private[delta] def normalizedTableRelativeObjectFile(tableRoot: Path): (String, String) = { + storageType match { + case INLINE_DV_MARKER => + (INLINE_DV_MARKER, pathOrInlineDv) + case UUID_DV_MARKER => + val (randomPrefix, uuid) = getRandomPrefixAndUuid.get + val fileName = assembleDeletionVectorFileName(uuid) + val relativePath = if (randomPrefix.isEmpty) fileName else s"$randomPrefix/$fileName" + (RELATIVE_DV_MARKER, relativePath) + case RELATIVE_DV_MARKER => + (RELATIVE_DV_MARKER, pathOrInlineDv) + case PATH_DV_MARKER => + val path = SparkPath.fromUrlString(pathOrInlineDv).toPath.toString + val relativePath = AMTUtils.relativizeLocation(tableRoot.toString, path) + if (AMTUtils.isAbsoluteLocation(relativePath)) { + (PATH_DV_MARKER, pathOrInlineDv) + } else { + (RELATIVE_DV_MARKER, relativePath) + } + case _ => + throw new IllegalArgumentException( + s"Unsupported deletion vector storage type: $storageType") + } + } + + /** + * Computes a normalized absolute object identity for this descriptor. + * Today this is only used for table clone. + */ + private[delta] def normalizedAbsoluteObjectId(tableRoot: Path): String = { + val (normalizedStorageType, normalizedPathOrInlineDv) = if (isInline) { + (INLINE_DV_MARKER, pathOrInlineDv) + } else { + (PATH_DV_MARKER, urlEncodedPath(tableRoot)) + } + formatIdentity(normalizedStorageType, normalizedPathOrInlineDv, offset) } + /** * Produce a copy of this DV, but using an absolute path. * @@ -169,7 +247,7 @@ case class DeletionVectorDescriptor( */ def copyWithAbsolutePath(tableLocation: Path): DeletionVectorDescriptor = { storageType match { - case UUID_DV_MARKER => + case UUID_DV_MARKER | RELATIVE_DV_MARKER => this.copy( storageType = PATH_DV_MARKER, pathOrInlineDv = urlEncodedPath(tableLocation)) @@ -183,14 +261,15 @@ case class DeletionVectorDescriptor( * * If the DV already has a relative path or is inline, then this is just a normal copy. */ - def copyWithNewRelativePath(id: UUID, randomPrefix: String): DeletionVectorDescriptor = { + def copyWithNewUuidRelativePath(id: UUID, randomPrefix: String): DeletionVectorDescriptor = { storageType match { case PATH_DV_MARKER => this.copy(storageType = UUID_DV_MARKER, pathOrInlineDv = encodeUUID(id, randomPrefix)) - case UUID_DV_MARKER | INLINE_DV_MARKER => this.copy() + case UUID_DV_MARKER | RELATIVE_DV_MARKER | INLINE_DV_MARKER => this.copy() } } + @JsonIgnore def inlineData: Array[Byte] = { require(isInline, "Can't get data for an on-disk DV from the log.") @@ -256,6 +335,63 @@ object DeletionVectorDescriptor { final val PATH_DV_MARKER: String = "p" final val INLINE_DV_MARKER: String = "i" final val UUID_DV_MARKER: String = "u" + final val RELATIVE_DV_MARKER: String = "r" + + private def formatIdentity( + storageType: String, + pathOrInlineDv: String, + offset: Option[Int]): String = { + offset match { + case Some(offsetValue) => s"$storageType$pathOrInlineDv@$offsetValue" + case None => s"$storageType$pathOrInlineDv" + } + } + + private def getRandomPrefixAndUuid( + storageType: String, + pathOrInlineDv: String): Option[(String, UUID)] = storageType match { + case UUID_DV_MARKER => + // If the file was written with a random prefix, we have to extract that, + // before decoding the UUID. + val randomPrefixLength = pathOrInlineDv.length - Codec.Base85Codec.ENCODED_UUID_LENGTH + val (randomPrefix, encodedUuid) = pathOrInlineDv.splitAt(randomPrefixLength) + Some((randomPrefix, Codec.Base85Codec.decodeUUID(encodedUuid))) + case _ => + None + } + + /** + * See comments of [[normalizedTableRelativeObjectId]] in class. + */ + private[delta] def normalizedTableRelativeObjectId( + storageType: String, + pathOrInlineDv: String, + offset: Option[Int], + tableRoot: Path): String = { + storageType match { + case INLINE_DV_MARKER => + formatIdentity(INLINE_DV_MARKER, pathOrInlineDv, offset) + case UUID_DV_MARKER => + val (randomPrefix, uuid) = getRandomPrefixAndUuid(storageType, pathOrInlineDv).get + val fileName = assembleDeletionVectorFileName(uuid) + val relativePath = if (randomPrefix.isEmpty) fileName else s"$randomPrefix/$fileName" + formatIdentity(RELATIVE_DV_MARKER, relativePath, offset) + case RELATIVE_DV_MARKER => + formatIdentity(RELATIVE_DV_MARKER, pathOrInlineDv, offset) + case PATH_DV_MARKER => + val path = SparkPath.fromUrlString(pathOrInlineDv).toPath.toString + val relativePath = AMTUtils.relativizeLocation(tableRoot.toString, path) + if (AMTUtils.isAbsoluteLocation(relativePath)) { + formatIdentity(PATH_DV_MARKER, pathOrInlineDv, offset) + } else { + formatIdentity(RELATIVE_DV_MARKER, relativePath, offset) + } + case _ => + throw new DeltaIllegalArgumentException( + errorClass = "INTERNAL_ERROR", + messageParameters = Array(s"Unsupported deletion vector storage type: $storageType")) + } + } private final val deletionVectorFileNameRegex = raw"${new Path(DELETION_VECTOR_FILE_NAME_CORE).toUri}_([^.]+)\.bin".r @@ -268,7 +404,7 @@ object DeletionVectorDescriptor { implicit def encoder: Encoder[DeletionVectorDescriptor] = _encoder.get /** Utility method to create an on-disk [[DeletionVectorDescriptor]] */ - def onDiskWithRelativePath( + def onDiskWithUuidRelativePath( id: UUID, randomPrefix: String = "", sizeInBytes: Int, @@ -298,6 +434,37 @@ object DeletionVectorDescriptor { cardinality = cardinality, maxRowIndex = maxRowIndex) + /** + * Utility method to create a [[DeletionVectorDescriptor]] for an unencoded path relative to the + * table root. Callers are responsible for relativizing the path before invoking this method. + * + * @param relativePath Relative path to the table root. Must be unencoded. + * @param sizeInBytes Size of the serialized DV in bytes (raw data size, before encoding). + * @param cardinality Number of rows the DV logically removes from the file. + * @param offset Start of the data for this DV. + * @param maxRowIndex Transient property that is used to validate DV correctness. It is not + * stored in the log. + */ + def createRelativePathDVDescriptor( + relativePath: String, + sizeInBytes: Int, + cardinality: Long, + offset: Option[Int] = None, + maxRowIndex: Option[Long] = None): DeletionVectorDescriptor = { + if (Utils.isTesting) { + require(!AMTUtils.isAbsoluteLocation(relativePath), + s"A '$RELATIVE_DV_MARKER' deletion vector must have a relative path, " + + s"but got: $relativePath") + } + DeletionVectorDescriptor( + storageType = RELATIVE_DV_MARKER, + pathOrInlineDv = relativePath, + offset = offset, + sizeInBytes = sizeInBytes, + cardinality = cardinality, + maxRowIndex = maxRowIndex) + } + /** Utility method to create an inline [[DeletionVectorDescriptor]] */ def inlineInLog( data: Array[Byte], @@ -309,19 +476,19 @@ object DeletionVectorDescriptor { cardinality = cardinality) /** - * Returns whether the path points to a deletion vector file. - * Note, external writers are no enforced to create DV files with the same naming convertions. - * This function is intended for testing. */ - private[delta] def isDeletionVectorPath(path: Path): Boolean = + * Returns whether the path points to a deletion vector file written by this Spark + * implementation. This function is intended for testing. Use at your own risk. + */ + private[delta] def isSparkImplDeletionVectorPath(path: Path): Boolean = deletionVectorFileNamePattern.matcher(path.getName).matches() - /** Only for testing. */ - private[delta] def isDeletionVectorPath(path: String): Boolean = - isDeletionVectorPath(new Path(path)) + /** This function is intended for testing. Use at your own risk. */ + private[delta] def isSparkImplDeletionVectorPath(path: String): Boolean = + isSparkImplDeletionVectorPath(new Path(path)) - /** Same as above but as a column expression. Only for testing. */ - private[delta] def isDeletionVectorPath(pathCol: Column): Column = - DeltaUDF.booleanFromString(isDeletionVectorPath)(pathCol) + /** Same as above but as a column expression. Use at your own risk. */ + private[delta] def isSparkImplDeletionVectorPath(pathCol: Column): Column = + DeltaUDF.booleanFromString(isSparkImplDeletionVectorPath)(pathCol) /** Returns a boolean column that corresponds to whether each deletion vector is inline. */ def isInline(dv: Column): Column = @@ -348,10 +515,10 @@ object DeletionVectorDescriptor { _.urlEncodedRelativePathIfExists(tablePath))(deletionVectorCol) /** - * This produces the same output as [[DeletionVectorDescriptor.uniqueId]] but as a column - * expression, so it can be used directly in a Spark query. + * This produces the same output as [[DeletionVectorDescriptor.legacyUniqueId]] but as a + * column expression, so it can be used directly in a Spark query. */ - def uniqueIdExpression(deletionVectorCol: Column): Column = { + def legacyUniqueIdExpression(deletionVectorCol: Column): Column = { when(deletionVectorCol("offset").isNotNull, concat( deletionVectorCol("storageType"), @@ -363,6 +530,31 @@ object DeletionVectorDescriptor { deletionVectorCol("pathOrInlineDv"))) } + /** + * Produces the same output as [[DeletionVectorDescriptor.uniqueId]] with object identity, but as + * a column expression, so it can be used directly in a Spark query. + */ + def objectUniqueIdExpression(deletionVectorCol: Column, tableRoot: Path): Column = { + val objectUniqueId = DeltaUDF.stringFromDeletionVectorDescriptor( + _.uniqueId(tableRoot, useObjectIdentity = true)) + when(deletionVectorCol.isNotNull, objectUniqueId(deletionVectorCol)) + } + + /** + * Produces the same output as [[DeletionVectorDescriptor.uniqueId]] but as a column expression, + * so it can be used directly in a Spark query. + */ + def uniqueIdExpression( + deletionVectorCol: Column, + tableRoot: Path, + useObjectIdentity: Boolean): Column = { + if (useObjectIdentity) { + objectUniqueIdExpression(deletionVectorCol, tableRoot) + } else { + legacyUniqueIdExpression(deletionVectorCol) + } + } + /** * Return the unique path under `parentPath` that is based on `id`. * diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/actions/InMemoryLogReplay.scala b/spark/src/main/scala/org/apache/spark/sql/delta/actions/InMemoryLogReplay.scala index 10c0cd74b30..c0c67108e79 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/actions/InMemoryLogReplay.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/actions/InMemoryLogReplay.scala @@ -16,7 +16,8 @@ package org.apache.spark.sql.delta.actions -import java.net.URI +import org.apache.spark.sql.delta.actions.FileAction.UniqueFileActionTuple +import org.apache.hadoop.fs.Path /** @@ -27,7 +28,7 @@ import java.net.URI * tombstone until `minFileRetentionTimestamp` has passed. If `minFileRetentionTimestamp` is * None, all [[RemoveFile]] actions are retained. * A [[RemoveFile]] "corresponds" to the [[AddFile]] that matches both the parquet file URI - * *and* the deletion vector's URI (if any). + * *and* the deletion vector id (if any). * - The most recent version for any `appId` in a [[SetTransaction]] wins. * - The most recent [[Metadata]] wins. * - The most recent [[Protocol]] version wins. @@ -35,12 +36,17 @@ import java.net.URI * (either [[AddFile]] or [[RemoveFile]]) * * This class is not thread safe. + * + * @param retainFirstBackreference Whether to retain the first non-empty back reference for each + * UniqueFileActionTuple or let the last action win. + * */ class InMemoryLogReplay( minFileRetentionTimestamp: Option[Long], - minSetTransactionRetentionTimestamp: Option[Long]) extends LogReplay { - - import InMemoryLogReplay._ + minSetTransactionRetentionTimestamp: Option[Long], + tableRoot: Path, + useDeletionVectorObjectIdentity: Boolean, + retainFirstBackreference: Boolean = false) extends LogReplay { private var currentProtocolVersion: Protocol = null private var currentVersion: Long = -1 @@ -54,6 +60,9 @@ class InMemoryLogReplay( // RemoveFiles that had NOT cancelled any AddFile during replay private val activeRemoveFiles = new scala.collection.mutable.HashMap[UniqueFileActionTuple, RemoveFile]() + // The first seen non-empty AMT BackReference for each unique file action. + private lazy val firstBackReferences = + new scala.collection.mutable.HashMap[UniqueFileActionTuple, BackReference]() override def append(version: Long, actions: Iterator[Action]): Unit = { assert(currentVersion == -1 || version == currentVersion + 1, @@ -72,18 +81,25 @@ class InMemoryLogReplay( case a: Protocol => currentProtocolVersion = a case add: AddFile => - val uniquePath = UniqueFileActionTuple(add.pathAsUri, add.getDeletionVectorUniqueId) + val uniquePath = add.toUniqueFileActionTuple(tableRoot, useDeletionVectorObjectIdentity) activeFiles(uniquePath) = add.copy(dataChange = false) // Remove the tombstone to make sure we only output one `FileAction`. cancelledRemoveFiles.remove(uniquePath) // Remove from activeRemoveFiles to handle commits that add a previously-removed file activeRemoveFiles.remove(uniquePath) + if (retainFirstBackreference) { + add.backReference.foreach(br => firstBackReferences.getOrElseUpdate(uniquePath, br)) + } case remove: RemoveFile => - val uniquePath = UniqueFileActionTuple(remove.pathAsUri, remove.getDeletionVectorUniqueId) + val uniquePath = + remove.toUniqueFileActionTuple(tableRoot, useDeletionVectorObjectIdentity) activeFiles.remove(uniquePath) match { case Some(_) => cancelledRemoveFiles(uniquePath) = remove case None => activeRemoveFiles(uniquePath) = remove } + if (retainFirstBackreference) { + remove.backReference.foreach(br => firstBackReferences.getOrElseUpdate(uniquePath, br)) + } case _: CommitInfo => // do nothing case _: AddCDCFile => // do nothing case _: Checkpoint => // AMT pointer; the manifest tree is loaded separately. @@ -91,13 +107,31 @@ class InMemoryLogReplay( } } + private def getLiveFiles: Iterable[AddFile] = { + if (retainFirstBackreference) { + activeFiles.map { case (uniquePath, add) => + firstBackReferences.get(uniquePath) + .map(preservedBackReference => add.copy(backReference = Some(preservedBackReference))) + .getOrElse(add) + } + } else { + activeFiles.values + } + } + private def getTombstones: Iterable[FileAction] = { - val allRemovedFiles = cancelledRemoveFiles.values ++ activeRemoveFiles.values + val allRemovedFiles = cancelledRemoveFiles.toSeq ++ activeRemoveFiles.toSeq val filteredRemovedFiles = minFileRetentionTimestamp match { case None => allRemovedFiles - case Some(timestamp) => allRemovedFiles.filter(_.delTimestamp > timestamp) + case Some(timestamp) => allRemovedFiles.filter(_._2.delTimestamp > timestamp) + } + filteredRemovedFiles.map { case (uniquePath, remove) => + if (retainFirstBackreference) { + remove.copy(dataChange = false, backReference = firstBackReferences.get(uniquePath)) + } else { + remove.copy(dataChange = false) + } } - filteredRemovedFiles.map(_.copy(dataChange = false)) } private[delta] def getTransactions: Iterable[SetTransaction] = { @@ -123,7 +157,7 @@ class InMemoryLogReplay( /** Returns the current state of the Table as an iterator of actions. */ override def checkpoint: Iterator[Action] = { - val fileActions = (activeFiles.values ++ getTombstones).toSeq.sortBy(_.path) + val fileActions = (getLiveFiles ++ getTombstones).toSeq.sortBy(_.path) Option(currentProtocolVersion).toIterator ++ Option(currentMetaData).toIterator ++ @@ -135,18 +169,3 @@ class InMemoryLogReplay( /** Returns all [[AddFile]] actions after the Log Replay */ private[delta] def allFiles: Seq[AddFile] = activeFiles.values.toSeq } - -object InMemoryLogReplay{ - /** The unit of path uniqueness in delta log actions is the tuple `(parquet file, dv)`. */ - final case class UniqueFileActionTuple(fileURI: URI, deletionVectorURI: Option[String]) - - implicit class UniqueAddFileTuple(a: AddFile) { - def toUniqueFileActionTuple: UniqueFileActionTuple = - UniqueFileActionTuple(a.pathAsUri, a.getDeletionVectorUniqueId) - } - - implicit class UniqueRemoveFileTuple(r: RemoveFile) { - def toUniqueFileActionTuple: UniqueFileActionTuple = - UniqueFileActionTuple(r.pathAsUri, r.getDeletionVectorUniqueId) - } -} diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/actions/actions.scala b/spark/src/main/scala/org/apache/spark/sql/delta/actions/actions.scala index e08942ee84b..2bbe7cacec3 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/actions/actions.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/actions/actions.scala @@ -30,7 +30,7 @@ import scala.util.control.NonFatal import com.databricks.spark.util.TagDefinition import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.ClassicColumnConversions._ -import org.apache.spark.sql.delta.amt.AMTUtils +import org.apache.spark.sql.delta.amt.{AMTPassthrough, AMTUtils} import org.apache.spark.sql.delta.commands.DeletionVectorUtils import org.apache.spark.sql.delta.metering.{DeltaLogging, DeltaLoggingProvider} import org.apache.spark.sql.delta.sources.DeltaSQLConf @@ -62,6 +62,7 @@ import org.apache.spark.sql.{Column, Encoder, SparkSession} import org.apache.spark.sql.catalyst.ScalaReflection import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder import org.apache.spark.sql.catalyst.expressions.Literal +import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{DataType, StructField, StructType} import org.apache.spark.util.Utils @@ -825,6 +826,39 @@ sealed trait FileAction extends Action { } } +object FileAction { + /** The path unique tuple in delta log actions: (parquet file, deletion vector id). */ + final case class UniqueFileActionTuple private (fileURI: URI, deletionVectorId: Option[String]) + + /** + * Whether deletion vectors should be reconciled by their normalized object identity for the + * given table (see [[DeletionVectorDescriptor.uniqueId]]). + * + * @param metadata the table metadata. + * @param protocol the table protocol. + */ + def useDeletionVectorObjectIdentity( + metadata: Metadata, + protocol: Protocol, + spark: SparkSession): Boolean = + AMTUtils.amtEnabled(metadata, protocol) || + spark.conf.get(DeltaSQLConf.DELETION_VECTORS_USE_OBJECT_IDENTITY_FOR_NON_AMT) + + /** + * The unique tuple `(parquet file, deletion vector id)` of a file action, as a method on + * [[AddFile]] / [[RemoveFile]]. See [[DeletionVectorDescriptor.uniqueId]] for the + * `useObjectIdentity` semantics. + */ + implicit class FileActionUniqueTupleOps(val action: FileAction) { + def toUniqueFileActionTuple( + tableRoot: Path, + useObjectIdentity: Boolean): UniqueFileActionTuple = + UniqueFileActionTuple( + action.pathAsUri, + Option(action.deletionVector).map(_.uniqueId(tableRoot, useObjectIdentity))) + } +} + case class ParsedStatsFields( numLogicalRecords: Option[Long], tightBounds: Option[Boolean]) @@ -917,7 +951,8 @@ case class AddFile( @JsonDeserialize(contentAs = classOf[java.lang.Long]) defaultRowCommitVersion: Option[Long] = None, clusteringProvider: Option[String] = None, - backReference: Option[BackReference] = None + backReference: Option[BackReference] = None, + amtPassthrough: Option[AMTPassthrough] = None ) extends FileAction with HasNumRecords { require(path.nonEmpty) @@ -925,6 +960,12 @@ case class AddFile( def remove: RemoveFile = removeWithTimestamp() + /** + * Returns the file sequence number, currently euquals to [[defaultRowCommitVersion]]. + */ + @JsonIgnore + def effectiveFileSequenceNumber: Option[Long] = defaultRowCommitVersion + def removeWithTimestamp( timestamp: Long = System.currentTimeMillis(), dataChange: Boolean = true @@ -965,8 +1006,10 @@ case class AddFile( case Some(_) => deletionVector.copy(maxRowIndex = None) case _ => deletionVector } - var addFileWithNewDv = - this.copy(deletionVector = dvDescriptorWithoutMaxRowIndex, dataChange = dataChange) + var addFileWithNewDv = this.copy( + deletionVector = dvDescriptorWithoutMaxRowIndex, + dataChange = dataChange, + backReference = None) if (updateStats) { addFileWithNewDv = addFileWithNewDv.withoutTightBoundStats } @@ -981,13 +1024,27 @@ case class AddFile( } /** - * Return the unique id of the deletion vector, if present, or `None` if there's no DV. + * Return the legacy unique id of the deletion vector, if present, or `None` if there's no DV. * - * The unique id differentiates DVs, even if there are multiple in the same file + * The legacy unique id differentiates DVs, even if there are multiple in the same file * or the DV is stored inline. */ @JsonIgnore - def getDeletionVectorUniqueId: Option[String] = Option(deletionVector).map(_.uniqueId) + def getLegacyDeletionVectorUniqueId: Option[String] = + Option(deletionVector).map(_.legacyUniqueId) + + /** + * Return the unique id of the deletion vector, if present, or `None` if there's no DV. + * + * This overload allows callers to use object identity for comparisons. Prefer this + * if possible. See document of [[DeletionVectorDescriptor.uniqueId]] for more details. + */ + @JsonIgnore + def getDeletionVectorUniqueId( + tableRoot: Path, + useObjectIdentity: Boolean): Option[String] = { + Option(deletionVector).map(_.uniqueId(tableRoot, useObjectIdentity)) + } /** Update stats to have tightBounds = false, if file has any stats. */ def withoutTightBoundStats: AddFile = { @@ -1141,7 +1198,7 @@ object AddFile { */ case class BackReference( manifest: String, - pos: Long) + pos: Int) object BackReference { @@ -1156,10 +1213,10 @@ object BackReference { * Logical removal of a given file from the reservoir. Acts as a tombstone before a file is * deleted permanently. * - * Note that for protocol compatibility reasons, the fields `partitionValues`, `size`, and `tags` - * are only present when the extendedFileMetadata flag is true. New writers should generally be - * setting this flag, but old writers (and FSCK) won't, so readers must check this flag before - * attempting to consume those values. + * Note that for protocol compatibility reasons, the fields `partitionValues` and `size` are only + * present when the extendedFileMetadata flag is true. New writers should generally be setting this + * flag, but old writers (and FSCK) won't, so readers must check this flag before attempting to + * consume those values. * * Since old tables would not have `extendedFileMetadata` and `size` field, we should make them * nullable by setting their type Option. @@ -1191,13 +1248,27 @@ case class RemoveFile( val delTimestamp: Long = deletionTimestamp.getOrElse(0L) /** - * Return the unique id of the deletion vector, if present, or `None` if there's no DV. + * Return the legacy unique id of the deletion vector, if present, or `None` if there's no DV. * - * The unique id differentiates DVs, even if there are multiple in the same file + * The legacy unique id differentiates DVs, even if there are multiple in the same file * or the DV is stored inline. */ @JsonIgnore - def getDeletionVectorUniqueId: Option[String] = Option(deletionVector).map(_.uniqueId) + def getLegacyDeletionVectorUniqueId: Option[String] = + Option(deletionVector).map(_.legacyUniqueId) + + /** + * Return the unique id of the deletion vector, if present, or `None` if there's no DV. + * + * This overload allows callers to use object identity for comparisons. Prefer this + * if possible. See document of [[DeletionVectorDescriptor.uniqueId]] for more details. + */ + @JsonIgnore + def getDeletionVectorUniqueId( + tableRoot: Path, + useObjectIdentity: Boolean): Option[String] = { + Option(deletionVector).map(_.uniqueId(tableRoot, useObjectIdentity)) + } /** * Create a copy with the new tag. `extendedFileMetadata` is copied unchanged. @@ -1216,7 +1287,8 @@ case class RemoveFile( /** Only for testing. */ @JsonIgnore - private [delta] def isDVTombstone: Boolean = DeletionVectorDescriptor.isDeletionVectorPath(new Path(path)) + private [delta] def isDVTombstone: Boolean = + DeletionVectorDescriptor.isSparkImplDeletionVectorPath(new Path(path)) } // scalastyle:on @@ -1406,6 +1478,7 @@ trait CommitMarker { * epoch in milliseconds when the commit write was started. This should * only be set when the feature inCommitTimestamps is enabled. * @param isBlindAppend Whether this commit has blindly appended without caring about existing files + * @param dataChange Whether this commit changes the data of the table logically. * @param engineInfo The information for the engine that makes the commit. * If a commit is made by Delta Lake 1.1.0 or above, it will be * `Apache-Spark/x.y.z Delta-Lake/x.y.z`. @@ -1432,6 +1505,7 @@ case class CommitInfo( readVersion: Option[Long], isolationLevel: Option[String], isBlindAppend: Option[Boolean], + dataChange: Option[Boolean], operationMetrics: Option[Map[String, String]], userMetadata: Option[String], tags: Option[Map[String, String]], @@ -1494,9 +1568,41 @@ object NotebookInfo { object CommitInfo { def empty(version: Option[Long] = None): CommitInfo = { CommitInfo(version, None, null, None, None, null, null, None, None, - None, None, None, None, None, None, None, None, None, None) + None, None, None, None, None, None, None, None, None, None, None) + } + + /** + * Derives the commit-level `dataChange` summary stored in [[CommitInfo.dataChange]] from the + * actions of a commit: a commit changes data if and only if at least one of its file actions + * does. + */ + def dataChangeFromActions(actions: Iterable[Action]): Boolean = actions.exists { + case f: FileAction => f.dataChange + case _ => false } + def useCommitInfoForDataChangeEnabled: Boolean = + SQLConf.get.getConf(DeltaSQLConf.DELTA_COMMIT_INFO_DATA_CHANGE_READ_ENABLED) + + /** + * The commit-level `dataChange` value. `None` when it has to fall back to + * inspecting the commit's file actions itself. + */ + def commitChangedData(commitInfo: Option[CommitInfo]): Option[Boolean] = { + if (!useCommitInfoForDataChangeEnabled) None else commitInfo.flatMap(_.dataChange) + } + + /** + * A predicate telling whether a file action of the commit changes data: + * taken from the commit-level summary when the commit records one, and from the action + * itself otherwise. + */ + def fileActionChangesData(commitInfo: Option[CommitInfo]): FileAction => Boolean = + commitChangedData(commitInfo) match { + case Some(commitChangesData) => _ => commitChangesData + case None => _.dataChange + } + // scalastyle:off argcount def apply( @@ -1508,12 +1614,28 @@ object CommitInfo { readVersion: Option[Long], isolationLevel: Option[String], isBlindAppend: Option[Boolean], + dataChange: Option[Boolean], operationMetrics: Option[Map[String, String]], userMetadata: Option[String], tags: Option[Map[String, String]], - txnId: Option[String]): CommitInfo = { - apply(None, time, operation, inCommitTimestamp, operationParameters, commandContext, - readVersion, isolationLevel, isBlindAppend, operationMetrics, userMetadata, tags, txnId) + txnId: Option[String], + lastManifestCommit: Option[LastManifestCommit]): CommitInfo = { + apply( + version = None, + time, + operation, + inCommitTimestamp, + operationParameters, + commandContext, + readVersion, + isolationLevel, + isBlindAppend, + dataChange, + operationMetrics, + userMetadata, + tags, + txnId, + lastManifestCommit) } def apply( @@ -1526,10 +1648,12 @@ object CommitInfo { readVersion: Option[Long], isolationLevel: Option[String], isBlindAppend: Option[Boolean], + dataChange: Option[Boolean], operationMetrics: Option[Map[String, String]], userMetadata: Option[String], tags: Option[Map[String, String]], - txnId: Option[String]): CommitInfo = { + txnId: Option[String], + lastManifestCommit: Option[LastManifestCommit]): CommitInfo = { val getUserName = commandContext.get("user").flatMap { case "unknown" => None @@ -1550,12 +1674,13 @@ object CommitInfo { readVersion, isolationLevel, isBlindAppend, + dataChange, operationMetrics, userMetadata, tags, getEngineInfo, txnId, - lastManifestCommit = None) + lastManifestCommit) } // scalastyle:on argcount @@ -1603,11 +1728,16 @@ sealed trait CheckpointOnlyAction extends Action * * @param path root manifest path, relative to the table root. * @param sizeInBytes size of the root manifest file in bytes. + * @param version table version this root reflects. Must be `<=` the enclosing + * [[Checkpoint.version]]; equal in a manifest commit, and less + * or equal in a standalone checkpoint (the gap is covered by + * inline file actions). * @param tags additional metadata about the AMT. See [[ContentRoot.Tags]] for known keys. */ case class ContentRoot( path: String, sizeInBytes: Long, + version: Long, tags: Map[String, String] = null) { private def tag(key: ContentRoot.Tags.KeyType): Option[String] = @@ -1616,6 +1746,9 @@ case class ContentRoot( /** Whether this manifest tree was built incrementally, if recorded. */ def isIncremental: Option[Boolean] = tag(ContentRoot.Tags.IS_INCREMENTAL).map(_.toBoolean) + /** Number of leaf manifests in this tree, if recorded; `0` means a root-only tree. */ + def numLeaves: Option[Long] = tag(ContentRoot.Tags.NUM_LEAVES).map(_.toLong) + /** The version of the most recent full (non-incremental) manifest rewrite, if recorded. */ def lastManifestCommitWithFullRewrite: Option[Long] = tag(ContentRoot.Tags.LAST_MANIFEST_COMMIT_WITH_FULL_REWRITE).map(_.toLong) @@ -1644,15 +1777,19 @@ object ContentRoot { def apply( path: String, sizeInBytes: Long, + version: Long, isIncremental: Boolean, - lastManifestCommitWithFullRewrite: Long): ContentRoot = { + lastManifestCommitWithFullRewrite: Long, + numLeaves: Long): ContentRoot = { ContentRoot( path = path, sizeInBytes = sizeInBytes, + version = version, tags = Map( Tags.IS_INCREMENTAL.name -> isIncremental.toString, Tags.LAST_MANIFEST_COMMIT_WITH_FULL_REWRITE.name -> - lastManifestCommitWithFullRewrite.toString + lastManifestCommitWithFullRewrite.toString, + Tags.NUM_LEAVES.name -> numLeaves.toString ) ) } @@ -1665,6 +1802,8 @@ object ContentRoot { /** The version of the most recent full (non-incremental) manifest rewrite. */ object LAST_MANIFEST_COMMIT_WITH_FULL_REWRITE extends KeyType("lastManifestCommitWithFullRewrite") + /** Number of leaf manifests in the tree; `0` means a root-only tree. */ + object NUM_LEAVES extends KeyType("numLeaves") } } @@ -1706,7 +1845,8 @@ object SidecarType { * * @param version version at which this checkpoint is valid. May belong to a previous * commit (the checkpoint can lag the enclosing commit). - * @param contentRoot pointer to the Iceberg v4 root manifest. + * @param contentRoot pointer to the Iceberg v4 root manifest. `contentRoot.version` is + * the table version the tree reflects. * @param protocol protocol snapshot at `version`. Must be non null. * @param metaData metadata snapshot at `version`. Must be non null. * @param domainMetadata all [[DomainMetadata]] entries carried inline. An empty list means @@ -1731,6 +1871,9 @@ case class Checkpoint( // AMT checkpoint sidecars must always declare their type. require(sidecars.forall(_.sidecarType.isDefined), "All sidecars in a Checkpoint must have a sidecarType.") + require( + contentRoot.version <= version, + s"contentRoot.version (${contentRoot.version}) must be <= checkpoint version ($version).") override def wrap: SingleAction = SingleAction(checkpoint = this) } diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/amt/AMTActionsProvider.scala b/spark/src/main/scala/org/apache/spark/sql/delta/amt/AMTActionsProvider.scala new file mode 100644 index 00000000000..42b3135bc7c --- /dev/null +++ b/spark/src/main/scala/org/apache/spark/sql/delta/amt/AMTActionsProvider.scala @@ -0,0 +1,101 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.amt + +import org.apache.spark.sql.delta.{DeltaLog, Snapshot} +import org.apache.spark.sql.delta.actions.{AddFile, DomainMetadata, Metadata, Protocol, SetTransaction} + +/** + * The previous-AMT state that an incremental write builds the next manifest tree on top of. + * + * This is everything [[IncrementalAMTWriter]] needs from the "old" tree. + * + * @param metadata the old AMT's inline table metadata. + * @param protocol the old AMT's inline table protocol. + * @param setTransactions the old AMT's inline `SetTransaction` actions. + * @param domainMetadatas the old AMT's inline `DomainMetadata` actions. + * @param fileActionsFromRoot the live root-resident file actions (leaf-resident files + * are carried forward by pointer, not materialized here). + * @param allLeafs the root's `DATA_MANIFEST` leaf pointers, one per leaf. + * @param version the table version the old AMT describes. + * @param lastManifestCommitWithFullRewrite the last-full-rewrite marker carried forward into the + * new tree's `ContentRoot`. + */ +case class BaseAMTActionsResult( + metadata: Metadata, + protocol: Protocol, + setTransactions: Seq[SetTransaction], + domainMetadatas: Seq[DomainMetadata], + fileActionsFromRoot: Seq[AddFile], + allLeafs: Seq[DataManifestEntry], + version: Long, + lastManifestCommitWithFullRewrite: Option[Long]) + +/** + * Loads the previous-AMT state that [[IncrementalAMTWriter]] extends. Abstracting the source lets + * the writer build on top of any old-AMT representation, not just a resolved checkpoint provider. + */ +trait BaseAMTActionsProvider { + def load(): BaseAMTActionsResult +} + +/** + * A [[BaseAMTActionsProvider]] backed by a Delta snapshot with no previous AMT. + * + * Currently supports only empty snapshots. Supporting nonempty snapshots requires preserving + * file tracking semantics and loading live files without collecting an unbounded snapshot on the + * driver. + */ +class BaseSnapshotActionsProvider( + snapshot: Snapshot) extends BaseAMTActionsProvider { + require(!snapshot.checkpointProvider.isInstanceOf[AMTCheckpointProvider], + "A snapshot-backed AMT base must not already have an AMT checkpoint.") + require(snapshot.numOfFiles == 0, + "The snapshot-backed AMT actions provider currently requires an empty snapshot.") + + override def load(): BaseAMTActionsResult = { + BaseAMTActionsResult( + metadata = snapshot.metadata, + protocol = snapshot.protocol, + setTransactions = snapshot.setTransactions, + domainMetadatas = snapshot.domainMetadata, + // Safe because of the empty-snapshot invariant above. A nonempty implementation must supply + // every live base file here without eagerly collecting an unbounded snapshot to the driver. + fileActionsFromRoot = Seq.empty, + allLeafs = Seq.empty, + version = snapshot.version, + lastManifestCommitWithFullRewrite = None) + } +} + +/** A [[BaseAMTActionsProvider]] backed by an already-resolved [[AMTCheckpointProvider]]. */ +class BaseAMTCheckpointActionsProvider( + deltaLog: DeltaLog, + checkpointProvider: AMTCheckpointProvider) extends BaseAMTActionsProvider { + override def load(): BaseAMTActionsResult = { + val checkpoint = checkpointProvider.checkpointAction + BaseAMTActionsResult( + metadata = checkpoint.metaData, + protocol = checkpoint.protocol, + setTransactions = checkpoint.txns, + domainMetadatas = checkpoint.domainMetadata, + fileActionsFromRoot = AMTCheckpointProvider.readLiveRootDataEntries(deltaLog, checkpoint), + allLeafs = checkpointProvider.leaves, + version = checkpoint.contentRoot.version, + lastManifestCommitWithFullRewrite = checkpoint.contentRoot.lastManifestCommitWithFullRewrite) + } +} diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/amt/AMTCheckpointProvider.scala b/spark/src/main/scala/org/apache/spark/sql/delta/amt/AMTCheckpointProvider.scala index bbda79d8cc2..e0036072255 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/amt/AMTCheckpointProvider.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/amt/AMTCheckpointProvider.scala @@ -16,20 +16,22 @@ package org.apache.spark.sql.delta.amt +import org.apache.spark.sql.delta.{RowIndexFilter, RowIndexFilterProvider} import org.apache.spark.sql.delta.{CheckpointPolicy, CheckpointProvider, DeltaLog, DeltaLogFileIndex, Snapshot} import org.apache.spark.sql.delta.DeltaLogFileIndex.COMMIT_VERSION_COLUMN -import org.apache.spark.sql.delta.actions.{Action, AddFile, BackReference, Checkpoint, ContentRoot, RemoveFile, SingleAction} -import org.apache.spark.sql.delta.deletionvectors.RoaringBitmapArray +import org.apache.spark.sql.delta.actions.{Action, AddFile, BackReference, Checkpoint, ContentRoot, FileAction, Metadata, Protocol, RemoveFile, SingleAction} +import org.apache.spark.sql.delta.actions.FileAction.UniqueFileActionTuple import org.apache.spark.sql.delta.util.DeltaEncoder import org.apache.hadoop.fs.{FileStatus, Path} import org.apache.spark.paths.SparkPath -import org.apache.spark.sql.{DataFrame, Dataset, Encoder, SparkSession} +import org.apache.spark.sql.{Column, DataFrame, Dataset, Encoder, SparkSession} +import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.execution.datasources.FileFormat.{FILE_PATH, METADATA_NAME} import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat -import org.apache.spark.sql.functions.{col, lit, struct} +import org.apache.spark.sql.expressions.Window +import org.apache.spark.sql.functions.{coalesce, col, lit, struct, sum, when} import org.apache.spark.sql.types.StructType -import org.apache.spark.util.SerializableConfiguration /** * A [[CheckpointProvider]] backed by an AMT (Adaptive Metadata Tree) manifest tree. @@ -37,18 +39,27 @@ import org.apache.spark.util.SerializableConfiguration * This provider is only for inline manifest-commit checkpoints, and this is not intended for * standalone checkpoint which also refers to an AMT. * + * @param manifestCommitVersion The version of the manifest commit that wrote this checkpoint. * @param checkpointAction The inline-emitted Checkpoint action this tree was committed with; * carries the version, contentRoot, and inline non-file state. * @param leaves The root's `DATA_MANIFEST` pointer entries, one per leaf reachable from * the root. Each entry's `location` is stored table-root-relative; use - * [[leafManifestAbsolutePaths]] to resolve them against the table root. + * [[liveLeafManifestAbsolutePaths]] to resolve them against the table root. * @param tableRoot The table's data path. */ final class AMTCheckpointProvider( + val manifestCommitVersion: Long, val checkpointAction: Checkpoint, val leaves: Seq[DataManifestEntry], val tableRoot: Path) - extends CheckpointProvider { + extends AMTCheckpointProviderImpl +{ +} + +trait AMTCheckpointProviderImpl extends CheckpointProvider { + self: AMTCheckpointProvider => + + import AMTCheckpointProvider.AMTDataEntryExtended /** The table version the manifest tree describes. */ def checkpointVersion: Long = checkpointAction.version @@ -59,14 +70,21 @@ final class AMTCheckpointProvider( /** Absolute [[Path]] to the root manifest parquet, resolved against the table root. */ private val rootManifestAbsolutePath: Path = contentRoot.getAbsolutePath(tableRoot) - /** Absolute [[Path]]s to the leaf manifest parquet files, resolved against the table root. */ - lazy val leafManifestAbsolutePaths: Seq[Path] = leaves.map(_.getAbsolutePath(tableRoot)) + /** The live leaf pointers which must have all the live [[DataEntry]]s. */ + private lazy val liveLeaves: Seq[DataManifestEntry] = + leaves.filter(l => Tracking.Status.liveEntryStatuses.contains(l.tracking.status)) - /** The root manifest as a [[FileStatus]]. */ - private lazy val rootStatus: FileStatus = contentRoot.toFileStatus(tableRoot) + /** The tracking to inherit from. */ + private lazy val parentTrackingByManifestPath: Map[String, InheritableTracking] = + liveLeaves.map { leaf => + SparkPath.fromPath(leaf.getAbsolutePath(tableRoot)).urlEncoded -> InheritableTracking(leaf) + }.toMap + + /** Absolute [[Path]]s to the live leaf manifest parquet files, resolved against the root. */ + lazy val liveLeafManifestAbsolutePaths: Seq[Path] = liveLeaves.map(_.getAbsolutePath(tableRoot)) - /** The leaf manifests as [[FileStatus]]es. */ - private lazy val leafStatuses: Seq[FileStatus] = leaves.map(_.toFileStatus(tableRoot)) + /** The root manifest as a [[FileStatus]]. */ + private lazy val rootFile: FileStatus = contentRoot.toFileStatus(tableRoot) override def version: Long = checkpointAction.version @@ -83,7 +101,7 @@ final class AMTCheckpointProvider( } override def effectiveCheckpointSizeInBytes(): Long = - contentRoot.sizeInBytes + leaves.map(_.file_size_in_bytes).sum + contentRoot.sizeInBytes + liveLeaves.map(_.file_size_in_bytes).sum override def checkpointPolicyForLogging: Option[CheckpointPolicy.Policy] = None @@ -112,10 +130,6 @@ final class AMTCheckpointProvider( * The full action set of this checkpoint as a distributed [[Dataset]] of [[SingleAction]]: the * live file `AddFile`s reconstructed from the AMT(root + leaves), unioned with the inline * non-content actions (protocol, metadata, domain metadata, txns) built on the driver. - * - * Note: Iceberg metadata inheritance (manifest entries inheriting fields such as partition - * values, sequence numbers, or snapshot id from the parent manifest) is not supported yet; - * entries are read as fully materialized rows. */ private def allActions(spark: SparkSession, deltaLog: DeltaLog): Dataset[SingleAction] = { import org.apache.spark.sql.delta.implicits._ @@ -136,94 +150,364 @@ final class AMTCheckpointProvider( */ private def liveAddSingleActions( spark: SparkSession, deltaLog: DeltaLog): Dataset[SingleAction] = { - import org.apache.spark.sql.delta.implicits._ - // Bind to a local so the `mapPartitions` closure captures it, not the (non-serializable) - // provider. - val localTableRoot = tableRoot - val encodedRootPath = SparkPath.fromPath(rootManifestAbsolutePath).urlEncoded - val serializableConf = new SerializableConfiguration(deltaLog.newDeltaHadoopConf()) - - val files = rootStatus +: leafStatuses - val fmt = DeltaLogFileIndex.CHECKPOINT_FILE_FORMAT_PARQUET - // Read every leaf row, then drop the MDV-marked (leaf, rowIndex) entries with a filter on - // top, using a broadcast of each leaf's manifest DV bitmap bytes keyed by its `_metadata` - // file path. - val index = DeltaLogFileIndex(fmt, files.toArray) - val mdvByLeaf: Map[String, Array[Byte]] = leaves.flatMap { leaf => + val root = rootLiveAddSingleActions(deltaLog) + if (liveLeaves.isEmpty) root else root.union(liveLeafAddSingleActions(deltaLog)) + } + + /** + * Reconstructs root-resident AddFiles. Root entries have no parent and therefore no inheritance. + */ + private def rootLiveAddSingleActions(deltaLog: DeltaLog): Dataset[SingleAction] = { + val index = DeltaLogFileIndex( + DeltaLogFileIndex.CHECKPOINT_FILE_FORMAT_PARQUET, Array(rootFile)) + val entries = liveDataEntries(deltaLog, index) + toAddSingleActions(entries) + } + + /** Reconstructs the AddFiles of every live leaf, taking care of inheritance. */ + private def liveLeafAddSingleActions( + deltaLog: DeltaLog): Dataset[SingleAction] = { + val files = liveLeaves.map(_.toFileStatus(tableRoot)).toArray + val entries = liveDataEntriesWithRowIdPrefix(deltaLog, files, liveLeaves) + toAddSingleActions(entries) + } + + /** + * Reads all DATA entries, computes their row-ID prefix over each unfiltered leaf, then applies + * status and MDV filtering. The returned live entries retain the computed prefix. + */ + private def liveDataEntriesWithRowIdPrefix( + deltaLog: DeltaLog, + files: Array[FileStatus], + dvLeaves: Seq[DataManifestEntry]): Dataset[AMTDataEntryExtended] = { + val index = leafFileIndexWithManifestFilters(files, dvLeaves) + val withPrefix = loadEntriesWithExtendedMetadata( + deltaLog, index, checkpointAction.metaData, checkpointAction.protocol, + includeRowIndexFilterMarker = true) + .where(col("entry.content_type") === lit(AMTSingleAction.ContentType.Type.Data)) + .withColumn(PRECEDING_RECORDS_COLUMN, precedingNullRowIdRecords) + .where(col("entry.tracking.status").isin(Tracking.Status.liveEntryStatuses.toSeq: _*)) + asExtendedEntries(filterOutRowIndexFilterEntries(withPrefix)) + } + + /** + * Reads visible DATA entries when no row-ID inheritance is needed. + */ + private def liveDataEntries( + deltaLog: DeltaLog, index: DeltaLogFileIndex): Dataset[AMTDataEntryExtended] = + asExtendedEntries( + loadEntriesWithExtendedMetadata( + deltaLog, index, checkpointAction.metaData, checkpointAction.protocol) + .where(col("entry.content_type") === lit(AMTSingleAction.ContentType.Type.Data)) + .where(col("entry.tracking.status").isin(Tracking.Status.liveEntryStatuses.toSeq: _*)) + .withColumn(PRECEDING_RECORDS_COLUMN, lit(0L))) + + /** Builds a leaf index with bitmap-format-independent manifest filter providers. */ + protected def leafFileIndexWithManifestFilters( + files: Array[FileStatus], + dvLeaves: Seq[DataManifestEntry]): DeltaLogFileIndex = { + val perFileRowIndexFilters: Map[String, RowIndexFilterProvider] = dvLeaves.flatMap { leaf => leaf.manifestDV.map { case (dvBytes, _) => - SparkPath.fromPath(leaf.getAbsolutePath(localTableRoot)).urlEncoded -> dvBytes + leaf.getAbsolutePath(tableRoot).toString -> + AMTUtils.deserializeMdv(dvBytes).toRowIndexFilterProvider(tableRoot) } }.toMap - val mdvBroadcast = spark.sparkContext.broadcast(mdvByLeaf) - val dataEntries = AMTCheckpointProvider.loadEntriesWithLocation(spark, deltaLog, index) - .where(col("entry.content_type") === lit(AMTSingleAction.ContentType.Type.Data)) - .where(col("entry.tracking.status").isin( - AMTCheckpointProvider.liveTrackingStatuses.toSeq: _*)) - .filter { entryWithLoc => - mdvBroadcast.value.get(entryWithLoc.leafPath) - .forall(bytes => !RoaringBitmapArray.readFrom(bytes).contains(entryWithLoc.pos)) - } + DeltaLogFileIndex(new AMTParquetFileFormat, files, perFileRowIndexFilters) + } + + /** Extends the Parquet read schema with the keep/drop marker. */ + protected def rowIndexFilterReadSchema(persistedSchema: StructType): StructType = + persistedSchema.add(AMTParquetFileFormat.IS_ROW_DELETED_STRUCT_FIELD) + + /** Name of the keep/drop marker carried through manifest decoding. */ + protected def rowIndexFilterMarkerColumnName: String = + AMTParquetFileFormat.IS_ROW_DELETED_COLUMN_NAME + + /** Selects the keep/drop marker from the Parquet scan. */ + protected def rowIndexFilterMarkerColumn: Column = + col(rowIndexFilterMarkerColumnName) + + /** Drops rows marked deleted by the manifest DV, then removes the marker. */ + protected def filterOutRowIndexFilterEntries(dataFrame: DataFrame): DataFrame = + dataFrame + .where(col(AMTParquetFileFormat.IS_ROW_DELETED_COLUMN_NAME) === + lit(RowIndexFilter.KEEP_ROW_VALUE)) + .drop(AMTParquetFileFormat.IS_ROW_DELETED_COLUMN_NAME) - dataEntries - .mapPartitions { entries => - val fs = localTableRoot.getFileSystem(serializableConf.value) - entries.map { entryWithLoc => - entryWithLoc.entry.unwrap match { - case data: DataEntry => - val backReference = if (entryWithLoc.leafPath == encodedRootPath) { - None + /** + * Converts manifest entries into the `AddFile` actions of the reconstructed state. + */ + private def toAddSingleActions( + entries: Dataset[AMTDataEntryExtended]): Dataset[SingleAction] = { + import org.apache.spark.sql.delta.implicits._ + val localTableRoot = tableRoot + val encodedRootPath = SparkPath.fromPath(rootManifestAbsolutePath).urlEncoded + val parentTracking = parentTrackingByManifestPath + + entries.mapPartitions { iter => + iter.map { entryWithLoc => + entryWithLoc.entry.unwrap match { + case data: DataEntry => + val isRootEntry = entryWithLoc.leafPath == encodedRootPath + val backReference = if (isRootEntry) { + None + } else { + val absLeaf = SparkPath.fromUrlString(entryWithLoc.leafPath).toPath + val relManifest = + AMTUtils.relativizeLocation(localTableRoot.toString, absLeaf.toString) + Some(BackReference(relManifest, entryWithLoc.pos.toInt)) + } + val resolvedTracking = + if (isRootEntry) { + // Root-resident DATA entries have no parent and retain their persisted tracking. + data.tracking } else { - val absLeaf = SparkPath.fromUrlString(entryWithLoc.leafPath).toPath - val relManifest = - AMTUtils.relativizeManifestPathToTableRoot(fs, localTableRoot, absLeaf) - Some(BackReference(relManifest, entryWithLoc.pos)) + Tracking.resolve( + childTracking = data.tracking, + parentTracking = parentTracking(entryWithLoc.leafPath), + prefixSumRecordCountForNullFirstRowId = + entryWithLoc.precedingNullFirstRowIdRecords, + childEntryLocationForLogging = data.location) } - val add = data.toAddFile(localTableRoot).copy(backReference = backReference) - SingleAction(add = add) - case other => throw new IllegalStateException( - s"Expected a DATA entry after filtering, got ${other.getClass.getSimpleName}.") - } + val add = data.copy(tracking = resolvedTracking) + .toAddFile(localTableRoot) + .copy(backReference = backReference) + SingleAction(add = add) + case other => throw new IllegalStateException( + s"Expected a DATA entry after filtering, got ${other.getClass.getSimpleName}.") } } + } + } + + private def asExtendedEntries(df: DataFrame): Dataset[AMTDataEntryExtended] = { + implicit val encoder: Encoder[AMTDataEntryExtended] = + AMTCheckpointProvider.amtDataEntryExtendedEncoder + df.as[AMTDataEntryExtended] + } + + /** Name of the [[AMTDataEntryExtended.precedingNullFirstRowIdRecords]] column. */ + private val PRECEDING_RECORDS_COLUMN: String = "precedingNullFirstRowIdRecords" + + /** + * The `first_row_id` prefix sum of a manifest row: the total `record_count` of the entries that + * precede it in the same manifest, have `ADDED` status, and carry a null `first_row_id`. + */ + private def precedingNullRowIdRecords: Column = { + val precedingRowsOfManifest = Window + .partitionBy(col("leafPath")) + .orderBy(col("pos")) + .rowsBetween(Window.unboundedPreceding, -1) + val contribution = when( + col("entry.tracking.status") === lit(Tracking.Status.Added) && + col("entry.tracking.first_row_id").isNull, + col("entry.record_count")).otherwise(lit(0L)) + // The frame is empty for the first row of a manifest, where the sum is null and the offset 0. + coalesce(sum(contribution).over(precedingRowsOfManifest), lit(0L)) } /** - * Test-only invariant check for AMT back references. + * Test-only invariant: verify the AMT back references carried by the current proposed commit's + * file actions. + * + * On an AMT-backed table a leaf-resident file's AddFile / RemoveFile carries a [[BackReference]] + * to the (leaf manifest, row position) its entry occupies in the tree, so a later commit can mask + * or supersede that leaf slot; a root-resident file carries none. `committedActions` are the + * current proposed commit's actions; they are checked against the live set of the AMT checkpoint + * this (pre-commit) snapshot is backed by, keyed by (path, dv id): + * - a file whose (path, dv) is live in the AMT checkpoint leaf must carry the back reference; + * - a file whose (path, dv) is absent from the AMT checkpoint leaf must carry none -- a net-new + * file, or the re-added copy of a same-path replace (re-added under a new dv). + * A (path, dv) that an intermediate commit (landed after the AMT checkpoint but before this one) + * already superseded is relaxed: this commit's later add/remove of it may omit backreference. + * + * Example: the AMT checkpoint is at version 10 and commits 11/12/13 sit on top of it while this + * commit is 14. File f1 lives at leaf-1 / pos-1 in the checkpoint. If commit 12 (say an + * ANALYZE TABLE COMPUTE STATS) already re-committed f1 -- carrying its back reference at that + * point -- then f1's add/remove in commit 14 need not carry a back reference. */ private[delta] def verifyCommitBackReferences( spark: SparkSession, deltaLog: DeltaLog, + catalogTableOpt: Option[CatalogTable], committedActions: Seq[Action]): Unit = { - val committedFiles: Seq[(String, Option[BackReference])] = committedActions.collect { - case a: AddFile => a.path -> a.backReference - case r: RemoveFile => r.path -> r.backReference + // Key by (path, dv) so a same-path replace is handled: the removed (path, oldDv) is checked + // against the AMT, while the re-added (path, newDv) is a distinct key absent from the tree. + val committedFiles = committedActions.collect { + case a: AddFile => + a.toUniqueFileActionTuple(tableRoot, useObjectIdentity = true) -> a.backReference + case r: RemoveFile => + r.toUniqueFileActionTuple(tableRoot, useObjectIdentity = true) -> r.backReference } if (committedFiles.isEmpty) return - val expectedPathToBackreferenceMap: Map[String, Option[BackReference]] = - liveAddSingleActions(spark, deltaLog) - .collect() - .map(sa => sa.add.path -> sa.add.backReference) + val expectedKeyToBackreferenceMap = + liveAddSingleActions(spark, deltaLog).collect() + .map { singleAction => + singleAction.add.toUniqueFileActionTuple(tableRoot, useObjectIdentity = true) -> + singleAction.add.backReference + } .toMap - committedFiles.foreach { case (path, actual) => - expectedPathToBackreferenceMap.get(path) match { - case Some(expected) if actual != expected => + // Keys an intermediate commit (after this AMT) already re-committed. The first superseding + // add/remove must carry a back reference; a 2nd superseding one of the same key need not. + val intermediateCommittedKeys = + deltaLog.getChanges(checkpointVersion + 1, catalogTableOpt).flatMap(_._2).collect { + case a: AddFile => a.toUniqueFileActionTuple(tableRoot, useObjectIdentity = true) + case r: RemoveFile => r.toUniqueFileActionTuple(tableRoot, useObjectIdentity = true) + }.toSet + + committedFiles.foreach { case (key, actual) => + expectedKeyToBackreferenceMap.get(key) match { + case Some(expected) + if actual != expected && !(intermediateCommittedKeys.contains(key) && actual.isEmpty) => throw new IllegalStateException( - s"AMT back reference for file '$path' does not match the AMT. " + + s"AMT back reference for file '${key.fileURI}' does not match the AMT. " + s"Expected $expected but the committed action carried $actual.") case None if actual.isDefined => throw new IllegalStateException( - s"File '$path' carries a back reference $actual but is not present in the AMT " + - "tree, so it must not carry one.") - case _ => // Present and matching, or absent and empty: as expected. + s"File '${key.fileURI}' carries a back reference $actual but is not present in " + + "the AMT tree, so it must not carry one.") + case _ => // Matching, omitted after a window supersession, or absent+empty: as expected. } } } + + private def getBackreferencesForFilePaths( + spark: SparkSession, + deltaLog: DeltaLog, + filePaths: Set[String]): Map[UniqueFileActionTuple, Option[BackReference]] = + liveAddSingleActions(spark, deltaLog) + .where(col("add.path").isin(filePaths.toSeq: _*)) + .collect() + .map { singleAction => + singleAction.add.toUniqueFileActionTuple(tableRoot, useObjectIdentity = true) -> + singleAction.add.backReference + } + .toMap + + /** + * Re-derives file actions' back references to match this (the latest) tree. + * The actions which have backreferences corresponding to leaves which + * hasn't changed (Same old leaf is still present with the new tree and MDV + * also hasn't changed) doesn't need to be re-calculated as the old + * backreferences are still valid. + */ + private[delta] def reStampBackReferences( + spark: SparkSession, + deltaLog: DeltaLog, + actions: Seq[Action]): AMTCheckpointProvider.ReStampedBackReferences = { + def existingBackReference(action: Action): Option[BackReference] = action match { + case a: AddFile => a.backReference + case r: RemoveFile => r.backReference + case _ => None + } + def isFileAction(action: Action): Boolean = action match { + case _: AddFile | _: RemoveFile => true + case _ => false + } + // An `isMasked(pos)` test over each referenced leaf's manifest DV (a leaf with no DV masks + // nothing), keyed by the leaf `location` a back reference's `manifest` holds. Only the leaves + // some back reference points at are deserialized. + val referencedManifests = + actions.iterator.flatMap(existingBackReference).map(_.manifest).toSet + val isMaskedByLocation: Map[String, Int => Boolean] = liveLeaves.iterator + .filter(leaf => referencedManifests.contains(leaf.location)) + .map { leaf => + val isMasked: Int => Boolean = leaf.manifestDV match { + case Some((dvBytes, _)) => + val mdv = AMTUtils.deserializeMdv(dvBytes) + pos => mdv.contains(pos) + case None => _ => false + } + leaf.location -> isMasked + }.toMap + // A back reference is still valid iff its leaf is present in this tree and its position there + // is not MDV-masked. If an action has no back reference against the older tree, we recalculate + // it too, as the entry might have spilled from the root to a new leaf in the latest tree. + def stillValid(backReference: Option[BackReference]): Boolean = + backReference.exists { case BackReference(manifest, pos) => + isMaskedByLocation.get(manifest).exists(isMasked => !isMasked(pos)) + } + def needsReStamp(action: Action): Boolean = action match { + case a: AddFile => !stillValid(a.backReference) + case r: RemoveFile => !stillValid(r.backReference) + case _ => false + } + if (!actions.exists(needsReStamp)) { + return AMTCheckpointProvider.ReStampedBackReferences( + actions, + numActionsReusingBackref = actions.count(isFileAction), + numActionsRegeneratingBackref = 0) + } + val pathsToReStamp: Set[String] = actions.iterator.collect { + case a: AddFile if needsReStamp(a) => a.path + case r: RemoveFile if needsReStamp(r) => r.path + }.toSet + val keyToBackRef = getBackreferencesForFilePaths(spark, deltaLog, pathsToReStamp) + def backRefFor(action: FileAction): Option[BackReference] = + keyToBackRef.getOrElse( + action.toUniqueFileActionTuple(tableRoot, useObjectIdentity = true), None) + val restamped = actions.map { + case a: AddFile if needsReStamp(a) => a.copy(backReference = backRefFor(a)) + case r: RemoveFile if needsReStamp(r) => r.copy(backReference = backRefFor(r)) + case other => other + } + AMTCheckpointProvider.ReStampedBackReferences( + restamped, + numActionsReusingBackref = actions.count(a => isFileAction(a) && !needsReStamp(a)), + numActionsRegeneratingBackref = actions.count(needsReStamp)) + } + + /** + * Like [[AMTCheckpointProvider.loadEntries]], but also captures each row's physical location. + * The optional row-index-filter marker is carried through manifest decoding so callers can apply + * it after computing any row-ID prefix. + */ + private def loadEntriesWithExtendedMetadata( + deltaLog: DeltaLog, + index: DeltaLogFileIndex, + metadata: Metadata, + protocol: Protocol, + includeRowIndexFilterMarker: Boolean = false): DataFrame = { + import org.apache.spark.sql.delta.implicits._ + val persistedSchema = AMTSingleAction.persistedSchema(metadata, protocol) + val readSchema = + if (includeRowIndexFilterMarker) rowIndexFilterReadSchema(persistedSchema) + else persistedSchema + val markerColumns = + if (includeRowIndexFilterMarker) Seq(rowIndexFilterMarkerColumn) + else Seq.empty + val persisted = deltaLog.loadIndex(index, readSchema) + .select( + (persistedSchema.fieldNames.toIndexedSeq.map(col) :+ + col(s"$METADATA_NAME.$FILE_PATH").as("leafPath") :+ + col(s"$METADATA_NAME.${ParquetFileFormat.ROW_INDEX}").as("pos")) ++ markerColumns: _*) + val decodedMarkerColumns = + if (includeRowIndexFilterMarker) Seq(col(rowIndexFilterMarkerColumnName)) + else Seq.empty + val withPartition = AMTPartitionValues.forRead(persisted, metadata.partitionSchema) + AMTContentStats.forRead(withPartition, metadata, protocol) + .select( + (Seq( + struct(amtSingleActionEncoder.schema.fieldNames.toIndexedSeq.map(col): _*).as("entry"), + col("leafPath"), + col("pos")) ++ decodedMarkerColumns): _*) + } } object AMTCheckpointProvider { + /** + * @param actions the actions with re-derived back references where needed. + * @param numActionsReusingBackref file actions whose existing back reference stayed valid. + * @param numActionsRegeneratingBackref file actions whose back reference was re-derived. + */ + case class ReStampedBackReferences( + actions: Seq[Action], + numActionsReusingBackref: Int, + numActionsRegeneratingBackref: Int) + /** * An [[AMTSingleAction]] entry paired with its physical read location in its manifest parquet. * @@ -232,55 +516,61 @@ object AMTCheckpointProvider { * (Spark's `_metadata.file_path`). * @param pos The 0-based position of the entry inside the manifest (Spark's * `_metadata.row_index`). + * @param precedingNullFirstRowIdRecords The entry's `first_row_id` prefix sum. */ - case class AMTDataEntryWithLocation(entry: AMTSingleAction, leafPath: String, pos: Long) + case class AMTDataEntryExtended( + entry: AMTSingleAction, + leafPath: String, + pos: Long, + precedingNullFirstRowIdRecords: Long) - private lazy val amtDataEntryWithLocationEncoder: Encoder[AMTDataEntryWithLocation] = - new DeltaEncoder[AMTDataEntryWithLocation].get + private[amt] lazy val amtDataEntryExtendedEncoder: Encoder[AMTDataEntryExtended] = + new DeltaEncoder[AMTDataEntryExtended].get /** * Builds a provider from an emitted [[Checkpoint]] action by reading the leaf pointers out of the * root manifest parquet. * - * @param spark Active SparkSession used to read the root parquet. * @param deltaLog The table's DeltaLog, used to read the root via `loadIndex` (which bypasses * the path-based Delta format check the root file under the table root would * otherwise trip). * @param checkpoint The inline-emitted checkpoint action carrying the `contentRoot`. */ def fromCheckpoint( - spark: SparkSession, deltaLog: DeltaLog, - checkpoint: Checkpoint): AMTCheckpointProvider = { + checkpoint: Checkpoint, + manifestCommitVersion: Long): AMTCheckpointProvider = { val tableRoot = deltaLog.dataPath - val rootStatus = checkpoint.contentRoot.toFileStatus(tableRoot) + val rootFile = checkpoint.contentRoot.toFileStatus(tableRoot) val index = - DeltaLogFileIndex(DeltaLogFileIndex.CHECKPOINT_FILE_FORMAT_PARQUET, Array(rootStatus)) + DeltaLogFileIndex(DeltaLogFileIndex.CHECKPOINT_FILE_FORMAT_PARQUET, Array(rootFile)) // The root manifest is small (one row per leaf), so collect it to the driver to enumerate the // leaf pointers. - val leaves = loadEntries(spark, deltaLog, index).collect().toSeq - .filter(_.content_type == AMTSingleAction.ContentType.Type.DataManifest) - .map(_.unwrap.asInstanceOf[DataManifestEntry]) - new AMTCheckpointProvider(checkpointAction = checkpoint, leaves = leaves, tableRoot = tableRoot) + val leaves = + loadEntries(deltaLog, index, checkpoint.metaData, checkpoint.protocol).collect() + .toSeq + .filter(_.content_type == AMTSingleAction.ContentType.Type.DataManifest) + .map(_.unwrap.asInstanceOf[DataManifestEntry]) + new AMTCheckpointProvider( + manifestCommitVersion = manifestCommitVersion, + checkpointAction = checkpoint, + leaves = leaves, + tableRoot = tableRoot) } - /** Tracking Status representing the live [[DataEntry]] in an AMT. */ - private[amt] val liveTrackingStatuses: Set[Int] = - Set(Tracking.Status.Existing, Tracking.Status.Added) - /** Reads the AMT root and returns the live [[DataEntry]]s tracked by root. */ private[amt] def readLiveRootDataEntries( - spark: SparkSession, deltaLog: DeltaLog, checkpoint: Checkpoint): Seq[AddFile] = { val tableRoot = deltaLog.dataPath - val rootStatus = checkpoint.contentRoot.toFileStatus(tableRoot) + val rootFile = checkpoint.contentRoot.toFileStatus(tableRoot) val index = - DeltaLogFileIndex(DeltaLogFileIndex.CHECKPOINT_FILE_FORMAT_PARQUET, Array(rootStatus)) - loadEntries(spark, deltaLog, index).collect().toSeq + DeltaLogFileIndex(DeltaLogFileIndex.CHECKPOINT_FILE_FORMAT_PARQUET, Array(rootFile)) + loadEntries(deltaLog, index, checkpoint.metaData, checkpoint.protocol).collect() + .toSeq .filter(_.content_type == AMTSingleAction.ContentType.Type.Data) .map(_.unwrap.asInstanceOf[DataEntry]) - .filter(e => liveTrackingStatuses.contains(e.tracking.status)) + .filter(e => Tracking.Status.liveEntryStatuses.contains(e.tracking.status)) .map(_.toAddFile(tableRoot)) } @@ -289,29 +579,17 @@ object AMTCheckpointProvider { * [[AMTSingleAction]]. */ private def loadEntries( - spark: SparkSession, deltaLog: DeltaLog, - index: DeltaLogFileIndex): Dataset[AMTSingleAction] = { + index: DeltaLogFileIndex, + metadata: Metadata, + protocol: Protocol): Dataset[AMTSingleAction] = { import org.apache.spark.sql.delta.implicits._ - deltaLog.loadIndex(index, spark.emptyDataset[AMTSingleAction].schema).as[AMTSingleAction] + val persistedSchema = + AMTSingleAction.persistedSchema(metadata, protocol) + val persisted = deltaLog.loadIndex(index, persistedSchema) + val withPartition = AMTPartitionValues.forRead(persisted, metadata.partitionSchema) + AMTContentStats.forRead(withPartition, metadata, protocol) + .as[AMTSingleAction] } - /** - * Like [[loadEntries]], but also captures each row's physical read location. - */ - private def loadEntriesWithLocation( - spark: SparkSession, - deltaLog: DeltaLog, - index: DeltaLogFileIndex): Dataset[AMTDataEntryWithLocation] = { - import org.apache.spark.sql.delta.implicits._ - implicit val entryLocEncoder: Encoder[AMTDataEntryWithLocation] = - amtDataEntryWithLocationEncoder - val amtSchema = spark.emptyDataset[AMTSingleAction].schema - deltaLog.loadIndex(index, amtSchema) - .select( - struct(amtSchema.fieldNames.toIndexedSeq.map(col): _*).as("entry"), - col(s"$METADATA_NAME.$FILE_PATH").as("leafPath"), - col(s"$METADATA_NAME.${ParquetFileFormat.ROW_INDEX}").as("pos")) - .as[AMTDataEntryWithLocation] - } } diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/amt/AMTContentStats.scala b/spark/src/main/scala/org/apache/spark/sql/delta/amt/AMTContentStats.scala new file mode 100644 index 00000000000..7c490137248 --- /dev/null +++ b/spark/src/main/scala/org/apache/spark/sql/delta/amt/AMTContentStats.scala @@ -0,0 +1,431 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.amt + +import org.apache.spark.sql.delta.{DeltaColumnMapping, DeltaColumnMappingMode} +import org.apache.spark.sql.delta.actions.{Metadata, Protocol} +import org.apache.spark.sql.delta.stats.{DeltaStatistics, SkippingEligibleDataType, StatisticsCollection, StatsCollectionUtils} + +import org.apache.spark.sql.{Column, DataFrame, SparkSession} +import org.apache.spark.sql.execution.datasources.parquet.ParquetUtils +import org.apache.spark.sql.functions.{coalesce, col, from_json, lit, struct, to_json, when} +import org.apache.spark.sql.types.{BooleanType, ByteType, DataType, DateType, DecimalType, DoubleType, FloatType, IntegerType, LongType, MetadataBuilder, ShortType, StringType, StructField, StructType, TimestampNTZType, TimestampType} + +/** + * Converts between Delta's stats JSON string and Iceberg V4's typed `content_stats` struct + * (field 146). + * + * Iceberg models `content_stats` as a per-table struct-of-structs: one sub-struct per table field, + * named by that field's logical name (informational), holding the individual statistics as typed + * values. The sub-struct's Parquet field id is derived from the table field id, and each statistic + * sits at a fixed offset from it, so a reader resolves stats by id. + * + * A table column's Delta column-mapping id is its Iceberg field id, so no id translation is needed: + * in both `id` and `name` column-mapping mode Delta assigns every column a stable id and stamps it + * as the Parquet `field_id`. We use that Iceberg table column field id to compute the stats + * sub-struct's field ids. + * + * Only the statistics Delta actually collects get a slot: `null_value_count`, `lower_bound`, + * `upper_bound` and `tight_bounds`. The remaining spec-optional statistics (`value_count`, + * `nan_value_count`, `avg_value_size`, `max_value_size`) are omitted. + */ +private[amt] object AMTContentStats { + + /** Name of the [[AMTSingleAction]] field this object shapes. */ + private val CONTENT_STATS_FIELD = "content_stats" + + /** + * Start of the Iceberg stats field-id space, and the number of ids reserved per table field. + * A table field's stats sub-struct is stamped `STATS_ID_START + STATS_IDS_PER_FIELD * fieldId`, + * and each statistic within it sits at a fixed offset from that base. See + * https://iceberg.apache.org/spec/#content-stats. + */ + private val STATS_ID_START: Long = 10000L + private val STATS_IDS_PER_FIELD: Long = 200L + + /** + * Per-statistic offsets from a sub-struct's base id, as defined by the Iceberg V4 spec + * (https://iceberg.apache.org/spec/#content-stats). + */ + private val LOWER_BOUND_ID_OFFSET: Long = 1L + private val UPPER_BOUND_ID_OFFSET: Long = 2L + private val TIGHT_BOUNDS_ID_OFFSET: Long = 3L + private val NULL_VALUE_COUNT_ID_OFFSET: Long = 5L + + /** Names of the per-column statistics, as defined by the Iceberg V4 spec. */ + private val LOWER_BOUND = "lower_bound" + private val UPPER_BOUND = "upper_bound" + private val TIGHT_BOUNDS = "tight_bounds" + private val NULL_VALUE_COUNT = "null_value_count" + + /** + * One statistics-collected leaf column: + * - `fieldId`: the id Delta column mapping assigned to the column. Delta stamps it as the + * Parquet `field_id` and Iceberg resolves fields by id, so this one value is both the Delta + * column-mapping id and the column's Iceberg field id. + * - `name`: the logical column name (the underscore-joined path). Informational only, and not + * unique -- a top-level `a_b` and a nested `a`.`b` both flatten to `a_b` -- so it is not used + * to resolve the sub-struct; see `fieldName`. + * - `path`: the physical-name path to the column inside a parsed Delta stats struct (nested). + * - `boundType`: the column's Iceberg bound type, or None. + * + * `boundType` is None for a column Delta counts nulls for but collects no min/max on (an array + * or map, say, or a type with no Iceberg bound representation). Such a leaf still gets a + * `null_value_count`; its bound fields are simply omitted. + */ + private case class StatsLeaf( + fieldId: Long, + name: String, + path: Seq[String], + boundType: Option[DataType]) { + def hasBounds: Boolean = boundType.isDefined + + /** + * The sub-struct's field name in the persisted schema: the logical name with the (unique) + * field id attached. Iceberg resolves content_stats entries by field id and treats the name as + * informational. Attaching the field id -- unique per column -- keeps every sub-struct name + * distinct, so name-based resolution stays unambiguous when two logical names collide. + */ + def fieldName: String = s"${name}_$fieldId" + } + + /** + * A [[StatisticsCollection]] describing the stats this table collects. + */ + private def statisticsCollection( + sparkSession: SparkSession, + tableMetadata: Metadata, + tableProtocol: Protocol): StatisticsCollection = { + new StatisticsCollection { + override protected def spark: SparkSession = sparkSession + // We consume only `statCollectionPhysicalSchema`, which restricts to the configured indexed + // columns (statsColumnSpec's numIndexedCols / explicit stats columns) applied to these + // schemas. There is no separate stats or attribute projection to reproduce from just + // (Metadata, Protocol), so all three are the full data schema; the indexed-column subsetting + // still happens inside `getIndexedColumns`. + override def tableSchema: StructType = tableMetadata.dataSchema + override def outputTableStatsSchema: StructType = tableMetadata.dataSchema + override def outputAttributeSchema: StructType = tableMetadata.dataSchema + override val statsColumnSpec = + StatisticsCollection.configuredDeltaStatsColumnSpec(tableMetadata) + override def columnMappingMode: DeltaColumnMappingMode = tableMetadata.columnMappingMode + override protected def protocol: Protocol = tableProtocol + override protected def getDataSkippingStringPrefixLength: Int = + StatsCollectionUtils.getDataSkippingStringPrefixLength(sparkSession, tableMetadata) + } + } + + /** + * The statistics-collected leaf columns that each get a `content_stats` entry, taken from + * `statCollectionPhysicalSchema` and ordered by field id so the persisted schema is stable across + * writes. That schema is Delta's set of columns it collects per-file statistics for. + * This method recurses into structs (a struct column contributes its scalar leaves, not itself) + * and sets `boundType` only for types with an Iceberg bound; a boundless leaf (an array, say) + * gets a `null_value_count` only. + * + * Example `statCollectionPhysicalSchema`: + * name type {id, physical name} + * d_long long {1, col-1} + * d_str string {2, col-2} + * d_nested struct {3, col-3} + * d_arr array {5, col-5} + * + * The physical names form the path into a parsed Delta stats struct; the logical names name the + * sub-structs. This yields (d_nested contributes only its scalar leaf `inner`, not itself): + * StatsLeaf(1, "d_long", ["col-1"], Some(LongType)) + * StatsLeaf(2, "d_str", ["col-2"], Some(StringType)) + * StatsLeaf(4, "d_nested_inner", ["col-3","col-4"], Some(IntegerType)) // struct leaf + * StatsLeaf(5, "d_arr", ["col-5"], None) // array: null count + * + * `name` is the field's full logical name. It names the sub-struct (informational, per the spec) + * while the field id is what a reader resolves by; using the full path rather than the bare leaf + * name reduces name clashes when two structs share a leaf name. + */ + private def statsLeafColumns(statCollectionPhysicalSchema: StructType): Seq[StatsLeaf] = { + def collect(schema: StructType, prefix: Seq[String], names: Seq[String]): Seq[StatsLeaf] = + schema.fields.toSeq.flatMap { field => + val path = prefix :+ DeltaColumnMapping.getPhysicalName(field) + val namePath = names :+ field.name + field.dataType match { + case nested: StructType => collect(nested, path, namePath) + case dt => + val boundType = if (isSupportedBoundType(dt)) Some(dt) else None + Some(StatsLeaf( + fieldId = DeltaColumnMapping.getColumnId(field).toLong, + name = namePath.mkString("_"), + path = path, + boundType = boundType)) + } + } + collect(statCollectionPhysicalSchema, Nil, Nil).sortBy(_.fieldId) + } + + /** + * Whether a min/max bound of this type can be represented as a typed Iceberg value. + * + * Mirrors `IcebergStatsConverter.isMinMaxStatTypeSupported`, the authority for Delta-to-Iceberg + * bound conversion. It is restated rather than called because that object lives in the + * `scala-bazel` source tree and pulls in the shaded Iceberg library, neither of which this + * package can depend on. Types outside this set are left out of `content_stats`, matching that + * converter's behavior of ignoring unsupported stats. + */ + private def isSupportedBoundType(dataType: DataType): Boolean = + SkippingEligibleDataType(dataType) && (dataType match { + case _: StringType | _: IntegerType | _: FloatType | _: DoubleType | _: DecimalType | + _: BooleanType | _: DateType | _: TimestampType | _: TimestampNTZType | _: LongType | + _: ByteType | _: ShortType => true + case _ => false + }) + + /** + * Whether the type's stored bound is truncated. Delta prefix-truncates string bounds and rounds + * timestamp bounds down to milliseconds, so for those types the stored bound is a valid superset + * endpoint but not the exact min/max. Iceberg's `tight_bounds` (bounds equal the min/max) must + * therefore be false for them even when Delta's file-level `tightBounds` is true. That flag + * describes only the non-truncated columns (everything except String/Timestamp/TimestampNTZ); + * the truncated columns are never tight regardless of it. + */ + private def isTruncatedType(dataType: DataType): Boolean = dataType match { + case _: StringType | _: TimestampType | _: TimestampNTZType => true + case _ => false + } + + /** + * The persisted `content_stats` struct for a table: one sub-struct per statistics-collected leaf + * column, named by that column's logical name. Empty when the table collects no per-column stats, + * in which case `AMTSingleAction.persistedSchema` drops the whole `content_stats` field (a struct + * with no fields is not Parquet-writable) -- the same way it drops `partition` when the table is + * unpartitioned. + */ + def persistedSchema(metadata: Metadata, protocol: Protocol): StructType = { + val physicalSchema = + statisticsCollection(SparkSession.active, metadata, protocol).statCollectionPhysicalSchema + val perColumn = statsLeafColumns(physicalSchema).map { leaf => + val baseId = STATS_ID_START + STATS_IDS_PER_FIELD * leaf.fieldId + // A leaf with no bounds gets only its null count; omitting the bound fields keeps them out + // of the Parquet schema rather than persisting columns that are always null. Fields are + // ordered by ascending id (bounds 1-3, then null count 5). + val boundFields = leaf.boundType.toSeq.flatMap { boundType => + Seq( + statField(LOWER_BOUND, boundType, baseId + LOWER_BOUND_ID_OFFSET), + statField(UPPER_BOUND, boundType, baseId + UPPER_BOUND_ID_OFFSET), + statField(TIGHT_BOUNDS, BooleanType, baseId + TIGHT_BOUNDS_ID_OFFSET)) + } + val nullCountField = + statField(NULL_VALUE_COUNT, LongType, baseId + NULL_VALUE_COUNT_ID_OFFSET) + // The sub-struct is named by the column's logical name with its field id attached (see + // StatsLeaf.fieldName) and stamped with the base id. Per the spec, stats are resolved by id; + // the name is informational, but the id suffix keeps it unique for Spark's name-based access. + statField(leaf.fieldName, StructType(boundFields :+ nullCountField), baseId) + } + StructType(perColumn) + } + + /** + * Builds one id-stamped field of a per-column stats sub-struct. Every field is nullable: a + * statistic can be absent for a given file (a column with no min/max, or a null bound), and the + * Iceberg V4 spec marks these per-column statistics optional. + */ + private def statField(name: String, dataType: DataType, id: Long): StructField = + StructField( + name, + dataType, + nullable = true, + metadata = new MetadataBuilder().putLong(ParquetUtils.FIELD_ID_METADATA_KEY, id).build()) + + /** + * Delta's stats JSON string -> the typed `content_stats` struct. + * + * `df` is a DataFrame of [[AMTSingleAction]] rows whose `content_stats` column still holds the + * encoder's shape (the raw Delta stats JSON). An entry with no statistics gets a null struct. + */ + def forWrite(df: DataFrame, metadata: Metadata, protocol: Protocol): DataFrame = { + val stats = statisticsCollection(df.sparkSession, metadata, protocol) + val statsSchema = stats.statsSchema // Delta's stats_parsed schema + val leaves = statsLeafColumns(stats.statCollectionPhysicalSchema) + // No per-column stats -> `content_stats` is dropped from the persisted schema (an empty struct + // is not Parquet-writable), so drop the column here too. Mirrors `AMTPartitionValues.forWrite` + // for an unpartitioned table. + if (leaves.isEmpty) return df.drop(CONTENT_STATS_FIELD) + // `content_stats` holds the raw Delta stats JSON string (the `Option[String]` encoder shape). + val rawJson = col(CONTENT_STATS_FIELD) + val statsParsedExpression = from_json(rawJson, statsSchema) + + // Delta's `tightBounds` is a per-file flag; Iceberg's `tight_bounds` is per column and asserts + // the bounds equal the min/max. The Delta spec treats an absent `tightBounds` as tight, so + // the default is true. + val fileTightBounds = + coalesce(statsParsedExpression.getField(DeltaStatistics.TIGHT_BOUNDS), lit(true)) + + val perColumn = leaves.map { leaf => + // Ordered by ascending id (bounds, then null count) to match the persisted schema. + val boundColumns = leaf.boundType.toSeq.flatMap { boundType => + // A column's bounds are exact only when the file is tight AND the type is not truncated, so + // a truncated column (string/timestamp) is never `tight_bounds = true`. + val columnTightBounds = + if (isTruncatedType(boundType)) lit(false) else fileTightBounds + Seq( + statPath(statsParsedExpression, DeltaStatistics.MIN, leaf.path).as(LOWER_BOUND), + statPath(statsParsedExpression, DeltaStatistics.MAX, leaf.path).as(UPPER_BOUND), + columnTightBounds.as(TIGHT_BOUNDS)) + } + val nullValueCount = statPath(statsParsedExpression, DeltaStatistics.NULL_COUNT, leaf.path) + .cast(LongType).as(NULL_VALUE_COUNT) + struct(boundColumns :+ nullValueCount: _*).as(leaf.fieldName) + } + + val contentStatsSchema = AMTContentStats.persistedSchema(metadata, protocol) + val contentStats = when(rawJson.isNull, lit(null).cast(contentStatsSchema)) + .otherwise(struct(perColumn: _*)) + replaceContentStats(df, contentStats) + } + + /** + * The typed `content_stats` struct -> the raw Delta stats JSON string the `Option[String]` field + * holds. + * + * `numRecords` is not persisted in `content_stats` (the entry carries it as Iceberg field 103 + * `record_count`); for an entry that has per-column stats it is folded back in here. An entry + * with no per-column stats (a table that collects none, a data file written without stats, or + * a manifest pointer) reconstructs to `null`; [[toStatsJson]] supplies its record count. + */ + def forRead(df: DataFrame, metadata: Metadata, protocol: Protocol): DataFrame = { + val stats = statisticsCollection(df.sparkSession, metadata, protocol) + val leaves = statsLeafColumns(stats.statCollectionPhysicalSchema) + val numRecords = col("record_count").cast(LongType).as(DeltaStatistics.NUM_RECORDS) + val jsonStats = if (leaves.isEmpty) { + // No per-column stats were persisted (`content_stats` was dropped from the schema); the + // encoder decodes `None` and `toStatsJson` supplies the record count. + lit(null).cast(StringType) + } else { + val typedStats = col(CONTENT_STATS_FIELD) + + // Rebuild the nested minValues/maxValues/nullCount shape from each leaf's recorded path, then + // serialize it back to the JSON string Delta expects. minValues/maxValues cover only the + // leaves that carry bounds, matching the sub-schemas Delta itself builds. + val boundLeaves = leaves.filter(_.hasBounds) + // Recover Delta's file-level `tightBounds`. On write each untruncated column carries + // `tight_bounds = fileTightBounds`; truncated columns are forced to `false` for Iceberg's + // exact-bounds semantics. So AND only the untruncated columns; if a file has none, fall back + // to `false` conservatively. + val tightBoundLeaves = boundLeaves.filter(_.boundType.exists(bt => !isTruncatedType(bt))) + val recoveredTightBounds = + if (tightBoundLeaves.nonEmpty) { + tightBoundLeaves + .map(leaf => coalesce(statOf(typedStats, leaf, TIGHT_BOUNDS), lit(false))) + .reduce(_ && _) + } else { + lit(false) + } + val statsFields = + boundLeaves.headOption.toSeq.flatMap { _ => + Seq( + nestByPath(boundLeaves, statOf(typedStats, _, LOWER_BOUND)).as(DeltaStatistics.MIN), + nestByPath(boundLeaves, statOf(typedStats, _, UPPER_BOUND)).as(DeltaStatistics.MAX)) + } ++ leaves.headOption.toSeq.map { _ => + nestByPath(leaves, statOf(typedStats, _, NULL_VALUE_COUNT)).as(DeltaStatistics.NULL_COUNT) + } :+ recoveredTightBounds.as(DeltaStatistics.TIGHT_BOUNDS) + val statsJson = to_json(struct(numRecords +: statsFields: _*)) + when(typedStats.isNull, lit(null).cast(StringType)).otherwise(statsJson) + } + if (df.columns.contains(CONTENT_STATS_FIELD)) { + replaceContentStats(df, jsonStats) + } else { + df.withColumn(CONTENT_STATS_FIELD, jsonStats) + } + } + + /** + * Projects one statistic of one leaf column out of a parsed Delta stats struct (used on write), + * following the leaf's nested physical-name path. + * + * Example: statPath(parsed, "minValues", ["col-3","col-4"]) => parsed.minValues.col-3.col-4. + */ + private def statPath(parsed: Column, statName: String, path: Seq[String]): Column = + path.foldLeft(parsed.getField(statName))((column, name) => column.getField(name)) + + /** + * Reads one statistic out of a leaf's typed, flat `content_stats` sub-struct (used on read) -- + * the inverse shape of [[statPath]]. The sub-struct is keyed by the leaf's unique `fieldName`. + * + * Example: statOf(typed, leaf(fieldName="d_nested_inner_4"), "lower_bound") reads the + * `lower_bound` of the sub-struct field named "d_nested_inner_4". + */ + private def statOf(typed: Column, leaf: StatsLeaf, statName: String): Column = + typed.getField(leaf.fieldName).getField(statName) + + /** + * Rebuilds Delta's nested stats shape from the flat leaves, grouping by each leaf's recorded path + * so a struct column's per-field statistics land back under that struct. + * + * Example -- rebuilding `minValues` from the three bound leaves below, where `value` reads + * each leaf's `lower_bound` out of the typed `content_stats` sub-struct (e.g. value of leaf id 1 + * = typed.`1`.lower_bound): + * + * StatsLeaf(1, ["col-1"]) + * StatsLeaf(2, ["col-2"]) + * StatsLeaf(4, ["col-3","col-4"]) + * + * Group by the first path segment -> col-1, col-2, col-3. col-1 and col-2 have a length-1 path so + * they take their value directly; col-3 has no length-1 entry, so it recurses on the remaining + * path ["col-4"] and builds a sub-struct. The result is: + * + * struct( + * "col-1" -> typed.`1`.lower_bound, + * "col-2" -> typed.`2`.lower_bound, + * "col-3" -> struct("col-4" -> typed.`4`.lower_bound)) + * + * which `to_json`s to {"col-1":.., "col-2":.., "col-3":{"col-4":..}} -- the physical-keyed, + * nested shape Delta writes minValues/maxValues/nullCount in, with col-4 back under col-3. + */ + private def nestByPath(leaves: Seq[StatsLeaf], value: StatsLeaf => Column): Column = { + def build(entries: Seq[(Seq[String], StatsLeaf)]): Column = { + val fields = entries + .groupBy(_._1.head) + .toSeq + .sortBy { case (_, group) => group.map(_._2.fieldId).min } + .map { case (name, group) => + group.find(_._1.length == 1) match { + case Some((_, leaf)) => value(leaf).as(name) + case None => build(group.map { case (path, leaf) => (path.tail, leaf) }).as(name) + } + } + struct(fields: _*) + } + build(leaves.map(leaf => (leaf.path, leaf))) + } + + private def replaceContentStats(df: DataFrame, contentStats: Column): DataFrame = + df.select(df.columns.map { name => + if (name == CONTENT_STATS_FIELD) contentStats.as(name) else col(name) + }.toIndexedSeq: _*) + + /** + * The value stored in the `content_stats` field: the `AddFile.stats` JSON string verbatim, or + * None when the file has none. + */ + def fromStatsJson(statsJson: String): Option[String] = + Option(statsJson).filter(_.nonEmpty) + + /** + * The `AddFile.stats` JSON for a data entry. [[forRead]] rebuilds `content_stats` as the complete + * stats JSON (including `numRecords`), so it is returned as-is; the fallback covers an entry with + * no `content_stats` by emitting the record count alone. The inverse of [[fromStatsJson]]. + */ + def toStatsJson(contentStats: Option[String], recordCount: Long): String = + contentStats.getOrElse(s"""{"${DeltaStatistics.NUM_RECORDS}":$recordCount}""") +} diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/amt/AMTParquetFileFormat.scala b/spark/src/main/scala/org/apache/spark/sql/delta/amt/AMTParquetFileFormat.scala new file mode 100644 index 00000000000..ded954d42ad --- /dev/null +++ b/spark/src/main/scala/org/apache/spark/sql/delta/amt/AMTParquetFileFormat.scala @@ -0,0 +1,149 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.spark.sql.delta.amt + +import scala.util.control.NonFatal + +import org.apache.spark.sql.delta.RowIndexFilterProvider +import org.apache.spark.sql.delta.{DeltaLogFileIndex, DeltaParquetWriteSupport, RowIndexFilterMetadataColumnUtils} +import org.apache.spark.sql.delta.RowIndexFilterMetadataColumnUtils.ColumnMetadata +import org.apache.spark.sql.delta.deletionvectors.KeepAllRowsFilter +import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.fs.Path +import org.apache.hadoop.mapreduce.Job +import org.apache.parquet.hadoop.ParquetOutputFormat +import org.apache.parquet.hadoop.util.ContextUtil + +import org.apache.spark.SparkException +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.execution.datasources.{OutputWriterFactory, PartitionedFile} +import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.sources.Filter +import org.apache.spark.sql.types.{ByteType, StructField, StructType} +import org.apache.spark.util.SerializableConfiguration + +/** + * Parquet file format for AMT manifests (root and leaves). + * + * Write side: AMT manifests are Iceberg-V4 manifests, so [[prepareWrite]] always writes them with + * the Iceberg-compatible Parquet settings. + * + * Read side: exposes Manifest-DV membership without dropping rows. AMT must compute row-ID + * inheritance over every physical entry before applying the Manifest DV. Consequently this format + * keeps files unsplit, disables pushed filters, and materializes a marker that the AMT checkpoint + * provider filters only after computing the inheritance prefix. + */ +private[delta] class AMTParquetFileFormat extends ParquetFileFormat { + import AMTParquetFileFormat._ + + // Set isSplitable to false so `first_row_id` inheritance uses the complete `record_count` + // prefix sum of preceding entries with null `first_row_id` in the same leaf. + override def isSplitable( + sparkSession: SparkSession, + options: Map[String, String], + path: Path): Boolean = false + + override def prepareWrite( + sparkSession: SparkSession, + job: Job, + options: Map[String, String], + dataSchema: StructType): OutputWriterFactory = { + val factory = super.prepareWrite(sparkSession, job, options, dataSchema) + // AMT manifests are Iceberg-V4 manifests. Iceberg requires timestamps as int64 + // TIMESTAMP(MICROS); Spark's default is INT96. + ContextUtil.getConfiguration(job).set( + SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key, + SQLConf.ParquetOutputTimestampType.TIMESTAMP_MICROS.toString) + // Write list-element / map key-value field ids (carried on the schema via + // `parquet.field.nested.ids`), which the stock `ParquetWriteSupport` omits. + ParquetOutputFormat.setWriteSupportClass(job, classOf[DeltaParquetWriteSupport]) + factory + } + + + override def buildReaderWithPartitionValues( + sparkSession: SparkSession, + dataSchema: StructType, + partitionSchema: StructType, + requiredSchema: StructType, + filters: Seq[Filter], + options: Map[String, String], + hadoopConf: Configuration): PartitionedFile => Iterator[InternalRow] = { + val parquetDataReader = super.buildReaderWithPartitionValues( + sparkSession, + dataSchema, + partitionSchema, + requiredSchema, + // Filtering before row-ID inheritance would change the prefix assigned to later entries. + filters = Seq.empty, + options = options, + hadoopConf = hadoopConf) + + val schemaWithIndices = requiredSchema.fields.zipWithIndex + def findColumn(name: String): Option[ColumnMetadata] = { + val matches = schemaWithIndices.filter(_._1.name == name) + require(matches.length <= 1, + s"More than one column with name '$name' was requested from the AMT reader") + matches.headOption.map { case (field, index) => ColumnMetadata(index, field) } + } + + val markerColumnOpt = findColumn(IS_ROW_DELETED_COLUMN_NAME) + if (markerColumnOpt.isEmpty) return parquetDataReader + + val rowIndexColumn = findColumn(ParquetFileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME).getOrElse { + throw SparkException.internalError( + "AMT Manifest-DV reads require the Parquet row-index metadata column") + } + val serializableHadoopConf = new SerializableConfiguration(hadoopConf) + val useOffHeapBuffers = sparkSession.sessionState.conf.offHeapColumnVectorEnabled + + (partitionedFile: PartitionedFile) => { + val parquetIterator = parquetDataReader(partitionedFile) + try { + val rowIndexFilter = partitionedFile.otherConstantMetadataColumnValues + .get(DeltaLogFileIndex.ROW_INDEX_FILTER_PROVIDER_METADATA_KEY) + .map(_.asInstanceOf[RowIndexFilterProvider].retrieve(serializableHadoopConf.value)) + .getOrElse(KeepAllRowsFilter) + RowIndexFilterMetadataColumnUtils.iteratorWithAdditionalMetadataColumns( + parquetIterator, + Some(rowIndexFilter), + markerColumnOpt, + Some(rowIndexColumn), + useOffHeapBuffers, + useMetadataRowIndex = true).asInstanceOf[Iterator[InternalRow]] + } catch { + case NonFatal(e) => + parquetIterator match { + case resource: AutoCloseable => + RowIndexFilterMetadataColumnUtils.closeQuietly(resource) + case _ => // Nothing to close. + } + throw e + } + } + } +} + +private[delta] object AMTParquetFileFormat { + val IS_ROW_DELETED_COLUMN_NAME = "__delta_internal_is_row_deleted" + val IS_ROW_DELETED_STRUCT_FIELD = StructField(IS_ROW_DELETED_COLUMN_NAME, ByteType) + + +} diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/amt/AMTPartitionValues.scala b/spark/src/main/scala/org/apache/spark/sql/delta/amt/AMTPartitionValues.scala new file mode 100644 index 00000000000..6190220538f --- /dev/null +++ b/spark/src/main/scala/org/apache/spark/sql/delta/amt/AMTPartitionValues.scala @@ -0,0 +1,129 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.amt + +import org.apache.spark.sql.delta.ClassicColumnConversions._ +import org.apache.spark.sql.delta.DeltaColumnMapping +import org.apache.spark.sql.delta.util.PartitionUtils + +import org.apache.spark.sql.{Column, DataFrame} +import org.apache.spark.sql.catalyst.expressions.Cast +import org.apache.spark.sql.execution.datasources.parquet.ParquetUtils +import org.apache.spark.sql.functions.{array, col, element_at, lit, map_from_arrays, struct, when} +import org.apache.spark.sql.types.{MapType, MetadataBuilder, StringType, StructType} + +/** + * Converts between Delta's physical-name string map and Iceberg's typed partition struct. + * + * TODO(v4amt): shrink this converter. It exists because the two sides disagree on + * representation: Delta stores partition values as `AddFile.partitionValues`, a physical-name to + * string map, while the Iceberg V4 `partition` field is a struct with one typed field per + * partition column. + * + * https://github.com/delta-io/delta/issues/6953 proposes typing partitionValues in the Delta log + * itself. That removes [[forWrite]]'s per-field cast, since the source would already be typed, + * but it does not remove [[forRead]]'s render: log replay materializes an in-memory `AddFile`, + * whose `partitionValues` is a `Map[String, String]`, so something still has to turn the typed + * struct back into strings. Retiring the converter outright needs that field typed as well, not + * just the log format. + */ +private[amt] object AMTPartitionValues { + + /** Iceberg assigns partition-field ids from 1000, in partition-spec order. */ + private val PARTITION_FIELD_ID_START: Long = 1000L + + /** + * The partition struct as persisted: every column keeps its real type and logical name, with + * Iceberg partition-field ids stamped from 1000 in partition-spec order. + */ + def persistedSchema(partitionSchema: StructType): StructType = + StructType(partitionSchema.fields.zipWithIndex.map { case (field, ordinal) => + field.copy( + nullable = true, + metadata = new MetadataBuilder() + .putLong(ParquetUtils.FIELD_ID_METADATA_KEY, PARTITION_FIELD_ID_START + ordinal) + .build()) + }) + + /** + * Physical-name string map -> persisted partition struct. + * + * The struct's fields carry the partition columns' *logical* names, as Iceberg expects. That is + * safe because an Iceberg reader resolves partition fields by field ID (stamped from 1000 in + * partition-spec order, see `AMTSingleAction.persistedSchema`), not by name, so the names here + * are informational and a column rename does not change how the struct is read. + */ + def forWrite(df: DataFrame, partitionSchema: StructType): DataFrame = { + if (partitionSchema.isEmpty) return df.drop("partition") + val raw = col("partition") + // String -> typed mirrors what UniForm's Iceberg conversion does when it parses partition + // values out of a directory path: `PartitionUtils.parsePartitionValue` casts the string to the + // partition column's type. The cast is ANSI here so an unparseable value fails the write rather + // than silently writing a null partition value into the manifest. + val typedFields = partitionSchema.map { field => + val physicalName = DeltaColumnMapping.getPhysicalName(field) + Column(Cast( + element_at(raw, lit(physicalName)).expr, + field.dataType, + ansiEnabled = true)).as(field.name) + } + val persistedPartitionSchema = AMTPartitionValues.persistedSchema(partitionSchema) + val partition = when(raw.isNull, lit(null).cast(persistedPartitionSchema)) + .otherwise(struct(typedFields: _*)) + replacePartition(df, partition) + } + + /** Persisted partition struct -> physical-name string map. */ + def forRead(df: DataFrame, partitionSchema: StructType): DataFrame = { + val values = if (partitionSchema.isEmpty) { + lit(null).cast(MapType(StringType, StringType, valueContainsNull = true)) + } else { + val typedPartition = col("partition") + val keys = array(partitionSchema.map(f => lit(DeltaColumnMapping.getPhysicalName(f))): _*) + // typed -> String reuses the renderer UniForm writes its partition values with: + // `ConvertUtils` calls `PartitionUtils.literalToNormalizedString` on each parsed literal. + // `expressionToNormalizedString` is that function in expression form, so a value converted + // here is byte-identical to what Delta would have logged for it. + // + // AMT always renders a timestamp UTC-normalized, rather than following + // `write.utcTimestampPartitionValues`. That conf exists to preserve a legacy write format, + // and it is a session conf the log does not record per file -- so honoring it here would make + // reading a manifest depend on the session doing the read, and a manifest written by one + // session unreadable by another. A UTC-normalized timestamp also needs no time zone to + // interpret, which is why `timeZoneId` is left unset: it would only be consulted on the + // branch this never takes. + val stringValues = array(partitionSchema.map { field => + Column(PartitionUtils.expressionToNormalizedString( + value = typedPartition.getField(field.name).expr, + dataType = field.dataType, + timeZoneId = None, + useUtcNormalizedTimestamp = true)) + }: _*) + when(typedPartition.isNull, lit(null)).otherwise(map_from_arrays(keys, stringValues)) + } + if (df.columns.contains("partition")) { + replacePartition(df, values) + } else { + df.withColumn("partition", values) + } + } + + private def replacePartition(df: DataFrame, partition: Column): DataFrame = + df.select(df.columns.map { name => + if (name == "partition") partition.as(name) else col(name) + }: _*) +} diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/amt/AMTUsageLogs.scala b/spark/src/main/scala/org/apache/spark/sql/delta/amt/AMTUsageLogs.scala index 5bbb55a8ec3..49113d96c54 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/amt/AMTUsageLogs.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/amt/AMTUsageLogs.scala @@ -19,11 +19,63 @@ package org.apache.spark.sql.delta.amt /** Usage logs emitted by the AMT (`adaptiveMetadata-preview`) code paths. */ object AMTUsageLogs { /** Common prefix for all AMT usage logs. */ - val PREFIX = "delta.v4amt" + val PREFIX = "delta.amt" /** * Usage log emitted when [[Snapshot.lastManifestCommitOpt]] reads `CommitInfo` as a fallback. */ val LAST_MANIFEST_COMMIT_READ_FROM_COMMIT_INFO = s"$PREFIX.lastManifestCommit.readFromCommitInfo" + + /** Usage log emitted when a losing full AMT OPTIMIZE checkpoint is retried. */ + val CHECKPOINT_FULL_REGENERATE_RETRY = s"$PREFIX.checkpoint.fullRegenerateRetry" + + /** Usage log emitted for each AMT conflict-resolution round. */ + val CONFLICT_RESOLUTION_ROUND = s"$PREFIX.conflictResolutionRound" + + /** Usage log emitted when an AMT write fails unexpectedly. */ + val WRITE_FAILED = s"$PREFIX.writeFailed" + + // ////////////////////// Invariant check usage log suffixes ////////////////////// + + // Prefix for all of them is "delta.assert." + + // A manifest_info contains both live files and tombstones. + val ALERT_MIXED_LEAF_CONTENT = "amt.mixedLeafContent" + + // A leaf without live files gains new manifest deletion vector positions. + val ALERT_MDV_WITHOUT_LIVE_FILES = "amt.commit.mdvWithoutLiveFiles" + + // A removed file without a back reference remains in the live set. + // This usually happens when the same (file, DV) is removed and added in one commit. + val ALERT_NO_BACKREF_REMOVE_STILL_LIVE = + "amt.commit.noBackrefRemoveStillLive" + + // A removed file without a back reference has no originating add action in the root or + // intermediate commits. + val ALERT_NO_BACKREF_REMOVE_MISSING_ADD = + "amt.commit.noBackrefRemoveMissingAdd" + + // A data-changing commit re-adds an already-live file. + val ALERT_DATA_CHANGE_READD = "amt.commit.dataChangeReAdd" + + // A dataChange=false compaction with remove also re-adds an already-live file. + val ALERT_COMPACTION_READD = "amt.commit.compactionReAdd" + + // A metadata refresh dataChange=false commit introduces a new file. + val ALERT_METADATA_REFRESH_DATA_CHANGE_FALSE_ADDS_NEW_FILE = + "amt.commit.metadataRefreshDataChangeFalseAddsNewFile" + + // A leaf entry has an unknown tracking status. + val ALERT_UNEXPECTED_LEAF_TRACKING_STATUS = + "amt.unexpectedLeafTrackingStatus" + + // A manifest deletion vector has inconsistent bytes and cardinality. + val ALERT_MALFORMED_MANIFEST_DV = "amt.malformedManifestDV" + + // Alert raised when a base-preserving winning commit's Add/Remove file carries a new commit + // sequence number (defaultRowCommitVersion newer than the losing full checkpoint's version) + // yet a non-empty back reference into the base tree -- contradictory signals. + val ALERT_FILE_CONTAINS_NEW_SEQ_NUMBERS_BUT_NON_EMPTY_BACKREFERENCE = + "amt.fileContainsNewSeqNumbersButNonEmptyBackreference" } diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/amt/AMTUtils.scala b/spark/src/main/scala/org/apache/spark/sql/delta/amt/AMTUtils.scala index 814e063a416..43988b543f0 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/amt/AMTUtils.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/amt/AMTUtils.scala @@ -16,9 +16,11 @@ package org.apache.spark.sql.delta.amt -import org.apache.spark.sql.delta.deletionvectors.{RoaringBitmapArray, RoaringBitmapArrayFormat} -import org.apache.spark.sql.delta.util.DeltaFileOperations -import org.apache.hadoop.fs.{FileSystem, Path} +import org.apache.spark.sql.delta.{AdaptiveMetadataTableFeature, CurrentTransactionInfo, DeltaIllegalStateException, DeltaLog, Snapshot, SnapshotDescriptor, WinningCommitSummary} +import org.apache.spark.sql.delta.actions.{LastManifestCommit, Metadata, Protocol} +import org.apache.spark.sql.delta.deletionvectors.ManifestBitmap +import org.apache.spark.sql.delta.metering.DeltaLogging +import org.apache.hadoop.fs.Path /** * Path helpers for AMT (Adaptive Metadata Tree) manifest files. @@ -28,15 +30,103 @@ import org.apache.hadoop.fs.{FileSystem, Path} * file lives under it and resolved back by string concatenation (`tableRoot + "/" + relative`). * This differs from Delta's `AddFile.path`, which is URL-encoded. */ -object AMTUtils { +object AMTUtils extends DeltaLogging { + /** + * Whether AMT (Adaptive Metadata Tree) writes are enabled for a table with this `protocol` and + * `metadata`. + */ + def amtEnabled(metadata: Metadata, protocol: Protocol): Boolean = + protocol.isFeatureSupported(AdaptiveMetadataTableFeature) + + /** Whether AMT writes are enabled for `snapshot`. */ + def amtEnabled(snapshot: SnapshotDescriptor): Boolean = + amtEnabled(snapshot.metadata, snapshot.protocol) + + /** Logs and throws an [[IllegalStateException]] when an AMT invariant does not hold. */ + def invariantCheckWithLogging( + checkInvariant: => Boolean, + opTypeSuffix: String, + message: String, + deltaLog: DeltaLog = null, + data: Map[String, Any] = Map.empty): Unit = { + if (!checkInvariant) { + val stackTrace = Thread.currentThread().getStackTrace.drop(2).take(10).mkString("\n\t") + deltaAssertAndThrow( + check = false, + name = opTypeSuffix, + msg = message, + throwable = new DeltaIllegalStateException( + errorClass = "INTERNAL_ERROR", + messageParameters = Array(message)), + deltaLog = deltaLog, + data = data ++ Map("message" -> message, "stackTrace" -> stackTrace)) + } + } + + private val PathSeparator = "/" + + /** + * Returns true if the location contains a URI scheme, per RFC 3986 section 3.1. + * https://datatracker.ietf.org/doc/html/rfc3986#section-3.1 + */ + private[amt] def hasScheme(location: String): Boolean = { + var i = 0 + while (i < location.length) { + val ch = location.charAt(i) + if (ch == ':') { + return i > 0 + } + if (!isSchemeChar(ch, i)) { + return false + } + i += 1 + } + false + } + + private def isSchemeChar(ch: Char, position: Int): Boolean = { + (ch >= 'a' && ch <= 'z') || + (ch >= 'A' && ch <= 'Z') || + (position > 0 && ((ch >= '0' && ch <= '9') || ch == '+' || ch == '-' || ch == '.')) + } /** - * Relativizes an AMT manifest file `path` against `tableRoot`, returning the raw - * (non-URL-encoded) string to store in a manifest `location` / `contentRoot.path`. Paths under - * the table root become relative; paths elsewhere are returned absolute. + * Returns true if a location is absolute. + * NOTE: This is not the same implementation as Hadoop [[Path.isAbsolute]]. */ - def relativizeManifestPathToTableRoot(fs: FileSystem, tableRoot: Path, path: Path): String = - DeltaFileOperations.tryRelativizePath(fs, tableRoot, path).toString + def isAbsoluteLocation(location: String): Boolean = { + hasScheme(location) || location.startsWith(PathSeparator) + } + + /** + * Relativizes a location against a table location. A trailing slash on `tableLocation` is + * ignored. If `location` starts with the normalized table location immediately followed by `/`, + * the prefix and separator are removed. Otherwise, `location` is returned as-is. + * This is a lightweight string manipulation. + * + * Because the relativization is prefix matching based, callers are expected to pass locations in + * the same format and encoding (either both raw or both URL-encoded). + * No such checks are performed here. + */ + def relativizeLocation(tableLocation: String, location: String): String = { + // Strip trailing slash from tableLocation if present. + val normalizedTableLocation = + if (tableLocation.length > PathSeparator.length && tableLocation.endsWith(PathSeparator)) { + tableLocation.dropRight(PathSeparator.length) + } else { + tableLocation + } + + // Prefix matching based location relativization + val prefixLength = normalizedTableLocation.length + if (location.length > prefixLength && + location.startsWith(PathSeparator, prefixLength) && + location.startsWith(normalizedTableLocation)) { + location.substring(prefixLength + PathSeparator.length) + } else { + location + } + } /** * Resolves a manifest `location` / `contentRoot.path` back to an absolute [[Path]] against @@ -50,11 +140,46 @@ object AMTUtils { else new Path(tableRoot, child) } + /** Returns a copy of the passed-in current transaction info with `lastManifestCommit` updated. */ + def updateCurrentTransactionInfo( + currentTransactionInfo: CurrentTransactionInfo, + newLastManifestCommit: LastManifestCommit): CurrentTransactionInfo = { + currentTransactionInfo.copy( + commitInfo = currentTransactionInfo.commitInfo.map(_.copy( + lastManifestCommit = Some(newLastManifestCommit))) + ) + } + + /** + * Returns a copy of the passed-in current transaction info folded onto the winning commit, ready + * for the next commit attempt. + */ + def updateCurrentTransactionInfo( + currentTransactionInfo: CurrentTransactionInfo, + winningCommitSummary: WinningCommitSummary): CurrentTransactionInfo = { + // If the winning commit emitted an inline AMT checkpoint, it is now the latest checkpoint + // before the next commit attempt. + val withWinnerTree = winningCommitSummary.amtCheckpoint.map { winningAMTCheckpoint => + currentTransactionInfo.copy( + // If the winning commit emitted an inline AMT checkpoint, it is now the latest checkpoint + // before the next commit attempt. + preCommitLatestAMTCheckpointOpt = Some(winningAMTCheckpoint), + // Update the current commitInfo to reflect the winning manifest commit. + commitInfo = currentTransactionInfo.commitInfo.map(_.copy( + lastManifestCommit = Some(LastManifestCommit( + version = winningCommitSummary.commitVersion, + contentRootVersion = winningAMTCheckpoint.contentRoot.version)))) + ) + }.getOrElse(currentTransactionInfo) + // Clear `currentCommitAttemptAMTCheckpointOpt` because it is stale once we rebase. + withWinnerTree.copy(currentCommitAttemptAMTCheckpointOpt = None) + } + // Serializes a Manifest Deletion Vector to the on-disk byte form carried in `manifest_info.dv`. - private[amt] def serializeMdv(mdv: RoaringBitmapArray): Array[Byte] = - mdv.serializeAsByteArray(RoaringBitmapArrayFormat.Portable) + private[amt] def serializeMdv(mdv: ManifestBitmap): Array[Byte] = + mdv.serializeAsByteArray() // Deserializes a Manifest Deletion Vector previously written by [[serializeMdv]]. - private[amt] def deserializeMdv(bytes: Array[Byte]): RoaringBitmapArray = - RoaringBitmapArray.readFrom(bytes) + private[amt] def deserializeMdv(bytes: Array[Byte]): ManifestBitmap = + ManifestBitmap.fromSerializedByteArray(bytes) } diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/amt/AMTWriteHelper.scala b/spark/src/main/scala/org/apache/spark/sql/delta/amt/AMTWriteHelper.scala index 8c6ae0a8a96..4c18ea99b86 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/amt/AMTWriteHelper.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/amt/AMTWriteHelper.scala @@ -19,23 +19,22 @@ package org.apache.spark.sql.delta.amt import java.util.concurrent.TimeUnit.NANOSECONDS // scalastyle:off import.ordering.noEmptyLine -import org.apache.spark.sql.delta.{Checkpoints, DeltaLog, DeltaParquetWriteSupport, Snapshot} -import org.apache.spark.sql.delta.actions.{Action, AddFile, Checkpoint, ContentRoot, DomainMetadata, Metadata, Protocol, SetTransaction} -import org.apache.spark.sql.delta.deletionvectors.RoaringBitmapArray +import org.apache.spark.sql.delta.{Checkpoints, DeltaLog, RowId, Snapshot} +import org.apache.spark.sql.delta.actions.{AddFile, Checkpoint, ContentRoot, DomainMetadata, Metadata, Protocol, SetTransaction} +import org.apache.spark.sql.delta.deletionvectors.ManifestBitmap import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util.FileNames import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileSystem, Path} import org.apache.hadoop.mapreduce.Job -import org.apache.parquet.hadoop.ParquetOutputFormat import org.apache.spark.TaskContext import org.apache.spark.paths.SparkPath import org.apache.spark.sql.{DataFrame, SparkSession} import org.apache.spark.sql.execution.SQLExecution -import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat import org.apache.spark.sql.functions.{col, hash, struct} +import org.apache.spark.sql.types.StructType import org.apache.spark.util.SerializableConfiguration /** Helpers for emitting an inline AMT checkpoint during a commit. */ @@ -53,25 +52,35 @@ object AMTWriteHelper extends DeltaLogging { commitVersion: Long, postCommitProtocol: Protocol, postCommitMetadata: Metadata, - trigger: String): (AMTWriteResult, SingleAMTWriteMetrics) = { + trigger: String): AMTWriteResult = { val deltaLog = readSnapshot.deltaLog val startNanos = System.nanoTime() val hadoopConf = deltaLog.newDeltaHadoopConf() val entriesPerLeaf = spark.sessionState.conf.getConf(DeltaSQLConf.AMT_ENTRIES_PER_LEAF) + // A full rewrite of the read snapshot's live files; it carries no commit actions, so the tree + // describes the read snapshot's version. + val contentTreeVersion = readSnapshot.version + // We derive DataManifestEntry.first_row_id from the high watermark without incrementing the + // watermark. This is valid because the DataEntries in each leaf already have first_row_id + // stamped, so writing the leaf does not issue any new row IDs. + val firstRowIdForNewLeaves = + firstRowIdAfter(RowId.extractHighWatermark(readSnapshot)) val (contentRootBase, leaves) = writeClusteredManifestTree( spark = spark, deltaLog = deltaLog, readSnapshot = readSnapshot, hadoopConf = hadoopConf, - entriesPerLeaf = entriesPerLeaf) - // A full rewrite of the read snapshot's live files; it carries no commit actions, so the tree - // describes the read snapshot's version. - val contentStateVersion = readSnapshot.version + metadata = postCommitMetadata, + protocol = postCommitProtocol, + entriesPerLeaf = entriesPerLeaf, + contentTreeVersion = contentTreeVersion, + firstRowIdForNewLeaves = firstRowIdForNewLeaves) val contentRoot = policyTaggedContentRoot( - readSnapshot, contentRootBase, incremental = false, contentStateVersion) + readSnapshot, contentRootBase, incremental = false, contentTreeVersion, + numLeaves = leaves.size.toLong) buildResult( - contentStateVersion = contentStateVersion, + contentTreeVersion = contentTreeVersion, contentRoot = contentRoot, leaves = leaves, postCommitProtocol = postCommitProtocol, @@ -84,35 +93,38 @@ object AMTWriteHelper extends DeltaLogging { // Tags the freshly written root with how the tree was produced, so a reader/maintenance job can // tell incremental trees apart from full re-materializations without inspecting the leaves. A - // full rewrite resets the "last full rewrite" marker to `contentStateVersion`; an incremental + // full rewrite resets the "last full rewrite" marker to `contentTreeVersion`; an incremental // rewrite carries forward the previous tree's marker. private def policyTaggedContentRoot( readSnapshot: Snapshot, contentRootBase: ContentRoot, incremental: Boolean, - contentStateVersion: Long): ContentRoot = { + contentTreeVersion: Long, + numLeaves: Long): ContentRoot = { val lastFullRewriteVersion = if (incremental) { previousAMTContentRoot(readSnapshot) .flatMap(_.lastManifestCommitWithFullRewrite) - .getOrElse(contentStateVersion) + .getOrElse(contentTreeVersion) } else { - contentStateVersion + contentTreeVersion } ContentRoot( path = contentRootBase.path, sizeInBytes = contentRootBase.sizeInBytes, + version = contentTreeVersion, isIncremental = incremental, - lastManifestCommitWithFullRewrite = lastFullRewriteVersion) + lastManifestCommitWithFullRewrite = lastFullRewriteVersion, + numLeaves = numLeaves) } /** - * Assembles the inline Checkpoint action, write result, and metric shared by both materialization - * paths. `contentStateVersion` is the table version the tree describes and is stamped on the + * Assembles the inline Checkpoint action and write result shared by both materialization + * paths. `contentTreeVersion` is the table version the tree describes and is stamped on the * checkpoint action and write result. */ private def buildResult( - contentStateVersion: Long, + contentTreeVersion: Long, contentRoot: ContentRoot, leaves: Seq[DataManifestEntry], postCommitProtocol: Protocol, @@ -120,25 +132,25 @@ object AMTWriteHelper extends DeltaLogging { domainMetadata: Seq[DomainMetadata], txns: Seq[SetTransaction], trigger: String, - startNanos: Long): (AMTWriteResult, SingleAMTWriteMetrics) = { + startNanos: Long): AMTWriteResult = { val checkpoint = Checkpoint( - version = contentStateVersion, + version = contentTreeVersion, contentRoot = contentRoot, protocol = postCommitProtocol, metaData = postCommitMetadata, domainMetadata = domainMetadata, txns = txns, sidecars = Seq.empty) - val result = AMTWriteResult( - contentRootVersion = contentStateVersion, - checkpoint = checkpoint, - leaves = leaves, - includeActionsInCommitJson = true) val singleMetric = SingleAMTWriteMetrics( trigger = trigger, incremental = contentRoot.isIncremental.map(_.toString).getOrElse("UNKNOWN"), materializeDurationMs = NANOSECONDS.toMillis(System.nanoTime() - startNanos)) - (result, singleMetric) + AMTWriteResult( + contentRootVersion = contentTreeVersion, + checkpoint = checkpoint, + leaves = leaves, + includeActionsInCommitJson = true, + amtWriteMetrics = singleMetric) } // The ContentRoot of the AMT tree `snapshot` is already backed by, if any. Used to carry forward @@ -151,16 +163,22 @@ object AMTWriteHelper extends DeltaLogging { /** * Writes a clustered AMT manifest tree for a full checkpoint of `readSnapshot`'s live files. - * All files are clustered and flushed into leaf manifests -- one leaf per Spark partition, - * written by executors. The root lists only leaf pointers (no inline data entries). Returns the - * [[ContentRoot]] plus the per-leaf [[DataManifestEntry]] pointers. + * Live files are clustered and flushed into manifests -- one per Spark partition, written by + * executors. A snapshot small enough to produce a single manifest needs no tree: that manifest is + * promoted to the root and no leaf pointers are returned. Otherwise a root listing one pointer + * per leaf is written. Returns the [[ContentRoot]] plus the per-leaf [[DataManifestEntry]] + * pointers, which are empty for a promoted single-manifest checkpoint. */ private def writeClusteredManifestTree( spark: SparkSession, deltaLog: DeltaLog, readSnapshot: Snapshot, hadoopConf: Configuration, - entriesPerLeaf: Int): (ContentRoot, Seq[DataManifestEntry]) = { + metadata: Metadata, + protocol: Protocol, + entriesPerLeaf: Int, + contentTreeVersion: Long, + firstRowIdForNewLeaves: Long): (ContentRoot, Seq[DataManifestEntry]) = { require(entriesPerLeaf > 0, "entriesPerLeaf must be positive.") val tableRoot = deltaLog.dataPath val fs = tableRoot.getFileSystem(hadoopConf) @@ -178,10 +196,33 @@ object AMTWriteHelper extends DeltaLogging { tableRoot = tableRoot, metadataDir = metadataDir, addFilesDf = addFilesDf, - desiredNumLeaves = desiredNumLeaves) - val contentRoot = - writeRoot(spark, fs, hadoopConf, tableRoot, metadataDir, leafEntries.map(_.wrap)) - (contentRoot, leafEntries) + metadata = metadata, + protocol = protocol, + desiredNumLeaves = desiredNumLeaves, + contentTreeVersion = contentTreeVersion, + firstRowIdForNewLeaves = firstRowIdForNewLeaves) + leafEntries match { + case Seq(onlyLeaf) => + // If there is only one leaf, promote it to the root. + val contentRoot = + ContentRoot( + path = onlyLeaf.location, + sizeInBytes = onlyLeaf.file_size_in_bytes, + version = contentTreeVersion) + (contentRoot, Seq.empty) + case _ => + val contentRoot = writeRoot( + spark = spark, + fs = fs, + hadoopConf = hadoopConf, + tableRoot = tableRoot, + metadataDir = metadataDir, + metadata = metadata, + protocol = protocol, + rows = leafEntries.map(_.wrap), + version = contentTreeVersion) + (contentRoot, leafEntries) + } } @@ -191,31 +232,42 @@ object AMTWriteHelper extends DeltaLogging { tableRoot: Path, metadataDir: Path, addFilesDf: DataFrame, - desiredNumLeaves: Int): Seq[DataManifestEntry] = { + metadata: Metadata, + protocol: Protocol, + desiredNumLeaves: Int, + contentTreeVersion: Long, + firstRowIdForNewLeaves: Long): Seq[DataManifestEntry] = { import org.apache.spark.sql.delta.implicits._ val addFilesDs = addFilesDf.as[AddFile] // Capture values so the closures do not reach back into the object / non-serializable Path. // The rewritten files already exist in the table, so their leaf entries are EXISTING. - val tracking = trackingWithStatus(Tracking.Status.Existing) + val dataEntryTracking = existingTrackingForDataEntry() val tableRootSparkPath = SparkPath.fromPath(tableRoot) val metadataDirSparkPath = SparkPath.fromPath(metadataDir) - - val amtDs = addFilesDs.map { add => - DataEntry.fromAddFile(add, tracking, tableRootSparkPath.toPath).wrap + val tableRootPath = tableRootSparkPath.toPath + val amtDs = addFilesDs.mapPartitions { iter => + iter.map { add => + require(add.baseRowId.isDefined, + s"Cannot write AMT leaf entry without a materialized baseRowId: ${add.path}.") + DataEntry.fromAddFile(add, dataEntryTracking, tableRootPath).wrap + } } - val schema = AMTSingleAction.schemaWithFieldId + val amtWithPartition = AMTPartitionValues.forWrite(amtDs.toDF(), metadata.partitionSchema) + val amtDf = AMTContentStats.forWrite(amtWithPartition, metadata, protocol) + val schema = AMTSingleAction.persistedSchema(metadata, protocol) + val recordCountIdx = amtDf.schema.fieldIndex("record_count") + val trackingIdx = amtDf.schema.fieldIndex("tracking") + val trackingSchema = amtDf.schema("tracking").dataType.asInstanceOf[StructType] + val sequenceNumberIdx = trackingSchema.fieldIndex("sequence_number") val (factory, serConf) = { - val format = new ParquetFileFormat() + val format = new AMTParquetFileFormat() val job = Job.getInstance(hadoopConf) val f = format.prepareWrite(spark, job, Map.empty, schema) - // Emit nested (list-element / map key-value) field ids, which the stock ParquetWriteSupport - // does not. Set after prepareWrite (before snapshotting the conf) so it flows to executors. - ParquetOutputFormat.setWriteSupportClass(job, classOf[DeltaParquetWriteSupport]) (f, new SerializableConfiguration(job.getConfiguration)) } - val qe = amtDs.queryExecution + val qe = amtDf.queryExecution SQLExecution.withNewExecutionId(qe, Some("AMT leaf checkpoint")) { qe.executedPlan.execute().mapPartitions { iter => // Skip empty partitions (so the root never points at an empty leaf for a non-empty table). @@ -225,6 +277,20 @@ object AMTWriteHelper extends DeltaLogging { val conf = serConf.value val leafFile = FileNames.newAMTLeafManifestFile(metadataDirSparkPath.toPath) + var entryCount = 0 + var entryRows = 0L + var minSequenceNumber = Long.MaxValue + val countingRows = iter.map { row => + entryCount += 1 + entryRows += row.getLong(recordCountIdx) + val tracking = row.getStruct(trackingIdx, trackingSchema.length) + require(!tracking.isNullAt(sequenceNumberIdx), + "Cannot write an EXISTING AMT leaf entry without a materialized " + + "tracking.sequence_number.") + minSequenceNumber = Math.min( + minSequenceNumber, tracking.getLong(sequenceNumberIdx)) + row + } val status = Checkpoints.writeSingleFileOnExecutor( conf = conf, factory = factory, @@ -234,26 +300,35 @@ object AMTWriteHelper extends DeltaLogging { useRename = false, partition = TaskContext.getPartitionId(), expectedNumParts = desiredNumLeaves, - rows = iter + rows = countingRows ) - val leafFs = leafFile.getFileSystem(conf) + // The root pointer to this leaf is newly ADDED, even though the leaf's own DATA entries + // are EXISTING (the referenced data files already lived in the table), so manifest_info + // counts every entry and its rows as EXISTING. + val (tracking, manifestInfo) = addedTrackingForLeaf( + addedFileAndRowCount = emptyFileRowCount, + existingFileAndRowCount = FileRowCount(entryCount, entryRows), + deletedFileAndRowCount = emptyFileRowCount, + replacedFileAndRowCount = emptyFileRowCount, + modifiedFileAndRowCount = emptyFileRowCount, + minSequenceNumber = minSequenceNumber, + contentTreeVersion = contentTreeVersion, + firstRowId = firstRowIdForNewLeaves) Iterator.single(DataManifestEntry( - location = AMTUtils.relativizeManifestPathToTableRoot( - leafFs, tableRootSparkPath.toPath, leafFile), + location = AMTUtils.relativizeLocation( + tableRootSparkPath.toPath.toString, leafFile.toString), file_format = AMTSingleAction.FileFormatParquet, - // The root pointer to this leaf is newly ADDED, even though the leaf's own DATA - // entries are EXISTING (the referenced data files already lived in the table). - tracking = trackingWithStatus(Tracking.Status.Added), - record_count = 0L, + tracking = tracking, + record_count = entryCount, file_size_in_bytes = status.length, - manifest_info = manifestInfoFor(Seq.empty))) + manifest_info = manifestInfo)) } }.collect().toSeq } } - // Tracking envelope for a freshly written entry with the given status. - private def trackingWithStatus(status: Int): Tracking = Tracking( + // Initializes a tracking envelope for a freshly written entry with the given status. + private def initializeTracking(status: Int): Tracking = Tracking( status = status, snapshot_id = None, dv_snapshot_id = None, @@ -263,14 +338,189 @@ object AMTWriteHelper extends DeltaLogging { deleted_positions = None, replaced_positions = None) - // Tracking envelope for a freshly written entry: ADDED. - private[amt] def addedTracking: Tracking = trackingWithStatus(Tracking.Status.Added) + /** Tracking helpers for [[DataEntry]] */ + private[amt] def addedTrackingForDataEntry() = initializeTracking(Tracking.Status.Added) + private[amt] def existingTrackingForDataEntry() = initializeTracking(Tracking.Status.Existing) + private[amt] def modifiedTrackingForDataEntry() = initializeTracking(Tracking.Status.Modified) + private[amt] def removedTrackingForDataEntry() = initializeTracking(Tracking.Status.Deleted) + private[amt] def replacedTrackingForDataEntry() = initializeTracking(Tracking.Status.Replaced) + + /** Paired file and row tallies for a leaf's entries in one tracking-status group. */ + private[amt] case class FileRowCount(fileCount: Int, rowCount: Long) + private[amt] val emptyFileRowCount = FileRowCount(0, 0L) + + /** + * Returns Delta's next unassigned row ID, which is the snapshot-level starting value for + * assigning `first_row_id` to new data manifests. AMT currently materializes `first_row_id` on + * every data entry, so no rows inherit from the manifest and all new leaves may use this same + * starting value without advancing Delta's high watermark. + */ + private[amt] def firstRowIdAfter(highWaterMark: Option[Long]): Long = + Math.addExact(highWaterMark.getOrElse(RowId.MISSING_HIGH_WATER_MARK), 1L) + + /** + * Tallies a freshly written leaf's entries into per-status file and row counts in a single pass, + * then builds its `(Tracking, ManifestInfo)`. Each entry contributes one file and its physical + * `record_count` rows to the count group for its tracking status (ADDED, EXISTING, DELETED, + * REPLACED, or MODIFIED). It also computes the minimum effective data sequence number over live + * entries (ADDED, EXISTING, and MODIFIED), starting from the content tree version. A missing + * sequence number or a leaf with no live entries therefore uses the current snapshot sequence. + */ + private[amt] def addedTrackingForLeaf( + entries: Seq[DataEntry], + contentTreeVersion: Long, + firstRowId: Long): (Tracking, ManifestInfo) = { + // Accumulate per-status file and row counts in a single pass. Plain Long accumulators avoid + // allocating an intermediate object per entry. + var addedFiles, existingFiles, deletedFiles, replacedFiles, modifiedFiles = 0 + var addedRows, existingRows, deletedRows, replacedRows, modifiedRows = 0L + var minSequenceNumber = contentTreeVersion + entries.foreach { entry => + val rows = entry.record_count + require(entry.tracking.first_row_id.isDefined, + s"Cannot write AMT leaf entry without a materialized first_row_id: ${entry.location}.") + if (Tracking.Status.liveEntryStatuses.contains(entry.tracking.status)) { + entry.tracking.sequence_number.foreach { currentEntrySequenceNumber => + minSequenceNumber = Math.min(minSequenceNumber, currentEntrySequenceNumber) + } + } + entry.tracking.status match { + case Tracking.Status.Added => + addedFiles += 1 + addedRows += rows + case Tracking.Status.Existing => + existingFiles += 1 + existingRows += rows + case Tracking.Status.Deleted => + deletedFiles += 1 + deletedRows += rows + case Tracking.Status.Replaced => + replacedFiles += 1 + replacedRows += rows + case Tracking.Status.Modified => + modifiedFiles += 1 + modifiedRows += rows + case other => + AMTUtils.invariantCheckWithLogging( + checkInvariant = false, + opTypeSuffix = AMTUsageLogs.ALERT_UNEXPECTED_LEAF_TRACKING_STATUS, + message = s"Unexpected leaf entry tracking status: $other.", + data = Map("unknownStatus" -> other)) + } + } + addedTrackingForLeaf( + addedFileAndRowCount = FileRowCount(addedFiles, addedRows), + existingFileAndRowCount = FileRowCount(existingFiles, existingRows), + deletedFileAndRowCount = FileRowCount(deletedFiles, deletedRows), + replacedFileAndRowCount = FileRowCount(replacedFiles, replacedRows), + modifiedFileAndRowCount = FileRowCount(modifiedFiles, modifiedRows), + minSequenceNumber = minSequenceNumber, + contentTreeVersion = contentTreeVersion, + firstRowId = firstRowId) + } + + /** Tracking + ManifestInfo for a freshly written leaf from its per-status file and row counts. */ + private[amt] def addedTrackingForLeaf( + addedFileAndRowCount: FileRowCount, + existingFileAndRowCount: FileRowCount, + deletedFileAndRowCount: FileRowCount, + replacedFileAndRowCount: FileRowCount, + modifiedFileAndRowCount: FileRowCount, + minSequenceNumber: Long, + contentTreeVersion: Long, + firstRowId: Long): (Tracking, ManifestInfo) = { + val tracking = initializeTracking(Tracking.Status.Added).copy( + sequence_number = Some(contentTreeVersion), + file_sequence_number = Some(contentTreeVersion), + first_row_id = Some(firstRowId)) + val manifestInfo = emptyManifestInfo.copy( + added_files_count = addedFileAndRowCount.fileCount, + existing_files_count = existingFileAndRowCount.fileCount, + deleted_files_count = deletedFileAndRowCount.fileCount, + replaced_files_count = replacedFileAndRowCount.fileCount, + modified_files_count = modifiedFileAndRowCount.fileCount, + added_rows_count = addedFileAndRowCount.rowCount, + existing_rows_count = existingFileAndRowCount.rowCount, + deleted_rows_count = deletedFileAndRowCount.rowCount, + replaced_rows_count = replacedFileAndRowCount.rowCount, + modified_rows_count = modifiedFileAndRowCount.rowCount, + min_sequence_number = minSequenceNumber) + (tracking, manifestInfo) + } - // Tracking envelope for a root-resident tombstone: REMOVED. - // `deletedPositions`, when present, is the within-file bitmap of rows this commit deleted - // (carried for CDF); it is left None for a whole-file removal. - private[amt] def removedTracking(deletedPositions: Option[Array[Byte]] = None): Tracking = - trackingWithStatus(Tracking.Status.Deleted).copy(deleted_positions = deletedPositions) + /** + * Tracking + ManifestInfo for a carried leaf that holds no live file (its live entries are all + * MDV-masked, or it only ever held tombstones): it decays to DELETED with its manifest_info kept + * and per-commit CDF positions cleared, so the next AMT drops it. + */ + private[amt] def deletedTrackingForCarriedLeaf( + oldEntry: DataManifestEntry): (Tracking, ManifestInfo) = { + val newTracking = oldEntry.tracking.copy( + status = Tracking.Status.Deleted, + // Clear the per-commit CDF positions: the leaf is being dropped, not changed this commit, so + // it must not contribute any deleted/replaced rows to this commit's Change Data Feed. + deleted_positions = None, + replaced_positions = None) + (newTracking, oldEntry.manifest_info) + } + + /** + * Tracking + ManifestInfo for a leaf carried unchanged into this tree: re-emitted EXISTING with + * its MDV kept and per-commit CDF positions cleared. + */ + private[amt] def existingTrackingForLeaf( + oldEntry: DataManifestEntry): (Tracking, ManifestInfo) = { + val newTracking = oldEntry.tracking.copy( + status = Tracking.Status.Existing, + deleted_positions = None, + replaced_positions = None) + // manifest_info file/row counts are immutable (they describe the leaf as written), and its MDV + // is unchanged here (no new masking), so carry manifest_info forward untouched. + (newTracking, oldEntry.manifest_info) + } + + /** + * Tracking + ManifestInfo for a carried leaf whose MDV grew this commit: MODIFIED, with + * `mdvPositions` accumulated into the cumulative MDV, and this commit's `deletedPositions` / + * `replacedPositions` recorded for CDF. If all the MDV bits are set, the leaf decays to DELETED. + */ + private[amt] def modifiedOrDeletedTrackingForLeaf( + oldEntry: DataManifestEntry, + mdvPositions: Seq[Int], + deletedPositions: Seq[Int], + replacedPositions: Seq[Int]): (Tracking, ManifestInfo) = { + val mutableMdv = oldEntry.manifest_info.dv + .map(AMTUtils.deserializeMdv).getOrElse(ManifestBitmap.fromPositions(Seq.empty)) + .toMutable + mdvPositions.foreach(mutableMdv.add) + val cumulativeMdv = mutableMdv.toManifestBitmap + def bitmapOf(positions: Seq[Int]): Option[Array[Byte]] = { + if (positions.isEmpty) None + else Some(AMTUtils.serializeMdv(ManifestBitmap.fromPositions(positions))) + } + // Every masked / CDF position indexes an entry within this leaf, so no count can exceed the + // leaf's entry count; a larger value signals a corrupt bitmap or a double-counted position. + def assertWithinLeaf(count: Long, label: String): Unit = + assert(count <= oldEntry.record_count, + s"leaf ${oldEntry.location}: $label $count exceeds record_count ${oldEntry.record_count}.") + assertWithinLeaf(cumulativeMdv.cardinality, "MDV cardinality") + assertWithinLeaf(deletedPositions.size.toLong, "deleted_positions") + assertWithinLeaf(replacedPositions.size.toLong, "replaced_positions") + // A leaf whose every live entry is masked by the cumulative MDV holds no live file, so it + // decays to DELETED rather than MODIFIED. + val noActiveFiles = + oldEntry.manifest_info.liveFilesCount.toLong - cumulativeMdv.cardinality <= 0 + val status = if (noActiveFiles) Tracking.Status.Deleted else Tracking.Status.Modified + val newTracking = oldEntry.tracking.copy( + status = status, + deleted_positions = bitmapOf(deletedPositions), + replaced_positions = bitmapOf(replacedPositions)) + // manifest_info file/row counts are immutable; only the MDV (dv/dv_cardinality) grows to mask + // the newly superseded positions. Live count is record_count - dv_cardinality, and this + // commit's CDF positions live on the tracking, not in the counts. + val manifestInfo = withUpdatedMdv(oldEntry.manifest_info, cumulativeMdv) + (newTracking, manifestInfo) + } /** * Writes a single leaf parquet file (DATA entries) and returns the DataManifestEntry @@ -282,34 +532,33 @@ object AMTWriteHelper extends DeltaLogging { hadoopConf: Configuration, tableRoot: Path, metadataDir: Path, - batch: Seq[AddFile]): DataManifestEntry = { + metadata: Metadata, + protocol: Protocol, + entries: Seq[DataEntry], + contentTreeVersion: Long, + firstRowId: Long): DataManifestEntry = { val leafFile = FileNames.newAMTLeafManifestFile(metadataDir) - val rows: Seq[AMTSingleAction] = - batch.map(add => - DataEntry - .fromAddFile( - add, - trackingWithStatus(Tracking.Status.Added), - tableRoot - ) - .wrap - ) - writeAMTParquet(spark, hadoopConf, leafFile, rows) - val status = fs.getFileStatus(leafFile) - val manifestInfo = manifestInfoFor(rows) + writeAMTParquet(spark, hadoopConf, leafFile, metadata, protocol, entries.map(_.wrap)) + val fileStatus = fs.getFileStatus(leafFile) + // A freshly written leaf is always ADDED -- even one holding only tombstones, whose + // manifest_info still counts the DELETED / REPLACED entries and their rows. + val (tracking, manifestInfo) = + addedTrackingForLeaf(entries, contentTreeVersion, firstRowId) DataManifestEntry( - location = AMTUtils.relativizeManifestPathToTableRoot(fs, tableRoot, leafFile), + location = AMTUtils.relativizeLocation(tableRoot.toString, leafFile.toString), file_format = AMTSingleAction.FileFormatParquet, - tracking = trackingWithStatus(Tracking.Status.Added), + tracking = tracking, // Number of content entries the referenced leaf manifest holds. - record_count = manifestInfo.added_files_count.toLong, - file_size_in_bytes = status.getLen, + record_count = entries.size.toLong, + file_size_in_bytes = fileStatus.getLen, manifest_info = manifestInfo) } /** - * Writes the root parquet file from a pre-built row set and returns the ContentRoot. The rows may - * be `DATA_MANIFEST` leaf pointers, root-resident `DATA` entries, or both. + * Writes the root parquet file from a pre-built row set and returns a [[ContentRoot]] + * carrying its path, size, and version. The rows may be `DATA_MANIFEST` leaf pointers, + * root-resident `DATA` entries, or both. Callers still attach tags before embedding it in a + * [[Checkpoint]]. */ private[amt] def writeRoot( spark: SparkSession, @@ -317,64 +566,69 @@ object AMTWriteHelper extends DeltaLogging { hadoopConf: Configuration, tableRoot: Path, metadataDir: Path, - rows: Seq[AMTSingleAction]): ContentRoot = { + metadata: Metadata, + protocol: Protocol, + rows: Seq[AMTSingleAction], + version: Long): ContentRoot = { val rootFile = FileNames.newAMTRootManifestFile(metadataDir) - writeAMTParquet(spark, hadoopConf, rootFile, rows) + writeAMTParquet(spark, hadoopConf, rootFile, metadata, protocol, rows) val status = fs.getFileStatus(rootFile) ContentRoot( - path = AMTUtils.relativizeManifestPathToTableRoot(fs, tableRoot, rootFile), - sizeInBytes = status.getLen) + path = AMTUtils.relativizeLocation(tableRoot.toString, rootFile.toString), + sizeInBytes = status.getLen, + version = version) } // Returns a copy of a carried-forward leaf's ManifestInfo with `mdv` recorded as its Manifest // Deletion Vector. - private[amt] def withUpdatedMdv(base: ManifestInfo, mdv: RoaringBitmapArray): ManifestInfo = { + private[amt] def withUpdatedMdv(base: ManifestInfo, mdv: ManifestBitmap): ManifestInfo = { if (mdv.isEmpty) { - base.copy(dv = None, dv_cardinality = None, deleted_files_count = 0) + base.copy(dv = None, dv_cardinality = None) } else { base.copy( dv = Some(AMTUtils.serializeMdv(mdv)), - dv_cardinality = Some(mdv.cardinality), - deleted_files_count = mdv.cardinality.toInt) + dv_cardinality = Some(mdv.cardinality)) } } - /** - * Summarizes a leaf's batch into a ManifestInfo. - */ - private def manifestInfoFor(batch: Seq[AMTSingleAction]): ManifestInfo = { + private def emptyManifestInfo: ManifestInfo = ManifestInfo( - added_files_count = batch.size, + added_files_count = 0, existing_files_count = 0, deleted_files_count = 0, replaced_files_count = 0, + modified_files_count = 0, added_rows_count = 0L, existing_rows_count = 0L, deleted_rows_count = 0L, replaced_rows_count = 0L, - // No data sequence numbers are assigned yet; 0 is the conventional "unset" minimum. + modified_rows_count = 0L, + // Placeholder overridden before a newly written manifest is emitted. min_sequence_number = 0L, dv = None, dv_cardinality = None) - } /** * Writes a sequence of AMTSingleActions to a Parquet file. */ - private def writeAMTParquet( + private[amt] def writeAMTParquet( spark: SparkSession, hadoopConf: Configuration, finalPath: Path, + metadata: Metadata, + protocol: Protocol, rows: Seq[AMTSingleAction]): Unit = { import org.apache.spark.sql.delta.implicits._ - val df = spark.createDataset(rows).toDF() + val withPartition = AMTPartitionValues.forWrite( + spark.createDataset(rows).toDF(), metadata.partitionSchema) + val df = AMTContentStats.forWrite(withPartition, metadata, protocol) Checkpoints.writeAtomicCheckpointParquetFile( spark = spark, df = df, finalPath = finalPath, hadoopConf = hadoopConf, useRename = false, - outputSchema = Some(AMTSingleAction.schemaWithFieldId), - useDeltaParquetWriteSupport = true) + outputSchema = Some(AMTSingleAction.persistedSchema(metadata, protocol)), + format = new AMTParquetFileFormat) } } diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/amt/AMTWriterManager.scala b/spark/src/main/scala/org/apache/spark/sql/delta/amt/AMTWriterManager.scala index 5336a8eae95..9e4267eb3ac 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/amt/AMTWriterManager.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/amt/AMTWriterManager.scala @@ -16,12 +16,21 @@ package org.apache.spark.sql.delta.amt -import org.apache.spark.sql.delta.{AdaptiveMetadataTableFeature, CurrentTransactionInfo, DeltaErrors, DeltaLog, DeltaOperations, LogSegment, MaintenanceOperation, Snapshot} -import org.apache.spark.sql.delta.actions.{Action, Checkpoint} +import java.util.concurrent.TimeUnit + +import scala.util.control.NonFatal + +import org.apache.spark.sql.delta.{ConcurrentAMTCheckpointLandedException, CurrentTransactionInfo, DeltaErrors, DeltaLog, DeltaOperations, FullAMTWriteFailedWithConflict, LogSegment, MaintenanceOperation, Snapshot, WinningCommitMetrics} +import org.apache.spark.sql.delta.actions.{Action, Checkpoint, FileAction} +import org.apache.spark.sql.delta.logging.DeltaLogKeys +import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.sources.DeltaSQLConf +import org.apache.spark.sql.delta.util.DeltaTestBarrier import org.apache.spark.sql.delta.util.FileNames +import org.apache.spark.internal.MDC import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.catalog.CatalogTable /** * Describes a trigger for AMT. @@ -51,11 +60,120 @@ object AMTTriggerMode { case object InlineWithLargeCommitIncremental extends AMTTriggerMode( name = "INLINE_WITH_LARGE_COMMIT_INCREMENTAL", isIncremental = true) + + /** + * An on-demand `DeltaLog.checkpoint` request on an AMT table: full rewrite. Such a request + * is done by callers like `commitLarge` e.g. RESTORE / CLONE that typically just replaced + * the file set wholesale, so there is nothing useful to build on incrementally. + */ + case object OnDemandCheckpointFull extends AMTTriggerMode( + name = "ON_DEMAND_CHECKPOINT_FULL", + isIncremental = false) } -/** Metrics describing an AMT (Adaptive Metadata Tree) write, one entry per attempt. */ -case class AMTWriteMetrics( - private[delta] var attempts: Seq[SingleAMTWriteMetrics] = Seq.empty) +/** Metrics for one complete round of AMT conflict resolution. */ +case class AMTConflictResolutionRoundMetrics( + losingAttemptVersion: Long, + nextAttemptVersion: Long, + losingCommitType: String, + losingTreeType: Option[String], + winningCommits: WinningAMTCommitSummary, + backRefRebaseMetrics: BackRefRebaseMetrics, + var treeOutcome: String, + var winnerTreeSatisfiesRequirement: Option[Boolean], + var exceptionThrown: Option[String]) { + + def updateOutcome( + treeOutcome: String, + winnerTreeSatisfiesRequirement: Option[Boolean] = None, + exceptionThrown: Option[String] = None): Unit = { + this.treeOutcome = treeOutcome + this.winnerTreeSatisfiesRequirement = winnerTreeSatisfiesRequirement + this.exceptionThrown = exceptionThrown + } +} + +object AMTConflictResolutionRoundMetrics { + /** For field [[AMTConflictResolutionRoundMetrics.losingCommitType]]. */ + val LOG_ONLY = "LOG_ONLY" + val INLINE_INCREMENTAL = "INLINE_INCREMENTAL" + val INCREMENTAL_CHECKPOINT = "INCREMENTAL_CHECKPOINT" + val FULL_CHECKPOINT = "FULL_CHECKPOINT" + + /** For field [[AMTConflictResolutionRoundMetrics.losingTreeType]]. */ + val INCREMENTAL_TREE = "INCREMENTAL_TREE" + val FULL_TREE = "FULL_TREE" + + /** For field [[AMTConflictResolutionRoundMetrics.treeOutcome]]. */ + val NO_TREE_REBASE = "NO_TREE_REBASE" + val REBUILT_INLINE_TREE = "REBUILT_INLINE_TREE" + val REBUILT_OPTIMIZE_CHECKPOINT_INCREMENTAL_TREE = "REBUILT_OPTIMIZE_CHECKPOINT_INCREMENTAL_TREE" + val REUSED_LOSING_TREE = "REUSED_LOSING_TREE" + val RETRY_VIA_NEW_TXN = "RETRY_VIA_NEW_TXN" + val SKIP_WINNER_SATISFIES_REQUIREMENT = "SKIP_WINNER_SATISFIES_REQUIREMENT" +} + +/** Aggregated shape of all winning commits processed in one conflict-checking round. */ +case class WinningAMTCommitSummary( + firstVersion: Long, + lastVersion: Long, + numLogOnly: Int, + numInlineIncremental: Int, + numIncrementalCheckpoints: Int, + numFullCheckpoints: Int, + allLogWinnersPreserveLosingTree: Option[Boolean]) + +/** Metrics emitted for one AMT write or conflict-resolution round. */ +case class AMTMetrics( + txnId: String, + roundId: Int, + var conflictResolutionMetrics: Option[AMTConflictResolutionRoundMetrics], + var singleAMTWriteMetrics: Option[SingleAMTWriteMetrics]) + +/** AMT metrics attached to the successful commit's commit stats. */ +case class AMTCommitStats( + contentRootVersion: Long, + lastAMTWriteMetrics: SingleAMTWriteMetrics, + includeActionsInCommitJson: Boolean) + +/** Back-reference rebase outcome and optional details for one conflict-resolution round. */ +case class BackRefRebaseMetrics( + skipped: Boolean, + skipReason: Option[String], + oldAMTVersion: Option[Long] = None, + newAMTVersion: Option[Long] = None, + totalTimeTakenMs: Option[Long] = None, + numActionsReusingBackref: Option[Int] = None, + numActionsRegeneratingBackref: Option[Int] = None) + +object BackRefRebaseMetrics { + /** Values for field [[BackRefRebaseMetrics.skipReason]]. */ + val SKIP_REASON_NO_NEW_TREE = "NO_NEW_TREE" + val SKIP_REASON_BLIND_APPEND = "BLIND_APPEND" + val SKIP_REASON_NO_FILE_ACTIONS = "NO_FILE_ACTIONS" + val SKIP_REASON_ALREADY_REBASED = "ALREADY_REBASED" + val SKIP_REASON_NO_AMT_PROVIDER = "NO_AMT_PROVIDER" + + /** Creates metrics for a skipped back-reference rebase. */ + def apply(skipReason: String): BackRefRebaseMetrics = + new BackRefRebaseMetrics(skipped = true, skipReason = Some(skipReason)) + + /** Creates metrics for an executed back-reference rebase. */ + def apply( + oldAMTVersion: Long, + newAMTVersion: Long, + totalTimeTakenMs: Long, + numActionsReusingBackref: Int, + numActionsRegeneratingBackref: Int): BackRefRebaseMetrics = + new BackRefRebaseMetrics( + skipped = false, + skipReason = None, + oldAMTVersion = Some(oldAMTVersion), + newAMTVersion = Some(newAMTVersion), + totalTimeTakenMs = Some(totalTimeTakenMs), + numActionsReusingBackref = Some(numActionsReusingBackref), + numActionsRegeneratingBackref = Some(numActionsRegeneratingBackref)) +} /** Metrics for a single AMT write attempt (one per commit attempt that materializes a tree). */ case class SingleAMTWriteMetrics( @@ -67,12 +185,25 @@ case class SingleAMTWriteMetrics( case class IncrementalAMTWriteMetrics( numIntermediateCommits: Int, - numExistingLeavesUpdated: Int, - numExistingLeavesUntouched: Int, + numOldLeavesUpdated: Int, + numOldLeavesUntouched: Int, numNewLeaves: Int, - numRootLiveAdds: Int, - numRootTombstones: Int, - numLeafMdvBitsAdded: Int) + // Per-status breakdown over root-resident DATA entries (see [[Tracking.Status]]). + numRootEntriesAddedStatus: Int, + numRootEntriesExistingStatus: Int, + numRootEntriesModifiedStatus: Int, + numRootEntriesReplacedStatus: Int, + numRootEntriesDeletedStatus: Int, + numLeafMdvBitsAdded: Int, + numLeafDeleteCDFBitsAdded: Int = 0, + numLeafReplaceCDFBitsAdded: Int = 0, + // Per-status breakdown over all leaf pointers in the new tree (see [[Tracking.Status]]), plus + // the stale DELETED tombstones from the previous tree that this rewrite dropped. + numLeavesAddedStatus: Int = 0, + numLeavesExistingStatus: Int = 0, + numLeavesModifiedStatus: Int = 0, + numLeavesDeletedStatus: Int = 0, + numStaleDeletedLeavesDropped: Int = 0) /** * The outcome of an AMT write for a single commit attempt. @@ -82,44 +213,192 @@ case class IncrementalAMTWriteMetrics( * @param leaves the root's `DATA_MANIFEST` pointer entries, one per leaf * @param includeActionsInCommitJson whether the transaction should still write the commit's file * actions inline in the commit JSON. + * @param amtWriteMetrics metrics for the materialization that produced this result */ case class AMTWriteResult( contentRootVersion: Long, checkpoint: Checkpoint, leaves: Seq[DataManifestEntry], - includeActionsInCommitJson: Boolean) + includeActionsInCommitJson: Boolean, + amtWriteMetrics: SingleAMTWriteMetrics) + +/** A lazily-materialized [[AMTCheckpointProvider]] for `checkpointOpt`. */ +class LazyAMTCheckpointProvider( + checkpointOpt: Option[Checkpoint], + readSnapshot: Snapshot, + manifestCommitVersion: Long) { + lazy val providerOpt: Option[AMTCheckpointProvider] = checkpointOpt.map { checkpoint => + readSnapshot.checkpointProvider match { + case amt: AMTCheckpointProvider if amt.checkpointAction.version == checkpoint.version => amt + case _ => + AMTCheckpointProvider.fromCheckpoint( + readSnapshot.deltaLog, checkpoint, manifestCommitVersion) + } + } +} /** * Orchestrates write of an AMT for a given transaction (including reattempts on a conflict). + * There is one [[AMTWriterManager]] per `OptimisticTransaction`. + * + * On the first attempt: + * - [[writeAMT]] must be invoked. It writes a new AMT if needed, returns the optional + * [[AMTWriteResult]], and sends a usage log when it writes a tree. + * + * On each conflict: + * - [[updatePreCommitLatestAMTCheckpointProvider]] updates the cached AMT provider after conflict + * resolution folds the winning commits. + * - [[rebaseBackReferences]] re-derives back references if needed. Its metrics are retained in + * `pendingBackRefRebaseMetrics` until the attempt is prepared. + * - [[writeAMT]] must be invoked again. It reuses or writes an AMT as needed, returns the optional + * [[AMTWriteResult]], and sends the usage log for the conflict round. The event combines the + * pending back-reference metrics with metrics for any new tree written by that attempt. */ class AMTWriterManager( + private val txnId: String, readSnapshot: Snapshot, - initialOperation: DeltaOperations.Operation) { + initialOperation: DeltaOperations.Operation) extends DeltaLogging { + + import AMTConflictResolutionRoundMetrics._ + import BackRefRebaseMetrics._ private def spark: SparkSession = SparkSession.active private def deltaLog: DeltaLog = readSnapshot.deltaLog - val metrics = AMTWriteMetrics() private var lastAMTWriteResultOpt: Option[AMTWriteResult] = None + private var nextRoundId = 0 + + /** The read snapshot's own AMT checkpoint, if it is AMT-backed. */ + private def readSnapshotAMTCheckpointOpt: Option[Checkpoint] = { + if (!AMTUtils.amtEnabled(readSnapshot)) return None + readSnapshot.checkpointProvider match { + case amt: AMTCheckpointProvider => Some(amt.checkpointAction) + case _ => None + } + } + + /** + * The AMT Checkpoint Provider corresponding to the last manifest commit corresponding to + * OptimisticTransaction.preCommitLogSegment. + * This is updated after every round of [[ConflictChecker]] rebase. + */ + private var preCommitLatestAMTCheckpointProvider: LazyAMTCheckpointProvider = + new LazyAMTCheckpointProvider(readSnapshotAMTCheckpointOpt, readSnapshot, readSnapshot.version) + + /** + * The folded-latest AMT provider the committed actions' back references are stamped against after + * conflict resolution. + */ + private[delta] def preCommitLatestAMTCheckpointProviderOpt: Option[AMTCheckpointProvider] = + preCommitLatestAMTCheckpointProvider.providerOpt + + /** The folded AMT tree version the committed actions were last re-stamped against. */ + private var lastRebasedAMTVersion: Option[Long] = None + + /** The version this transaction's previous attempt targeted. */ + private var lastAttemptVersion: Long = readSnapshot.version + 1 + + private var pendingBackRefRebaseMetrics = + BackRefRebaseMetrics(SKIP_REASON_NO_NEW_TREE) /** * Builds the AMT write for a commit attempt, or `None` when no AMT should be written. Serves both * the first attempt and any conflict-resolution retry. * - * @param commitVersion the version this attempt targets + * @param nextAttemptVersion the version this attempt targets * @param currentTransactionInfo the in-flight transaction (its actions, protocol, metadata) * @param preCommitLogSegment the log segment prior to this commit + * @param winningCommitMetricsForConflictedRange per-winning-commit metrics from conflict + * resolution, one per commit in the conflicted range [lastAttemptVersion, nextAttemptVersion) + * @return the AMT write result; `None` if no AMT write is triggered or it's a non-AMT table. */ def writeAMT( - commitVersion: Long, + nextAttemptVersion: Long, currentTransactionInfo: CurrentTransactionInfo, - preCommitLogSegment: LogSegment): Option[AMTWriteResult] = { - if (!amtEnabled(commitVersion)) return None - if (preCommitLogSegment.version > readSnapshot.version) { - throw DeltaErrors.concurrentWriteException(conflictingCommit = None) - } + preCommitLogSegment: LogSegment, + winningCommitMetricsForConflictedRange: Seq[WinningCommitMetrics]): Option[AMTWriteResult] = { + if (!AMTUtils.amtEnabled(readSnapshot)) return None val actionsToCommit = currentTransactionInfo.actions - val resultOpt = initialOperation match { + val roundId = nextRoundId + nextRoundId += 1 + val metrics = AMTMetrics( + txnId = txnId, + roundId = roundId, + conflictResolutionMetrics = None, + singleAMTWriteMetrics = None) + // Whether this attempt would (re)write a manifest tree. + val writesTree = initialOperation match { + case _: DeltaOperations.OptimizeCheckpoint => true + case _ => shouldDoInlineIncrementalCheckpoint(actionsToCommit) + } + + val rebasing = preCommitLogSegment.version > readSnapshot.version + if (rebasing) { + validateWinningCommitMetrics( + winningCommitMetricsForConflictedRange, lastAttemptVersion, nextAttemptVersion) + initializeAMTMetricsDuringRebase( + metrics, nextAttemptVersion, winningCommitMetricsForConflictedRange) + // A concurrent commit won our target version and we are rebasing. This matrix summarizes the + // result by losing commit (rows) and winning commit (columns): + // + // |-------------------|----------------|----------------|----------------|----------------| + // | Winning commit -> | Log-only | Inline | Incremental | Full | + // | ------------------| | incremental | checkpoint | checkpoint | + // | Losing commit | | | | | + // |-------------------|----------------|----------------|----------------|----------------| + // | Log-only | Conflict check | Rebase on winner tree; | + // | | keep existing | re-derive back references [1] | + // | | back refs | | + // |-------------------|----------------|----------------|----------------|----------------| + // | Inline | Rebuild inline | Use winner tree as base; | + // | incremental | tree; include | re-derive affected back references [1] | + // | | winner in the | rebuild inline tree | + // | | fold window | | + // |-------------------|----------------|----------------|----------------|----------------| + // | Incremental | Reuse base if | Skip: winner | Skip: winner | Skip: winner | + // | checkpoint | valid [2]; | installed an | installed an | installed a | + // | | otherwise | incremental | incremental | full tree | + // | | recreate it | tree | tree | | + // | | in-txn | | | | + // |-------------------|----------------|----------------|----------------|----------------| + // | Full | Reuse base if | Regenerate: | Regenerate: | Skip: winner | + // | checkpoint | valid [2]; | winner made | winner made | installed a | + // | | otherwise | incr. tree; | incr. tree; | full tree | + // | | regenerate via | we want full | we want full | | + // | | new txn retry | chkpt | chkpt | | + // |-------------------|----------------|----------------|----------------|----------------| + // + // [1] back references are selectively re-derived against an + // incremental winner and fully re-derived against a full winner. Back references are not + // re-derived for blind appends. With multiple tree winners, an incremental checkpoint always + // skips; a full checkpoint skips if any winner wrote a full tree, even when a later winner + // wrote an incremental tree. + // [2] The losing new tree remains valid if every winning log commit's file actions have a + // defaultRowCommitVersion newer than the losing tree's content-root version. + } + + def materializeNewTree(incremental: Boolean, trigger: String): AMTWriteResult = { + val result = materialize( + nextAttemptVersion, + currentTransactionInfo, + preCommitLogSegment, + incremental, + trigger) + metrics.singleAMTWriteMetrics = Some(result.amtWriteMetrics) + result + } + + def writeTree(): Option[AMTWriteResult] = initialOperation match { + case optimize: DeltaOperations.OptimizeCheckpoint if rebasing => + // A concurrent commit won our target version: reuse the already-written base tree as-is + // or signal a full-AMT regenerate. + handleLosingOptimizeCheckpoint( + nextAttemptVersion, + preCommitLogSegment, + currentTransactionInfo, + winningCommitMetricsForConflictedRange, + optimize.triggerName, + metrics) case optimize: DeltaOperations.OptimizeCheckpoint => assert(actionsToCommit.isEmpty, s"OPTIMIZE checkpoint commit must carry no actions, got ${actionsToCommit.size}.") @@ -127,22 +406,193 @@ class AMTWriterManager( // rewrite even when the trigger requested incremental (e.g. the JSON-size threshold). val incremental = optimize.incremental && AMTWriteHelper.previousAMTContentRoot(readSnapshot).isDefined - Some(materialize( - commitVersion, currentTransactionInfo, preCommitLogSegment, - incremental = incremental, trigger = optimize.triggerName)) + Some(materializeNewTree(incremental, optimize.triggerName)) case _ if shouldDoInlineIncrementalCheckpoint(actionsToCommit) => // A large business commit rebuilds its manifest tree inline (incrementally). val mode = AMTTriggerMode.InlineWithLargeCommitIncremental - Some(materialize( - commitVersion, currentTransactionInfo, preCommitLogSegment, - incremental = mode.isIncremental, trigger = mode.name)) + val result = materializeNewTree(mode.isIncremental, mode.name) + metrics.conflictResolutionMetrics.foreach(_.updateOutcome(REBUILT_INLINE_TREE)) + Some(result) case _ => + // A commit that writes no tree emits no AMT. + assert(!writesTree, + s"writeAMT reached the no-tree branch for a tree-writing commit: $initialOperation.") None } + + val resultOpt = try { + val result = writeTree() + recordAMTMetrics(metrics, rebasing) + result + } catch { + case e: FullAMTWriteFailedWithConflict if rebasing => + recordAMTMetrics(metrics, rebasing) + throw e + case e: ConcurrentAMTCheckpointLandedException if rebasing => + recordAMTMetrics(metrics, rebasing) + throw e + case NonFatal(e) => + recordDeltaEvent( + deltaLog, + opType = AMTUsageLogs.WRITE_FAILED, + data = Map( + "exception" -> e.getMessage, + "stackTrace" -> e.getStackTrace.take(30).mkString("\n\t"))) + throw e + } lastAMTWriteResultOpt = resultOpt + // Advance the `lastAttemptVersion` so that if we get a conflict again, we rebase + // starting from [lastAttemptVersion, ...]. + lastAttemptVersion = nextAttemptVersion resultOpt } + /** + * Asserts the winning commit metrics cover exactly the conflicted range [lastAttemptVersion, + * nextAttemptVersion) -- one per concurrent winner this attempt lost to. + */ + private def validateWinningCommitMetrics( + winningCommitMetrics: Seq[WinningCommitMetrics], + lastAttemptVersion: Long, + nextAttemptVersion: Long): Unit = { + assert( + winningCommitMetrics.size == nextAttemptVersion - lastAttemptVersion, + s"winning commit metrics (${winningCommitMetrics.size}) must cover exactly the " + + s"conflicted range [$lastAttemptVersion, $nextAttemptVersion).") + } + + /** + * Decides the AMT write for a losing OPTIMIZE checkpoint on a conflict-resolution retry. + * + * A losing OPTIMIZE checkpoint carries no user actions, so there is nothing for doCommit to + * re-stamp. When its first attempt already wrote a tree (describing the read snapshot) and it + * lost only to log-only winners, and every winner touched only files created strictly after that + * tree's content-root version (min defaultRowCommitVersion > the base version), the tree is still + * exact -- recommit it as-is (this holds whether the base was a full or an incremental rewrite). + * Otherwise a winner changed content the base tree describes, so the tree must be rebuilt: a + * losing incremental checkpoint recreates itself from scratch against the new pre-commit log + * segment inside this transaction (its base is unchanged and its fold window is small); a losing + * full checkpoint signals [[FullAMTWriteFailedWithConflict]] for the caller (CheckpointHook) to + * refresh and redo the rewrite from scratch. A winner that installed its own tree makes a + * redundant checkpoint skip as a clean no-op ([[ConcurrentAMTCheckpointLandedException]]); a + * losing full checkpoint whose winner wrote only an incremental tree signals + * [[FullAMTWriteFailedWithConflict]] to regenerate. + */ + private def handleLosingOptimizeCheckpoint( + nextAttemptVersion: Long, + preCommitLogSegment: LogSegment, + currentTransactionInfo: CurrentTransactionInfo, + winningCommitMetricsForConflictedRange: Seq[WinningCommitMetrics], + triggerName: String, + metrics: AMTMetrics): Option[AMTWriteResult] = { + def updateExceptionOutcome( + treeOutcome: String, + winnerTreeSatisfiesRequirement: Option[Boolean], + exception: Throwable): Unit = { + metrics.conflictResolutionMetrics.foreach(_.updateOutcome( + treeOutcome = treeOutcome, + winnerTreeSatisfiesRequirement = winnerTreeSatisfiesRequirement, + exceptionThrown = Some(exception.getClass.getSimpleName))) + } + + val baseResult = lastAMTWriteResultOpt.getOrElse { + // An OPTIMIZE checkpoint always materializes a tree on its first attempt, so by the time it + // rebases there must be a cached result. + throw new IllegalStateException( + "A losing OPTIMIZE checkpoint has no cached AMT write from its first attempt.") + } + if (winningCommitInstalledNewAMTTree(currentTransactionInfo)) { + // - If this txn wants an Incremental AMT checkpoint and an incremental / full AMT + // checkpoint already landed, then abort this transaction and signal this to the + // caller via a [[ConcurrentAMTCheckpointLandedException]] so it can stop retrying. + // - Same holds true when this txn wants a Full AMT and a Full AMT happens concurrently. + // - If this txn wants a Full AMT, but winning commit is an Incremental AMT, throw an + // exception [[FullAMTWriteFailedWithConflict]] so that caller can retry if they are + // not exhausted. + val loserWroteFullTree = + baseResult.checkpoint.contentRoot.isIncremental.contains(false) + val winnerWroteFullTree = winningCommitMetricsForConflictedRange.exists( + _.checkpointAction.exists(_.contentRoot.isIncremental.contains(false))) + val latestWinnerTree = currentTransactionInfo.preCommitLatestAMTCheckpointOpt.get + val lastManifestCommitOpt = + currentTransactionInfo.commitInfo.flatMap(_.lastManifestCommit) + val winnerManifestCommitVersion = + lastManifestCommitOpt.map(_.version).getOrElse(latestWinnerTree.version) + val winnerMakesCheckpointRedundant = + if (loserWroteFullTree) winnerWroteFullTree else true + if (winnerMakesCheckpointRedundant) { + val exception = DeltaErrors.concurrentAMTCheckpointLandedException( + latestManifestCommitVersion = winnerManifestCommitVersion, + latestContentRootVersion = lastManifestCommitOpt.map(_.contentRootVersion) + .getOrElse(latestWinnerTree.contentRoot.version)) + updateExceptionOutcome( + treeOutcome = SKIP_WINNER_SATISFIES_REQUIREMENT, + winnerTreeSatisfiesRequirement = Some(true), + exception = exception) + throw exception + } + val exception = DeltaErrors.fullAMTWriteFailedWithConflict( + conflictingCommitVersion = winnerManifestCommitVersion) + updateExceptionOutcome( + treeOutcome = RETRY_VIA_NEW_TXN, + winnerTreeSatisfiesRequirement = Some(false), + exception = exception) + throw exception + } + val baseVersion = baseResult.contentRootVersion + if (winningCommitMetricsForConflictedRange.forall( + _.allFileActionsHaveDefaultCommitVersionNewerThan(baseVersion))) { + // Every file action in every winner commit is a new FileAction with + // defaultRowCommitVersion > Losing FULL AMT's checkpoint version (X) + // This means, none of them should have backreferences also. + // Reasoning: if they have backreferences (which points to an old tree pointing + // to version < X) + // => they are old files (readded / removed) + // => they should have old defaultRowCommitVersion + // Which contradicts the above check. + winningCommitMetricsForConflictedRange.foreach { m => + AMTUtils.invariantCheckWithLogging( + checkInvariant = m.numAddFilesWithBackreferences == 0 && + m.numRemoveFilesWithBackreferences == 0, + opTypeSuffix = + AMTUsageLogs.ALERT_FILE_CONTAINS_NEW_SEQ_NUMBERS_BUT_NON_EMPTY_BACKREFERENCE, + message = "A base-preserving winner must carry no back references into the " + + s"base tree at version $baseVersion, but found " + + s"${m.numAddFilesWithBackreferences} Add and " + + s"${m.numRemoveFilesWithBackreferences} Remove file actions with back references.", + deltaLog = deltaLog) + } + metrics.conflictResolutionMetrics.foreach(_.updateOutcome(REUSED_LOSING_TREE)) + Some(baseResult) + } else if (baseResult.checkpoint.contentRoot.isIncremental.contains(true)) { + // A winner changed content the losing INCREMENTAL checkpoint's base describes, so it cannot + // be reused as-is. Its base tree (the prior checkpoint it extends) is unchanged -- the + // winners were log-only -- and its fold window is small, so recreate the incremental + // checkpoint from scratch against the new pre-commit log segment (folding the winners in) + // inside this transaction, rather than raising [[FullAMTWriteFailedWithConflict]] and + // regenerating it from scratch via an external retry. + metrics.conflictResolutionMetrics.foreach( + _.updateOutcome(REBUILT_OPTIMIZE_CHECKPOINT_INCREMENTAL_TREE)) + val result = materialize( + nextAttemptVersion, currentTransactionInfo, preCommitLogSegment, + incremental = true, trigger = triggerName) + metrics.singleAMTWriteMetrics = Some(result.amtWriteMetrics) + Some(result) + } else { + // A winner changed content the full base tree describes, so it cannot be reused as-is. + // The caller should do a retry in this case. + val winnerWroteTree = + winningCommitMetricsForConflictedRange.exists(_.checkpointAction.isDefined) + val exception = DeltaErrors.fullAMTWriteFailedWithConflict( + conflictingCommitVersion = preCommitLogSegment.version) + updateExceptionOutcome( + treeOutcome = RETRY_VIA_NEW_TXN, + winnerTreeSatisfiesRequirement = if (winnerWroteTree) Some(false) else None, + exception = exception) + throw exception + } + } + /** * Whether this Writer should write its changed actions inline as part of a new AMT. * True only when the commit is large enough (by action count) AND the table already has a full @@ -152,6 +602,24 @@ class AMTWriterManager( actionsToCommit.size.toLong >= largeCommitActionsCountThresholdForInlineManifestCommit && AMTWriteHelper.previousAMTContentRoot(readSnapshot).isDefined + /** True when there was a winning manifest commit concurrent to this transaction */ + private def winningCommitInstalledNewAMTTree( + currentTransactionInfo: CurrentTransactionInfo): Boolean = { + val readSnapshotAMTVersion = readSnapshot.lastManifestCommitOpt.map(_.contentRootVersion) + val preCommitAMTVersion = currentTransactionInfo.preCommitLatestAMTCheckpointOpt.map(_.version) + (readSnapshotAMTVersion, preCommitAMTVersion) match { + case (Some(_), None) => + throw new IllegalStateException( + "The read snapshot has an AMT but the winning commits has no AMT -- this can happen " + + "only during downgrade -- not supported yet") + case (Some(readVersion), Some(foldedVersion)) if readVersion > foldedVersion => + throw new IllegalStateException( + s"The rebased AMT moved backwards: read-snapshot tree version $readVersion is newer " + + s"than the folded tree version $foldedVersion.") + case (readOpt, foldedOpt) => readOpt != foldedOpt + } + } + // Materializes the manifest tree for this commit and records its metrics. An incremental rewrite // packs the post-commit live files into leaves in input order on the driver; a full rewrite // clusters the read snapshot's live files and flushes them into leaves distributed across @@ -162,37 +630,299 @@ class AMTWriterManager( preCommitLogSegment: LogSegment, incremental: Boolean, trigger: String): AMTWriteResult = { - val amtProviderOpt = readSnapshot.checkpointProvider match { - case amt: AMTCheckpointProvider => Some(amt) - case _ => None + val amtProviderOpt = preCommitLatestAMTCheckpointProvider.providerOpt + assert( + amtProviderOpt.map(_.checkpointAction.version) == + currentTransactionInfo.preCommitLatestAMTCheckpointOpt.map(_.version), + s"Cached AMT provider ${amtProviderOpt.map(_.checkpointAction.version)} is out of sync " + + "with preCommitLatestAMTCheckpointOpt " + + s"${currentTransactionInfo.preCommitLatestAMTCheckpointOpt.map(_.version)}.") + if (incremental) { + // A retry may have advanced to a winning AMT. Prefer that tree over snapshot bootstrap. + val (baseActionsProvider, oldAMTVersion): (BaseAMTActionsProvider, Long) = + amtProviderOpt match { + case Some(amtProvider) => + (new BaseAMTCheckpointActionsProvider(deltaLog, amtProvider), + amtProvider.checkpointAction.contentRoot.version) + case None => + (new BaseSnapshotActionsProvider(readSnapshot), readSnapshot.version) + } + // The commits written after the old AMT, up to the last committed version. + val intermediateLogCommits = preCommitLogSegment.deltas + .filter(f => FileNames.getFileVersion(f) > oldAMTVersion) + new IncrementalAMTWriter(spark, deltaLog).writeIncremental( + oldAMTActionsProvider = baseActionsProvider, + intermediateLogCommits = intermediateLogCommits, + attemptVersion = commitVersion, + actionsToCommit = currentTransactionInfo.actions, + trigger = trigger) + } else { + assert(currentTransactionInfo.actions.isEmpty, + "A full AMT rewrite must carry no actions, got " + + s"${currentTransactionInfo.actions.size}.") + AMTWriteHelper.writeFullMaterialization( + spark = spark, + readSnapshot = readSnapshot, + commitVersion = commitVersion, + postCommitProtocol = currentTransactionInfo.protocol, + postCommitMetadata = currentTransactionInfo.metadata, + trigger = trigger) + } + } + + private def largeCommitActionsCountThresholdForInlineManifestCommit: Long = + spark.sessionState.conf.getConf( + DeltaSQLConf.AMT_LARGE_COMMIT_ACTIONS_COUNT_THRESHOLD_FOR_INLINE_MANIFEST_COMMIT) + + /** + * Updates the pre-commit AMTCheckpointProvider after resolving conflicts via [[ConflictChecker]]. + */ + def updatePreCommitLatestAMTCheckpointProvider( + currentTransactionInfo: CurrentTransactionInfo): Unit = { + val manifestCommitVersion = currentTransactionInfo.commitInfo + .flatMap(_.lastManifestCommit).map(_.version) + .orElse(currentTransactionInfo.preCommitLatestAMTCheckpointOpt.map(_.version)).getOrElse(0L) + preCommitLatestAMTCheckpointProvider = new LazyAMTCheckpointProvider( + currentTransactionInfo.preCommitLatestAMTCheckpointOpt, readSnapshot, manifestCommitVersion) + } + + /** + * Re-derives the file actions' back references against the AMT this attempt builds on. Only runs + * on a rebase where a winning commit installed a new tree; `reStampBackReferences` re-derives + * each file action whose back reference that tree invalidated (a leaf it dropped or a position it + * newly MDV-masked because the file moved or was removed) and leaves the rest -- those still + * pointing at a live leaf entry -- unchanged. A blind append is skipped entirely: it only adds + * brand-new files, so none of its actions can point at a leaf the winner's tree invalidated. + */ + def rebaseBackReferences( + currentTransactionInfo: CurrentTransactionInfo): CurrentTransactionInfo = { + pendingBackRefRebaseMetrics = + BackRefRebaseMetrics(SKIP_REASON_NO_NEW_TREE) + val actions = currentTransactionInfo.actions + if (!AMTUtils.amtEnabled(readSnapshot)) { + return currentTransactionInfo + } + if (!winningCommitInstalledNewAMTTree(currentTransactionInfo)) { + return currentTransactionInfo + } + // A blind append only adds brand-new files and reads or removes nothing, so none of its actions + // reference a leaf the winner's tree could have dropped or MDV-masked. Those new files are + // absent from the winner's tree and get a fresh back reference when this attempt's own tree + // folds them in, so skip the re-stamp rather than re-deriving back references that do not exist + // in the winner's tree. + if (currentTransactionInfo.commitInfo.flatMap(_.isBlindAppend).getOrElse(false)) { + pendingBackRefRebaseMetrics = + BackRefRebaseMetrics(SKIP_REASON_BLIND_APPEND) + return currentTransactionInfo + } + // A commit with no file actions has no back references to re-derive, so short-circuit before + // materializing the winning tree's (potentially expensive) AMT provider. + if (!actions.exists(_.isInstanceOf[FileAction])) { + pendingBackRefRebaseMetrics = + BackRefRebaseMetrics(SKIP_REASON_NO_FILE_ACTIONS) + return currentTransactionInfo } - val (result, singleMetric) = - if (incremental && amtProviderOpt.isDefined) { - val oldAMTVersion = amtProviderOpt.get.checkpointAction.version - // The commits written after the old AMT, up to the last committed version. - val intermediateLogCommits = preCommitLogSegment.deltas - .filter(f => FileNames.getFileVersion(f) > oldAMTVersion) - new IncrementalAMTWriter(spark, deltaLog).writeIncremental( + val foldedAMTVersion = currentTransactionInfo.preCommitLatestAMTCheckpointOpt.map(_.version) + if (foldedAMTVersion == lastRebasedAMTVersion) { + // No new tree was installed since the last rebase, so the actions are already re-stamped + // against it -- nothing to re-derive. + pendingBackRefRebaseMetrics = + BackRefRebaseMetrics(SKIP_REASON_ALREADY_REBASED) + return currentTransactionInfo + } + // The tree the actions were last stamped against: the previous rebase target, or -- on the + // first rebase -- the read snapshot's own tree, which is what the writer originally stamped. + val oldAMTVersion = lastRebasedAMTVersion + .orElse(readSnapshot.lastManifestCommitOpt.map(_.contentRootVersion)) + .getOrElse(0L) + val foldedContentRootVersion = + currentTransactionInfo.preCommitLatestAMTCheckpointOpt.map(_.contentRoot.version) + val providerContentRootVersion = + preCommitLatestAMTCheckpointProvider.providerOpt.map(_.checkpointAction.contentRoot.version) + assert(foldedContentRootVersion == providerContentRootVersion, + "the cached AMT provider must correspond to the transaction's folded AMT checkpoint.") + val startNs = System.nanoTime() + val restampedActions = preCommitLatestAMTCheckpointProvider.providerOpt match { + case Some(provider) => + val result = provider.reStampBackReferences(spark, deltaLog, actions) + val rebaseMetrics = BackRefRebaseMetrics( oldAMTVersion = oldAMTVersion, - oldAMTCheckpointProvider = amtProviderOpt.get, - intermediateLogCommits = intermediateLogCommits, - attemptVersion = commitVersion, - actionsToCommit = currentTransactionInfo.actions, - trigger = trigger) + newAMTVersion = foldedAMTVersion.getOrElse(oldAMTVersion), + totalTimeTakenMs = + TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNs), + numActionsReusingBackref = result.numActionsReusingBackref, + numActionsRegeneratingBackref = result.numActionsRegeneratingBackref) + pendingBackRefRebaseMetrics = rebaseMetrics + result.actions + case None => + pendingBackRefRebaseMetrics = + BackRefRebaseMetrics(SKIP_REASON_NO_AMT_PROVIDER) + actions + } + lastRebasedAMTVersion = foldedAMTVersion + DeltaTestBarrier.waitIfEnabled(AMTWriterManager.REBASE_BACK_REFERENCES_TEST_BARRIER) + currentTransactionInfo.copy(actions = restampedActions) + } + // **************** AMT Metric related helpers **************** + + private def initializeAMTMetricsDuringRebase( + metrics: AMTMetrics, + nextAttemptVersion: Long, + winningCommitMetrics: Seq[WinningCommitMetrics]): Unit = { + metrics.conflictResolutionMetrics = Some(AMTConflictResolutionRoundMetrics( + losingAttemptVersion = lastAttemptVersion, + nextAttemptVersion = nextAttemptVersion, + losingCommitType = initialOperation match { + case _: DeltaOperations.OptimizeCheckpoint => + if (lastAMTWriteResultOpt.exists( + _.checkpoint.contentRoot.isIncremental.contains(true))) { + INCREMENTAL_CHECKPOINT + } else { + FULL_CHECKPOINT + } + case _ if lastAMTWriteResultOpt.isDefined => INLINE_INCREMENTAL + case _ => LOG_ONLY + }, + losingTreeType = lastAMTWriteResultOpt.map { result => + if (result.checkpoint.contentRoot.isIncremental.contains(true)) { + INCREMENTAL_TREE + } else { + FULL_TREE + } + }, + winningCommits = summarizeWinningCommits( + winningCommitMetrics, lastAttemptVersion, nextAttemptVersion), + backRefRebaseMetrics = pendingBackRefRebaseMetrics, + treeOutcome = NO_TREE_REBASE, + winnerTreeSatisfiesRequirement = None, + exceptionThrown = None)) + } + + private def summarizeWinningCommits( + winningCommitMetrics: Seq[WinningCommitMetrics], + firstVersion: Long, + nextAttemptVersion: Long): WinningAMTCommitSummary = { + var numLogOnly = 0 + var numInlineIncremental = 0 + var numIncrementalCheckpoints = 0 + var numFullCheckpoints = 0 + winningCommitMetrics.foreach { winner => + winner.checkpointAction match { + case None => numLogOnly += 1 + case Some(checkpoint) if checkpoint.contentRoot.isIncremental.contains(false) => + numFullCheckpoints += 1 + case Some(_) if winner.numAdds + winner.numRemoves > 0 => + numInlineIncremental += 1 + case Some(_) => numIncrementalCheckpoints += 1 + } + } + val allWinnersAreLogOnly = numLogOnly == winningCommitMetrics.size + val allLogWinnersPreserveLosingTree = lastAMTWriteResultOpt.flatMap { result => + if (winningCommitMetrics.nonEmpty && allWinnersAreLogOnly) { + Some(winningCommitMetrics.forall( + _.allFileActionsHaveDefaultCommitVersionNewerThan(result.contentRootVersion))) } else { - assert(currentTransactionInfo.actions.isEmpty, - "A full AMT rewrite must carry no actions, got " + - s"${currentTransactionInfo.actions.size}.") - AMTWriteHelper.writeFullMaterialization( - spark = spark, - readSnapshot = readSnapshot, - commitVersion = commitVersion, - postCommitProtocol = currentTransactionInfo.protocol, - postCommitMetadata = currentTransactionInfo.metadata, - trigger = trigger) + None } - metrics.attempts :+= singleMetric - result + } + WinningAMTCommitSummary( + firstVersion = firstVersion, + lastVersion = nextAttemptVersion - 1, + numLogOnly = numLogOnly, + numInlineIncremental = numInlineIncremental, + numIncrementalCheckpoints = numIncrementalCheckpoints, + numFullCheckpoints = numFullCheckpoints, + allLogWinnersPreserveLosingTree = allLogWinnersPreserveLosingTree) + } + + private def recordAMTMetrics(metrics: AMTMetrics, rebasing: Boolean): Unit = { + assert(!rebasing || metrics.conflictResolutionMetrics.isDefined) + if (metrics.conflictResolutionMetrics.isEmpty && metrics.singleAMTWriteMetrics.isEmpty) return + recordDeltaEvent( + deltaLog, opType = AMTUsageLogs.CONFLICT_RESOLUTION_ROUND, data = metrics) + if (rebasing) { + pendingBackRefRebaseMetrics = BackRefRebaseMetrics(SKIP_REASON_NO_NEW_TREE) + } + } + +} +object AMTWriterManager extends DeltaLogging { + /** + * Test barrier label. When a suite arms this barrier, [[AMTWriterManager.rebaseBackReferences]] + * pauses right after a rebase that installed a new tree, letting the suite commit another + * concurrent winner and drive a second conflict round. + */ + private[delta] val REBASE_BACK_REFERENCES_TEST_BARRIER = + "AMTWriterManager.rebaseBackReferences" + + /** + * Test barrier label. When a suite arms this barrier, [[emitAMTCheckpoint]] pauses on each + * full-AMT regenerate retry (after refreshing its snapshot), letting the suite land another + * conflicting winner and drive the retry loop to its configured bound. + */ + private[delta] val FULL_AMT_REGENERATE_RETRY_TEST_BARRIER = + "AMTWriterManager.fullAMTRegenerateRetry" + + + /** + * Emits the AMT for `snapshot` by committing a follow-up OPTIMIZE CHECKPOINT that + * rewrites the manifest tree, full or incremental per `amtTriggerModeOpt` (full when absent). + * + * A losing full checkpoint that cannot reuse its base against a concurrent winner surfaces a + * [[FullAMTWriteFailedWithConflict]]; refresh the snapshot and retry, bounded so a pathological + * run of concurrent winners cannot spin forever. At least one attempt always runs. + */ + def emitAMTCheckpoint( + snapshot: Snapshot, + catalogTableOpt: Option[CatalogTable], + amtTriggerModeOpt: Option[AMTTriggerMode]): Unit = { + val triggerMode = amtTriggerModeOpt.getOrElse(AMTTriggerMode.OnDemandCheckpointFull) + val deltaLog = snapshot.deltaLog + var attemptsRemaining = math.max(1, SparkSession.active.sessionState.conf.getConf( + DeltaSQLConf.AMT_CONFLICT_CHECKING_MAX_FULL_REGENERATE_RETRIES)) + var readSnapshot: Snapshot = snapshot + while (attemptsRemaining > 0) { + attemptsRemaining -= 1 + val checkpointTxn = deltaLog.startTransaction(catalogTableOpt, Some(readSnapshot)) + val attemptStartNs = System.nanoTime() + try { + checkpointTxn.commit( + Seq.empty, + DeltaOperations.OptimizeCheckpoint(triggerMode.isIncremental, triggerMode.name)) + return + } catch { + case e: ConcurrentAMTCheckpointLandedException => + // A concurrent winner already installed an up-to-date AMT tree while this maintenance + // checkpoint was rebasing, so its work is redundant. Skip it as a graceful no-op -- the + // winner's tree already serves as the checkpoint -- rather than surfacing an error or + // rescheduling via a deferred ConcurrentWriteException. + logInfo(log"Skipping redundant AMT checkpoint on " + + log"${MDC(DeltaLogKeys.PATH, deltaLog.dataPath)}: a concurrent commit already " + + log"installed an AMT tree at manifest commit version " + + log"${MDC(DeltaLogKeys.VERSION, e.manifestCommitVersion)} " + + log"(content-root version ${MDC(DeltaLogKeys.VERSION2, e.contentRootVersion)}).") + return + case e: FullAMTWriteFailedWithConflict if attemptsRemaining > 0 => + // A concurrent winner changed content the base tree describes, so this full checkpoint + // cannot reuse it and must regenerate its full AMT against the post-winner snapshot. + // Refresh and retry rather than surfacing the conflict. + recordDeltaEvent( + deltaLog, + opType = AMTUsageLogs.CHECKPOINT_FULL_REGENERATE_RETRY, + data = Map( + "conflictingCommitVersion" -> e.conflictingCommitVersion, + "attemptsRemaining" -> attemptsRemaining, + "timeTakenMs" -> + TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - attemptStartNs))) + logInfo(log"Retrying full AMT checkpoint on " + + log"${MDC(DeltaLogKeys.PATH, deltaLog.dataPath)} after a concurrent commit at " + + log"version ${MDC(DeltaLogKeys.VERSION, e.conflictingCommitVersion)}; regenerating " + + log"against the refreshed snapshot.") + readSnapshot = deltaLog.update(catalogTableOpt = catalogTableOpt) + // Lets a test land another conflicting winner between retries to exercise the bound. + DeltaTestBarrier.waitIfEnabled(FULL_AMT_REGENERATE_RETRY_TEST_BARRIER) + } + } } /** @@ -200,17 +930,21 @@ class AMTWriterManager( * The maintenance work will be done by CheckpointHook */ def planMaintenance( + spark: SparkSession, + readSnapshot: Snapshot, + initialOperation: DeltaOperations.Operation, commitVersion: Long, postCommitSnapshot: Snapshot): MaintenanceOperation = { // if the commit itself was to do a checkpoint, don't schedule any maintenance as part // of its post-commit hook. - if (!amtEnabled(commitVersion) + if (!AMTUtils.amtEnabled(readSnapshot) || initialOperation.isInstanceOf[DeltaOperations.OptimizeCheckpoint]) { return MaintenanceOperation() } - val amtTriggerModeOpt = followUpTriggerMode(commitVersion, postCommitSnapshot) + val amtTriggerModeOpt = + followUpTriggerMode(spark, readSnapshot, commitVersion, postCommitSnapshot) MaintenanceOperation( shouldCheckpoint = amtTriggerModeOpt.isDefined, amtTriggerModeOpt = amtTriggerModeOpt) @@ -224,15 +958,18 @@ class AMTWriterManager( * checkpointInterval * fullRewriteCheckpointIntervalMultiplier. */ def planMaintenanceAfterInlineWrite( + spark: SparkSession, + readSnapshot: Snapshot, + initialOperation: DeltaOperations.Operation, commitVersion: Long, postCommitSnapshot: Snapshot): MaintenanceOperation = { // The follow-up OPTIMIZE CHECKPOINT commit itself must never schedule more maintenance. - if (!amtEnabled(commitVersion) + if (!AMTUtils.amtEnabled(readSnapshot) || initialOperation.isInstanceOf[DeltaOperations.OptimizeCheckpoint]) { return MaintenanceOperation() } - val checkpointInterval = deltaLog.checkpointInterval(postCommitSnapshot.metadata) - if (isFullCheckpointOverdue(commitVersion, postCommitSnapshot, checkpointInterval)) { + val checkpointInterval = readSnapshot.deltaLog.checkpointInterval(postCommitSnapshot.metadata) + if (isFullCheckpointOverdue(spark, commitVersion, postCommitSnapshot, checkpointInterval)) { MaintenanceOperation( shouldCheckpoint = true, amtTriggerModeOpt = Some(AMTTriggerMode.CheckpointIntervalFull)) @@ -243,9 +980,11 @@ class AMTWriterManager( /** [[AMTTriggerMode]] for a followup AMT Checkpoint commit if any. */ private def followUpTriggerMode( + spark: SparkSession, + readSnapshot: Snapshot, commitVersion: Long, postCommitSnapshot: Snapshot): Option[AMTTriggerMode] = { - val checkpointInterval = deltaLog.checkpointInterval(postCommitSnapshot.metadata) + val checkpointInterval = readSnapshot.deltaLog.checkpointInterval(postCommitSnapshot.metadata) // -- case-1 -- // Assume v0 has an AMT. This is to make sure future AMTs land on even boundaries // e.g. 10/20/30 instead of 9/19/29 (as classic checkpoints do). @@ -260,7 +999,8 @@ class AMTWriterManager( // If checkpointInterval is 200 and fullRewriteCheckpointIntervalMultiplier is 5 // Then if 10220 is full tree, then 10420, 10620, 10820, 11020 will be incremental // and then 11220 will be full tree again. - val fullRewriteSpan = checkpointInterval.toLong * fullRewriteCheckpointIntervalMultiplier + val fullRewriteSpan = + checkpointInterval.toLong * fullRewriteCheckpointIntervalMultiplier(spark) val needsFullRewrite = AMTWriteHelper.previousAMTContentRoot(postCommitSnapshot) .flatMap(_.lastManifestCommitWithFullRewrite) .forall(lastFull => commitVersion - lastFull >= fullRewriteSpan) @@ -283,7 +1023,7 @@ class AMTWriterManager( // version a full span has elapsed, and a racing follow-up that has not landed yet does not // re-trigger on the very next commit (only once per interval), matching case-1's racing // behavior. - if (isFullCheckpointOverdue(commitVersion, postCommitSnapshot, checkpointInterval)) { + if (isFullCheckpointOverdue(spark, commitVersion, postCommitSnapshot, checkpointInterval)) { return Some(AMTTriggerMode.CheckpointIntervalFull) } @@ -298,10 +1038,11 @@ class AMTWriterManager( * at most once per interval, not on every commit -- matching `followUpTriggerMode`'s case-1. */ private def isFullCheckpointOverdue( + spark: SparkSession, commitVersion: Long, postCommitSnapshot: Snapshot, checkpointInterval: Long): Boolean = { - val fullRewriteSpan = checkpointInterval * fullRewriteCheckpointIntervalMultiplier + val fullRewriteSpan = checkpointInterval * fullRewriteCheckpointIntervalMultiplier(spark) AMTWriteHelper.previousAMTContentRoot(postCommitSnapshot) .flatMap(_.lastManifestCommitWithFullRewrite) .exists { lastFull => @@ -311,20 +1052,7 @@ class AMTWriterManager( } } - /** - * Whether AMT write is possible at all for this commit. - * Note: We don't create AMT at commit 0 as of now but this could be relaxed in future. - */ - private def amtEnabled(commitVersion: Long): Boolean = { - if (commitVersion <= 0) return false - readSnapshot.protocol.isFeatureSupported(AdaptiveMetadataTableFeature) - } - - private def largeCommitActionsCountThresholdForInlineManifestCommit: Long = - spark.sessionState.conf.getConf( - DeltaSQLConf.AMT_LARGE_COMMIT_ACTIONS_COUNT_THRESHOLD_FOR_INLINE_MANIFEST_COMMIT) - - private def fullRewriteCheckpointIntervalMultiplier: Int = + private def fullRewriteCheckpointIntervalMultiplier(spark: SparkSession): Int = spark.sessionState.conf.getConf( DeltaSQLConf.AMT_FULL_REWRITE_CHECKPOINT_INTERVAL_MULTIPLIER) } diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/amt/IncrementalAMTWriter.scala b/spark/src/main/scala/org/apache/spark/sql/delta/amt/IncrementalAMTWriter.scala index 8b7f43baff7..1ee86c24e67 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/amt/IncrementalAMTWriter.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/amt/IncrementalAMTWriter.scala @@ -21,12 +21,12 @@ import java.util.concurrent.TimeUnit.NANOSECONDS import scala.collection.mutable.ArrayBuffer import org.apache.spark.sql.delta.{DeltaFileProviderUtils, DeltaLog, SingleCommit} -import org.apache.spark.sql.delta.actions.{Action, AddFile, Checkpoint, ContentRoot, InMemoryLogReplay, RemoveFile} -import org.apache.spark.sql.delta.actions.InMemoryLogReplay.UniqueFileActionTuple -import org.apache.spark.sql.delta.deletionvectors.RoaringBitmapArray +import org.apache.spark.sql.delta.RowId.RowTrackingMetadataDomain +import org.apache.spark.sql.delta.actions.{Action, AddFile, BackReference, Checkpoint, ContentRoot, DomainMetadata, FileAction, InMemoryLogReplay, Metadata, Protocol, RemoveFile, SetTransaction} +import org.apache.spark.sql.delta.actions.FileAction.UniqueFileActionTuple import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util.FileNames -import org.apache.hadoop.fs.FileStatus +import org.apache.hadoop.fs.{FileStatus, Path} import org.apache.spark.sql.SparkSession @@ -43,15 +43,37 @@ import org.apache.spark.sql.SparkSession * (part-2) + commitActions that we want to commit (part-3) -- the logReplay of above becomes * [[DataEntry]]s for the new AMT root. * - The previous tree's leaves ([[DataManifestEntry]]s) are carried forward by pointer. - * - The [[RemoveFile]] actions from part-2 / part-3 that have backreferences contribute towards - * MDVs for existing leafs. + * - The [[AddFile]] / [[RemoveFile]] actions from part-2 / part-3 that have backreferences + * contribute towards MDVs for existing leafs. * - Tombstones: * - The RemoveFile from the current proposed commit (part-3) contributes for CDF. - * - The one with backreferences contributes to deleted_positions on the [[DataManifestEntry]] - * corresponding to existing leafs. - * - The one without backreferences contributes to a tombstone [[DataEntry]] in the new root. - * - If the root crosses the maxEntriesPerLeaf threshold, live [[DataEntry]]s are spilled to a new - * leaf. + * - The one with backreferences contributes to deleted_positions / replaced_positions on the + * [[DataManifestEntry]] corresponding to existing leafs. + * - The one without backreferences contributes to a tombstone [[DataEntry]] in the new root / + * spilled leaves. + * - If the root crosses the maxEntriesPerLeaf threshold, live [[DataEntry]]s / tombstone + * [[DataEntry]]s are spilled to a new leaf. + * + * [[DataEntry]] tracking.status is set to reflect THIS commit's CDF: only this commit's own actions + * contribute ADDED (inserts), DELETED (deletes), and REPLACED + MODIFIED (updates -- the prior and + * new copies of a re-added file). A file carried forward untouched is EXISTING and contributes no + * CDF. Window (intermediate) commits already emitted their CDF, so their files carry forward as + * EXISTING, not re-attributed to this commit. + * + * [[DataManifestEntry]] tracking.status state transitions (re-derived on every carry-forward to a + * new AMT, from the masking the leaf gains that commit -- prior live status does not matter): + * - A new leaf is always born ADDED, whether it holds spilled live [[DataEntry]]s or only spilled + * tombstones (a tombstone-only leaf holds no live file, but is still born ADDED). + * - ADDED -> + * - EXISTING -- the leaf still holds a live file and gains no new masking this commit. + * - MODIFIED -- this commit masks some, but not all, of its live entries. + * - DELETED -- the leaf holds no live file: this commit masks its last live entry, or it only + * ever held tombstones; the pointer becomes a tombstone. + * - EXISTING -> + * - EXISTING / MODIFIED / DELETED -- same rule as ADDED, from this commit's masking. + * - MODIFIED -> + * - MODIFIED / EXISTING / DELETED -- same rule as ADDED, from this commit's masking. + * - DELETED is terminal: the tombstone pointer is emitted once, then dropped from the next AMT. */ class IncrementalAMTWriter(spark: SparkSession, deltaLog: DeltaLog) { @@ -60,6 +82,8 @@ class IncrementalAMTWriter(spark: SparkSession, deltaLog: DeltaLog) { private val hadoopConf = deltaLog.newDeltaHadoopConf() private val tableRoot = deltaLog.dataPath + // Always use object identity for AMT tables here. + private val useDeletionVectorObjectIdentity = true private val fs = tableRoot.getFileSystem(hadoopConf) private val metadataDir = FileNames.amtMetadataDirPath(tableRoot) @@ -67,8 +91,8 @@ class IncrementalAMTWriter(spark: SparkSession, deltaLog: DeltaLog) { * Materializes the incremental manifest for `attemptVersion` and returns the inline * [[Checkpoint]] write result plus its metrics. * - * @param oldAMTVersion version the previous AMT checkpoint describes. - * @param oldAMTCheckpointProvider provider for the previous AMT (root, leaves, inline state). + * @param oldAMTActionsProvider provider for the previous AMT (version, root files, leaves, + * inline state); loaded once at the start of this write. * @param intermediateLogCommits the commit log files written after the old AMT and up to the * last committed version (the window between the old AMT and * this commit), in commit order. @@ -78,253 +102,368 @@ class IncrementalAMTWriter(spark: SparkSession, deltaLog: DeltaLog) { * @param trigger trigger name recorded in metrics. */ def writeIncremental( - oldAMTVersion: Long, - oldAMTCheckpointProvider: AMTCheckpointProvider, + oldAMTActionsProvider: BaseAMTActionsProvider, intermediateLogCommits: Seq[FileStatus], attemptVersion: Long, actionsToCommit: Seq[Action], - trigger: String): (AMTWriteResult, SingleAMTWriteMetrics) = { + trigger: String): AMTWriteResult = { val startNanos = System.nanoTime() - val oldCheckpoint = oldAMTCheckpointProvider.checkpointAction + val oldAMT = oldAMTActionsProvider.load() + val oldAMTVersion = oldAMT.version // ---- Step 0: validate the window is contiguous and lines up with our assumptions. ---- // The intermediate commits after the old AMT must contiguously cover // [oldAMTVersion + 1, attemptVersion) with no holes. assertWindowCoversRange(intermediateLogCommits, oldAMTVersion, attemptVersion) - // ---- Step 1: gather the actions of all three parts. ---- + // ---- Step 1: gather the actions of all three parts and process them. ---- // 1.a: old AMT root -- its live root-resident files, plus its inline non-content state // (protocol, metadata, setTxns, domainMetadata). Leaf-resident files are NOT read here. - val fileActionsFromOldRoot = AMTCheckpointProvider.readLiveRootDataEntries( - spark, deltaLog, oldCheckpoint) - val nonContentFromOldRoot: Iterator[Action] = - Iterator(oldCheckpoint.protocol, oldCheckpoint.metaData) ++ - oldCheckpoint.txns.iterator ++ oldCheckpoint.domainMetadata.iterator + val fileActionsFromOldRoot = oldAMT.fileActionsFromRoot + val nonContentFromOldCheckpoint: Seq[Action] = + Seq[Action](oldAMT.protocol, oldAMT.metadata) ++ + oldAMT.setTransactions ++ oldAMT.domainMetadatas // 1.b: the log commits / minor compactions committed after the old AMT, read in parallel (the // MinorCompactionHook primitive). Each segment delta file becomes a SingleCommit keyed by its // version (a compacted delta's version is its endVersion, via getFileVersion). val windowCommits = intermediateLogCommits.map(f => SingleCommit(deltaLog, FileNames.getFileVersion(f), f)) val actionsFromDeltas = - DeltaFileProviderUtils.parallelReadAndParseDeltaFilesAsSeq(spark, deltaLog, windowCommits) + DeltaFileProviderUtils.parallelReadAndParseDeltaFilesAsSeq(spark, windowCommits) // 1.c: this commit attempt's own actions (actionsToCommit). + val processedActions = new ProcessedActions( + oldAMTVersion = oldAMTVersion, + oldRootAdds = fileActionsFromOldRoot, + nonContentFromOldCheckpoint = nonContentFromOldCheckpoint, + windowCommits = windowCommits, + windowCommitActions = actionsFromDeltas, + attemptVersion = attemptVersion, + actionsToCommit = actionsToCommit, + tableRoot = tableRoot, + useDeletionVectorObjectIdentity = useDeletionVectorObjectIdentity) + + // ---- Step 2: carry the old leaf pointers forward, patching MDV + CDF positions. ---- + val (carriedLeafPointers, leafPositions) = + carryForwardLeaves( + oldAMT.allLeafs, + processedActions.mdvSupersededBackrefs, + processedActions.cdfDeletedBackrefs, + processedActions.cdfReplacedBackrefs) - // ---- Step 2: log-replay all three parts, keyed by their real commit versions. ---- - // The version handed to `append` feeds InMemoryLogReplay's monotonic-progress assert: the old - // AMT root describes oldAMTVersion, each window delta carries its own commit version - // (FileNames.getFileVersion; for a minor compaction that is a good proxy), and this commit - // lands at attemptVersion. Because 1.a seeds the old inline non-content state and 1.b/1.c - // override it (last-writer-wins), the replay's protocol/metadata/setTxns/domainMetadata equal - // the post-commit values. - val replay = new InMemoryLogReplay( - minFileRetentionTimestamp = None, minSetTransactionRetentionTimestamp = None) - replay.append( - oldAMTVersion, - fileActionsFromOldRoot.iterator ++ nonContentFromOldRoot) - windowCommits.zip(actionsFromDeltas).foreach { case (commit, actions) => - replay.append(commit.version, actions.iterator) + // ---- Step 3: build the root DATA entries (with per-file status). ---- + // 3.a: live files -- classify each by whether THIS commit changed it: + // - ADDED if inserted by this commit (not live before it). + // - MODIFIED if replaced (re-added under a new DV) this commit. + // - else EXISTING -- carried from the old root, or a window (intermediate) commit, whose own + // CDF already fired, so this commit proposes no change to it. + val liveAddFiles = processedActions.liveAddFiles + // A re-add of a file already live before this commit (carries a leaf back reference, or its key + // is in the pre-commit live set; not a replace) is a metadata-only refresh: it stays EXISTING + // and its old leaf slot is MDV-masked (ProcessedActions asserts the re-add invariants). + def trackingForLiveAdd(add: AddFile): Tracking = { + if (processedActions.replacedPaths.contains(add.path)) { + AMTWriteHelper.modifiedTrackingForDataEntry() + } else if (add.backReference.isDefined || + processedActions.preCommitLiveKeys.contains( + add.toUniqueFileActionTuple(tableRoot, useDeletionVectorObjectIdentity))) { + AMTWriteHelper.existingTrackingForDataEntry() + } else { + AMTWriteHelper.addedTrackingForDataEntry() + } } - replay.append(attemptVersion, actionsToCommit.iterator) - - // ---- Step 3: gather the removes that drive MDV masking and CDF, from different scopes. ---- - val windowActions = actionsFromDeltas.flatten - // 3.a: MDV masking -- ALL leaf-resident removes since the old AMT (window + commit). The MDV - // must hide every entry deleted since the last full tree, so it accumulates the window's - // removes too, regardless of which commit deleted them. - // NOTE: this can NOT come from `replay.getActiveRemoveFiles`. Leaf files are not seeded into - // the replay, and a leaf file removed then re-added within the window (e.g. RESTORE) has its - // remove cancelled by the re-add there, so getActiveRemoveFiles would omit it -- yet its old - // leaf entry still needs masking (the re-added copy lives in the root). We must mask by every - // backref position deleted since the old AMT, which is exactly this scan. - val mdvBackrefRemoves = (windowActions ++ actionsToCommit) - .collect { case r: RemoveFile if r.backReference.isDefined => r } - // 3.b: CDF -- ONLY this commit's removes. A window remove already emitted its CDF (deleted DV / - // root tombstone) in its own commit; re-emitting here would double-count. So this commit's - // with-backref removes drive tracking.deleted_positions on the touched leaf pointers, and its - // without-backref removes drive the root tombstones. - val commitRemoves = actionsToCommit.collect { case r: RemoveFile => r } - val (cdfBackrefRemoves, cdfNoBackrefRemoves) = - commitRemoves.partition(_.backReference.isDefined) - - // ---- Step 4: carry the old leaf pointers forward, patching MDV + deleted_positions. ---- - // Also returns the MDV positions this write added per leaf (keyed by relative location), reused - // for the shape metrics below rather than recomputed. - val (carriedLeafPointers, mdvPositionsByLeaf) = - carryForwardLeaves(oldAMTCheckpointProvider, mdvBackrefRemoves, cdfBackrefRemoves) - - // ---- Step 5: build the root DATA entries. ---- - // 5.a: live files (added/existing) -- the net-new files, held directly in the root. - val liveAdds = replay.allFiles - // 5.b: removed tombstones (for CDF) -- net-removed no-backref files removed by THIS commit, - // built from the matching AddFile (root + window) so CDF gets full stats, never from the sparse - // RemoveFile. - val rootAndWindowAdds = fileActionsFromOldRoot ++ windowActions.collect { case a: AddFile => a } - val tombstoneEntries = buildTombstones(cdfNoBackrefRemoves, liveAdds, rootAndWindowAdds) - - // ---- Step 6: spill live adds into new leaves if the root would exceed the per-leaf cap. ---- - val fixedRootCount = carriedLeafPointers.size + tombstoneEntries.size - val (rootAdds, spilledLeafPointers) = - spillIfNeeded(liveAdds, fixedRootCount) - - // ---- Step 7: write the new root. ---- - val addedTracking = AMTWriteHelper.addedTracking - val rootRows = - (carriedLeafPointers ++ spilledLeafPointers).map(_.wrap) ++ - rootAdds.map(add => DataEntry.fromAddFile(add, addedTracking, tableRoot).wrap) ++ - tombstoneEntries.map(_.wrap) - val contentRootBase = - AMTWriteHelper.writeRoot(spark, fs, hadoopConf, tableRoot, metadataDir, rootRows) + val liveEntries = + liveAddFiles.map(add => DataEntry.fromAddFile(add, trackingForLiveAdd(add), tableRoot)) + // 3.b: root remove entries (for CDF) -- this commit's net-removed no-backref files, built from + // the matching AddFile (old root + window) so CDF gets full stats. + val removeEntries = buildRootRemoveEntries( + processedActions.cdfNoBackrefRemoves, liveAddFiles, processedActions.rootAndWindowAdds, + processedActions.commitAddedPaths) - // ---- Step 8: generate the Checkpoint action. ---- // The version the tree describes: an inline commit describes itself; a deferred OPTIMIZE // CHECKPOINT (no user actions) describes the last committed version (attemptVersion - 1). - val contentStateVersion = + val contentTreeVersion = if (actionsToCommit.isEmpty) attemptVersion - 1 else attemptVersion + val firstRowIdForNewLeaves = AMTWriteHelper.firstRowIdAfter( + processedActions.domainMetadatas.collectFirst { + case RowTrackingMetadataDomain(domain) => domain.rowIdHighWaterMark + }) + + // ---- Step 4: spill entries into new leaves if the root would exceed the per-leaf cap. ---- + val fixedRootCount = carriedLeafPointers.size + val (rootLiveEntries, rootRemoveEntries, spilledLeafPointers) = + spillIfNeeded(liveEntries, removeEntries, fixedRootCount, + processedActions.postCommitMetadata, processedActions.postCommitProtocol, + contentTreeVersion, firstRowIdForNewLeaves) + val allLeafPointers = carriedLeafPointers ++ spilledLeafPointers + + // ---- Step 5: write the new root. The post-commit metadata and protocol shape the persisted + // manifest schema (the Iceberg partition struct and the typed per-column content stats), so + // every manifest write needs them. + val rootRows = + allLeafPointers.map(_.wrap) ++ + rootLiveEntries.map(_.wrap) ++ + rootRemoveEntries.map(_.wrap) + val contentRootBase = AMTWriteHelper.writeRoot( + spark, fs, hadoopConf, tableRoot, metadataDir, processedActions.postCommitMetadata, + processedActions.postCommitProtocol, rootRows, + version = contentTreeVersion) + + // ---- Step 6: generate the Checkpoint action. ---- // An incremental rewrite carries forward the previous tree's last-full-rewrite marker. - val lastFullRewriteVersion = oldCheckpoint.contentRoot - .lastManifestCommitWithFullRewrite.getOrElse(contentStateVersion) + val lastFullRewriteVersion = + oldAMT.lastManifestCommitWithFullRewrite.getOrElse(contentTreeVersion) val contentRoot = ContentRoot( path = contentRootBase.path, sizeInBytes = contentRootBase.sizeInBytes, + version = contentTreeVersion, isIncremental = true, - lastManifestCommitWithFullRewrite = lastFullRewriteVersion) - val postCommitProtocol = replay.getProtocol.getOrElse( - throw new IllegalStateException("Replay produced no protocol for the incremental AMT.")) - val postCommitMetadata = replay.getMetadata.getOrElse( - throw new IllegalStateException("Replay produced no metadata for the incremental AMT.")) + lastManifestCommitWithFullRewrite = lastFullRewriteVersion, + numLeaves = allLeafPointers.size.toLong) val checkpoint = Checkpoint( - version = contentStateVersion, + version = contentTreeVersion, contentRoot = contentRoot, - protocol = postCommitProtocol, - metaData = postCommitMetadata, - domainMetadata = replay.getDomainMetadatas.toSeq, - txns = replay.getTransactions.toSeq, + protocol = processedActions.postCommitProtocol, + metaData = processedActions.postCommitMetadata, + domainMetadata = processedActions.domainMetadatas, + txns = processedActions.transactions, sidecars = Seq.empty) - val allLeafPointers = carriedLeafPointers ++ spilledLeafPointers - val result = AMTWriteResult( - contentRootVersion = contentStateVersion, - checkpoint = checkpoint, - leaves = allLeafPointers, - includeActionsInCommitJson = true) - val numExistingLeavesUpdated = carriedLeafPointers.count(p => - mdvPositionsByLeaf.getOrElse(p.location, Seq.empty).nonEmpty) + val numOldLeavesUpdated = carriedLeafPointers.count(p => + leafPositions.newMDVPositionsByLeaf.getOrElse(p.location, Set.empty[Int]).nonEmpty) + // Per-status breakdown over every leaf pointer in the new tree (carried + newly spilled). + val leavesByStatus = allLeafPointers.groupBy(_.tracking.status).map { + case (status, ps) => status -> ps.size + } + val numStaleDeletedLeavesDropped = + oldAMT.allLeafs.count(_.tracking.status == Tracking.Status.Deleted) + def positionCount(byLeaf: Map[String, Set[Int]]): Int = byLeaf.valuesIterator.map(_.size).sum + // Per-status breakdown over the new tree's root-resident DATA entries (live + remove entries). + val rootEntriesByStatus = (rootLiveEntries ++ rootRemoveEntries) + .groupBy(_.tracking.status).map { case (status, es) => status -> es.size } val incrementalWriteMetrics = IncrementalAMTWriteMetrics( numIntermediateCommits = intermediateLogCommits.size, - numExistingLeavesUpdated = numExistingLeavesUpdated, - numExistingLeavesUntouched = carriedLeafPointers.size - numExistingLeavesUpdated, + numOldLeavesUpdated = numOldLeavesUpdated, + numOldLeavesUntouched = carriedLeafPointers.size - numOldLeavesUpdated, numNewLeaves = spilledLeafPointers.size, - numRootLiveAdds = rootAdds.size, - numRootTombstones = tombstoneEntries.size, - numLeafMdvBitsAdded = mdvPositionsByLeaf.valuesIterator.map(_.size).sum) + numRootEntriesAddedStatus = rootEntriesByStatus.getOrElse(Tracking.Status.Added, 0), + numRootEntriesExistingStatus = rootEntriesByStatus.getOrElse(Tracking.Status.Existing, 0), + numRootEntriesModifiedStatus = rootEntriesByStatus.getOrElse(Tracking.Status.Modified, 0), + numRootEntriesReplacedStatus = rootEntriesByStatus.getOrElse(Tracking.Status.Replaced, 0), + numRootEntriesDeletedStatus = rootEntriesByStatus.getOrElse(Tracking.Status.Deleted, 0), + numLeafMdvBitsAdded = positionCount(leafPositions.newMDVPositionsByLeaf), + numLeafDeleteCDFBitsAdded = positionCount(leafPositions.deleteCDFPositionByLeaf), + numLeafReplaceCDFBitsAdded = positionCount(leafPositions.replaceCDFPositionByLeaf), + numLeavesAddedStatus = leavesByStatus.getOrElse(Tracking.Status.Added, 0), + numLeavesExistingStatus = leavesByStatus.getOrElse(Tracking.Status.Existing, 0), + numLeavesModifiedStatus = leavesByStatus.getOrElse(Tracking.Status.Modified, 0), + numLeavesDeletedStatus = leavesByStatus.getOrElse(Tracking.Status.Deleted, 0), + numStaleDeletedLeavesDropped = numStaleDeletedLeavesDropped) val metric = SingleAMTWriteMetrics( trigger = trigger, // This writer only ever produces an incremental tree. incremental = "true", materializeDurationMs = NANOSECONDS.toMillis(System.nanoTime() - startNanos), incrementalWriteMetrics = Some(incrementalWriteMetrics)) - (result, metric) + AMTWriteResult( + contentRootVersion = contentTreeVersion, + checkpoint = checkpoint, + leaves = allLeafPointers, + includeActionsInCommitJson = true, + amtWriteMetrics = metric) } /** - * Carries the previous tree's leaf pointers forward. Two independent, differently-scoped updates - * are applied to each carried pointer: + * Carries the previous tree's leaf pointers forward. Three differently-scoped decisions apply + * per pointer: + * - a pointer the previous AMT already marked DELETED is dropped, not carried forward: it was + * kept only to emit that commit's CDF, and re-emitting it here would produce wrong CDF; * - `manifest_info.dv` (MDV, masking) accumulates ALL leaf positions removed since the old AMT * (`mdvRemoves` = window + commit), so the reader hides every entry deleted since the last * full tree; and - * - `tracking.deleted_positions` (CDF) is RESET to just THIS commit's removed positions - * (`cdfRemoves`) -- a window remove already emitted its CDF in its own commit, and the old - * pointer's stale positions must not carry forward, so this is cleared on every pointer and - * re-set only for leaves this commit touched. + * - `tracking.{deleted_positions / replaced_positions}` (CDF) is RESET to just THIS commit's + * removed positions -- a window remove already emitted its CDF in its own commit, and the + * old pointer's stale positions must not carry forward, so this is cleared on every pointer + * and re-set only for leaves this commit touched. * Leaf parquet files are never re-read or rewritten. * - * Returns the carried-forward pointers and the MDV positions this write added per leaf (keyed by - * relative location), so the caller can derive the shape metrics without recomputing it. + * Returns the carried-forward pointers and an [[MDVAndCDFPositions]] that has been set per-leaf. */ private def carryForwardLeaves( - provider: AMTCheckpointProvider, - mdvRemoves: Seq[RemoveFile], - cdfRemoves: Seq[RemoveFile]): (Seq[DataManifestEntry], Map[String, Seq[Long]]) = { - def positionsByLeaf(removes: Seq[RemoveFile]): Map[String, Seq[Long]] = - removes.flatMap(r => r.backReference.map(br => br.manifest -> br.pos)) - .groupBy(_._1).map { case (leaf, pairs) => leaf -> pairs.map(_._2) } - val mdvPositionsByLeaf = positionsByLeaf(mdvRemoves) - val cdfPositionsByLeaf = positionsByLeaf(cdfRemoves) - val pointers = provider.leaves - .map { pointer => - val leafKey = pointer.location - val newMdvPositions = mdvPositionsByLeaf.getOrElse(leafKey, Seq.empty) - val cdfPositions = cdfPositionsByLeaf.getOrElse(leafKey, Seq.empty) - // Cumulative MDV = old dv + every position removed from this leaf since the old AMT. - val manifestInfo = - if (newMdvPositions.isEmpty) { - pointer.manifest_info - } else { - val cumulativeDV = pointer.manifest_info.dv - .map(AMTUtils.deserializeMdv).getOrElse(new RoaringBitmapArray) - newMdvPositions.foreach(cumulativeDV.add) - AMTWriteHelper.withUpdatedMdv(pointer.manifest_info, cumulativeDV) - } - // A carried leaf pointer stays live (ADDED); only deleted_positions conveys CDF. It is - // per-commit: reset to THIS commit's removed positions (empty when this commit did not - // delete from this leaf), never the old pointer's stale value. + oldLeaves: Seq[DataManifestEntry], + mdvSupersededBackrefs: Seq[BackReference], + cdfDeletedBackrefs: Seq[BackReference], + cdfReplacedBackrefs: Seq[BackReference]) + : (Seq[DataManifestEntry], MDVAndCDFPositions) = { + // A (leaf, position) can be superseded multiple times, e.g. a leaf file removed, re-added and + // removed again. We use a Set to dedupe the positions so the count matches the number of bits + // gained by the MDV or CDF bitmap. + def positionsByLeaf(backrefs: Seq[BackReference]): Map[String, Set[Int]] = + backrefs.map(br => br.manifest -> br.pos) + .groupBy(_._1).map { case (leaf, pairs) => leaf -> pairs.map(_._2).toSet } + val newMDVPositionsByLeaf = positionsByLeaf(mdvSupersededBackrefs) + val deletedPositionsByLeaf = positionsByLeaf(cdfDeletedBackrefs) + val replacedPositionsByLeaf = positionsByLeaf(cdfReplacedBackrefs) + val pointers = oldLeaves.flatMap { pointer => + if (pointer.tracking.status == Tracking.Status.Deleted) None + else { + val newMdvPositions = + newMDVPositionsByLeaf.getOrElse(pointer.location, Set.empty[Int]).toSeq val deletedPositions = - if (cdfPositions.isEmpty) None - else Some(AMTUtils.serializeMdv(RoaringBitmapArray(cdfPositions: _*))) - val tracking = AMTWriteHelper.addedTracking.copy(deleted_positions = deletedPositions) - pointer.copy(manifest_info = manifestInfo, tracking = tracking) + deletedPositionsByLeaf.getOrElse(pointer.location, Set.empty[Int]).toSeq + val replacedPositions = + replacedPositionsByLeaf.getOrElse(pointer.location, Set.empty[Int]).toSeq + Some(carryForwardOneLeaf(pointer, newMdvPositions, deletedPositions, replacedPositions)) } - (pointers, mdvPositionsByLeaf) + } + (pointers, + MDVAndCDFPositions(newMDVPositionsByLeaf, deletedPositionsByLeaf, replacedPositionsByLeaf)) + } + + // Visible for testing. + private[amt] def carryForwardOneLeaf( + pointer: DataManifestEntry, + newMdvPositions: Seq[Int], + deletedPositions: Seq[Int], + replacedPositions: Seq[Int]): DataManifestEntry = { + // The file counts in manifest_info describe the leaf when it was initially created. They are + // not updated when the leaf is carried forward; only its manifest deletion vector can grow. + val liveFileCount = pointer.manifest_info.liveFilesCount + val tombstoneFileCount = pointer.manifest_info.tombstoneFilesCount + AMTUtils.invariantCheckWithLogging( + checkInvariant = liveFileCount == 0 || tombstoneFileCount == 0, + opTypeSuffix = AMTUsageLogs.ALERT_MIXED_LEAF_CONTENT, + message = "ManifestInfo representing a leaf's initial state must not contain both live " + + "files and tombstones.", + deltaLog = deltaLog, + data = Map("liveFileCount" -> liveFileCount, "tombstoneFileCount" -> tombstoneFileCount)) + AMTUtils.invariantCheckWithLogging( + checkInvariant = newMdvPositions.isEmpty || liveFileCount > 0, + opTypeSuffix = AMTUsageLogs.ALERT_MDV_WITHOUT_LIVE_FILES, + message = s"Leaf ${pointer.location} holds no live file but gained new MDV positions.", + deltaLog = deltaLog, + data = Map( + "leafLocation" -> pointer.location, + "liveFileCount" -> liveFileCount, + "tombstoneFileCount" -> tombstoneFileCount, + "newMdvPositionCount" -> newMdvPositions.size)) + val (tracking, manifestInfo) = + if (tombstoneFileCount > 0) { + // A carried leaf with no live file -- a tombstone-only leaf born ADDED last commit -- + // decays to DELETED so the next AMT drops it. + AMTWriteHelper.deletedTrackingForCarriedLeaf(pointer) + } else { + // this case means tombstoneFileCount = 0 and liveFileCount >= 0 + // In this scenario, mark the leaf as DELETED if any of below is true: + // a) all live files are marked for deletion in this commit (with newMdvPositions) OR + // b) all the live files were already striked off in previous commit itself but the + // leaf was not marked as DELETED then (this is not done by our implementation but + // could be done by external engine). + if (newMdvPositions.nonEmpty) { + // The MDV grew this commit: MODIFIED, or DELETED once every live entry is masked. + AMTWriteHelper.modifiedOrDeletedTrackingForLeaf( + pointer, newMdvPositions, deletedPositions, replacedPositions) + } else if (pointer.manifest_info.dv_cardinality.getOrElse(0L) == pointer.record_count) { + AMTWriteHelper.deletedTrackingForCarriedLeaf(pointer) + } else { + // Untouched leaf (no new MDVs) that still holds live files: carry it forward EXISTING. + AMTWriteHelper.existingTrackingForLeaf(pointer) + } + } + pointer.copy(manifest_info = manifestInfo, tracking = tracking) } /** - * Builds root-resident `tracking=removed` tombstones for THIS commit's net-removed no-backref - * files, so Change Data Feed can recover them. (Window removes already emitted their CDF in their - * own commits, so they are not passed here.) Each tombstone is built from the removed file's - * `AddFile` (found among root + window adds, matched by the `(path, dvId)` key) -- NOT from - * the sparse `RemoveFile` -- so it carries full stats/DV. A remove is a tombstone only when its - * file is net-removed (not present in the replay's live set) and its `AddFile` is found. + * Builds root-resident entries for THIS commit's net-removed no-backref files, so Change Data + * Feed can recover them. (Window actions already emitted their CDF in their own commits, so they + * are not passed here.) + * - Each entry is built from the removed file's `AddFile` (found among root + window adds) -- + * NOT from the sparse `RemoveFile` -- so it carries full stats / DV. + * - A `RemoveFile` for a file ALSO re-added this commit under a new DV yields a REPLACED + * `DataEntry` (its prior state). + * - A `RemoveFile` with no matching re-add for the same path (no DV change -- the full + * `(path, dv)` pair is removed) yields a DELETED `DataEntry`. */ - private def buildTombstones( + private def buildRootRemoveEntries( withoutBackrefRemoves: Seq[RemoveFile], liveAdds: Seq[AddFile], - rootAndWindowAdds: Seq[AddFile]): Seq[DataEntry] = { + rootAndWindowAdds: Seq[AddFile], + commitAddedPaths: Set[String]): Seq[DataEntry] = { if (withoutBackrefRemoves.isEmpty) return Seq.empty - // Implicit `toUniqueFileActionTuple` on AddFile / RemoveFile. - import InMemoryLogReplay.{UniqueAddFileTuple, UniqueRemoveFileTuple} val liveKeys: Set[UniqueFileActionTuple] = - liveAdds.iterator.map(_.toUniqueFileActionTuple).toSet - // AddFile lookup from root + window (NOT this commit -- Delta disallows add+remove in one - // commit, so a removed file's add always predates the commit). + liveAdds.iterator + .map(_.toUniqueFileActionTuple(tableRoot, useDeletionVectorObjectIdentity)) + .toSet + // AddFile lookup from root + window: the removed file's add predates this commit (a replace + // re-adds the same path under a new DV -- a distinct key -- so this still finds the prior add). val addByKey: Map[UniqueFileActionTuple, AddFile] = - rootAndWindowAdds.map(a => a.toUniqueFileActionTuple -> a).toMap - withoutBackrefRemoves.flatMap { r => - val removeKey = r.toUniqueFileActionTuple - // Only tombstone a genuinely removed file whose originating add we can find. - if (liveKeys.contains(removeKey)) None - else addByKey.get(removeKey).map(add => - DataEntry.fromAddFile(add, AMTWriteHelper.removedTracking(), tableRoot)) + rootAndWindowAdds + .map(a => a.toUniqueFileActionTuple(tableRoot, useDeletionVectorObjectIdentity) -> a) + .toMap + withoutBackrefRemoves.map { r => + val removeKey = r.toUniqueFileActionTuple(tableRoot, useDeletionVectorObjectIdentity) + // Two invariants a no-backref remove must satisfy (it targets a root-resident or + // window-added file, never a leaf); violating either means a corrupt tree: + // - it is net-removed, so its (path, dv) is absent from the live set (a still-live key + // means a malformed add + remove of the same key in one commit); and + // - its originating add is present among root + window adds. + AMTUtils.invariantCheckWithLogging( + checkInvariant = !liveKeys.contains(removeKey), + opTypeSuffix = AMTUsageLogs.ALERT_NO_BACKREF_REMOVE_STILL_LIVE, + message = + s"Net-removed no-backref file ${r.path} (key $removeKey) is still in the live set; " + + "a file cannot be both removed and live in the same commit.", + deltaLog = deltaLog, + data = Map("filePath" -> r.path, "removeKey" -> removeKey.toString)) + AMTUtils.invariantCheckWithLogging( + checkInvariant = addByKey.contains(removeKey), + opTypeSuffix = AMTUsageLogs.ALERT_NO_BACKREF_REMOVE_MISSING_ADD, + message = s"No originating AddFile for net-removed no-backref file ${r.path} (key " + + s"$removeKey); cannot build its CDF root entry.", + deltaLog = deltaLog, + data = Map("filePath" -> r.path, "removeKey" -> removeKey.toString)) + val priorAddFile = addByKey(removeKey) + val tracking = + if (commitAddedPaths.contains(r.path)) AMTWriteHelper.replacedTrackingForDataEntry() + else AMTWriteHelper.removedTrackingForDataEntry() + DataEntry.fromAddFile(priorAddFile, tracking, tableRoot) } } /** - * If the root would exceed `entriesPerLeaf` (carried pointers + tombstones + live adds), moves - * whole `entriesPerLeaf`-sized batches of live adds into new leaves until the root's row count - * fits. Returns the live adds that remain root-resident and the pointers for any new leaves. + * If the root would exceed `entriesPerLeaf` (carried pointers + live entries + remove entries), + * moves whole `entriesPerLeaf`-sized batches into new leaves until the root's row count fits -- + * live entries first, then remove (DELETED / REPLACED) entries. Each spill adds one leaf pointer + * to the fixed root count. Returns the live entries and remove entries that remain root-resident, + * plus the pointers for any new leaves. */ private def spillIfNeeded( - liveAdds: Seq[AddFile], - fixedRootCount: Int): (Seq[AddFile], Seq[DataManifestEntry]) = { - var remaining = liveAdds + liveEntries: Seq[DataEntry], + removeEntries: Seq[DataEntry], + fixedRootCount: Int, + metadata: Metadata, + protocol: Protocol, + contentTreeVersion: Long, + firstRowIdForNewLeaves: Long) + : (Seq[DataEntry], Seq[DataEntry], Seq[DataManifestEntry]) = { val spilled = ArrayBuffer.empty[DataManifestEntry] - // Each spill adds one leaf pointer to the fixed root count; keep spilling while the root would - // still overflow. - while (fixedRootCount + spilled.size + remaining.size > entriesPerLeaf && remaining.nonEmpty) { - val (batch, rest) = remaining.splitAt(entriesPerLeaf) + var remainingLive = liveEntries + var remainingRemoves = removeEntries + def rootRowCount: Int = + fixedRootCount + spilled.size + remainingLive.size + remainingRemoves.size + while (rootRowCount > entriesPerLeaf && remainingLive.nonEmpty) { + val (batch, rest) = remainingLive.splitAt(entriesPerLeaf) + spilled += AMTWriteHelper.writeLeaf( + spark, fs, hadoopConf, tableRoot, metadataDir, metadata, protocol, batch, + contentTreeVersion, firstRowIdForNewLeaves) + remainingLive = rest + } + while (rootRowCount > entriesPerLeaf && remainingRemoves.nonEmpty) { + val (batch, rest) = remainingRemoves.splitAt(entriesPerLeaf) spilled += AMTWriteHelper.writeLeaf( - spark, fs, hadoopConf, tableRoot, metadataDir, batch) - remaining = rest + spark, fs, hadoopConf, tableRoot, metadataDir, metadata, protocol, batch, + contentTreeVersion, firstRowIdForNewLeaves) + remainingRemoves = rest } - (remaining, spilled.toSeq) + (remainingLive, remainingRemoves, spilled.toSeq) } // Asserts the intermediate commit files cover EXACTLY [oldAMTVersion+1, attemptVersion) -- no @@ -347,3 +486,165 @@ class IncrementalAMTWriter(spark: SparkSession, deltaLog: DeltaLog) { s"unexpected ${(covered -- expected).toList.sorted}.") } } + +private class ProcessedActions( + oldAMTVersion: Long, + oldRootAdds: Seq[AddFile], + nonContentFromOldCheckpoint: Seq[Action], + windowCommits: Seq[SingleCommit], + windowCommitActions: Seq[Seq[Action]], + attemptVersion: Long, + actionsToCommit: Seq[Action], + tableRoot: Path, + useDeletionVectorObjectIdentity: Boolean) { + private val replay = new InMemoryLogReplay( + minFileRetentionTimestamp = None, + minSetTransactionRetentionTimestamp = None, + tableRoot = tableRoot, + useDeletionVectorObjectIdentity = useDeletionVectorObjectIdentity) + private val rootAndWindowAddsBuf = ArrayBuffer.empty[AddFile] + private val mdvSupersededBuf = ArrayBuffer.empty[BackReference] + // This commit's own file actions, split out for CDF and re-add classification. + private val commitAddsBuf = ArrayBuffer.empty[AddFile] + private val commitRemovesBuf = ArrayBuffer.empty[RemoveFile] + + // Log replay of part-1/2/3. + replay.append(oldAMTVersion, oldRootAdds.iterator ++ nonContentFromOldCheckpoint.iterator) + windowCommits.zip(windowCommitActions).foreach { case (commit, actions) => + replay.append(commit.version, actions.iterator) + } + // Keys of files live in the {old root + window}'s replay BEFORE this commit's actions; a + // live add already here (or carrying a leaf back reference) is EXISTING, not ADDED. + val preCommitLiveKeys: Set[UniqueFileActionTuple] = + replay.allFiles.iterator + .map(_.toUniqueFileActionTuple(tableRoot, useDeletionVectorObjectIdentity)) + .toSet + replay.append(attemptVersion, actionsToCommit.iterator) + + // One more pass over part-1/2/3 to initialize the auxiliary buffers above. + rootAndWindowAddsBuf ++= oldRootAdds + windowCommits.zip(windowCommitActions).foreach { case (commit, actions) => + actions.foreach { + case a: AddFile => + rootAndWindowAddsBuf += a + a.backReference.foreach(mdvSupersededBuf += _) + case r: RemoveFile => r.backReference.foreach(mdvSupersededBuf += _) + case _ => + } + } + actionsToCommit.foreach { + case a: AddFile => + commitAddsBuf += a + a.backReference.foreach(mdvSupersededBuf += _) + case r: RemoveFile => + commitRemovesBuf += r + r.backReference.foreach(mdvSupersededBuf += _) + case _ => + } + + /** The net-live files held directly in the new root. */ + val liveAddFiles: Seq[AddFile] = replay.allFiles + /** Post-commit non-content metadata. */ + val postCommitProtocol: Protocol = replay.getProtocol.getOrElse( + throw new IllegalStateException("Replay produced no protocol for the incremental AMT.")) + val postCommitMetadata: Metadata = replay.getMetadata.getOrElse( + throw new IllegalStateException("Replay produced no metadata for the incremental AMT.")) + val domainMetadatas: Seq[DomainMetadata] = replay.getDomainMetadatas.toSeq + val transactions: Seq[SetTransaction] = replay.getTransactions.toSeq + + /** + * AddFiles resident in the old root + window: used to generate the tombstone DataEntries for the + * RemoveFile actions in actionsToCommit (Part-3). The prior AddFile entries carry richer + * information than the RemoveFile, so use them to construct the tombstone. + */ + val rootAndWindowAdds: Seq[AddFile] = rootAndWindowAddsBuf.toSeq + + /** + * Leaf positions superseded since the old AMT (window + commit), by a RemoveFile (deleted) or a + * back-referenced AddFile (re-added -- the fresh copy lands in the root, so its original leaf + * slot is stale). Masking both keeps a stale slot from surfacing. + */ + val mdvSupersededBackrefs: Seq[BackReference] = mdvSupersededBuf.toSeq + + /** Paths this commit adds; a removed path also present here is a same-commit replace. */ + val commitAddedPaths: Set[String] = commitAddsBuf.iterator.map(_.path).toSet + val replacedPaths: Set[String] = + commitRemovesBuf.iterator.map(_.path).toSet.intersect(commitAddedPaths) + + /** + * The commit's dataChange flag: the first FileAction's (Delta enforces one value per commit), + * defaulting to true when the commit carries no file actions. + */ + val dataChange: Boolean = + actionsToCommit.collectFirst { case f: FileAction => f.dataChange }.getOrElse(true) + + // CDF is driven ONLY by this commit's removes; a backref remove is REPLACED when its path is + // re-added here (under a new DV), else DELETED; a no-backref remove drives a root entry likewise. + private def reAddedHere(r: RemoveFile): Boolean = commitAddedPaths.contains(r.path) + val cdfReplacedBackrefs: Seq[BackReference] = + commitRemovesBuf.iterator.filter(r => r.backReference.isDefined && reAddedHere(r)) + .map(_.backReference.get).toSeq + val cdfDeletedBackrefs: Seq[BackReference] = + commitRemovesBuf.iterator.filter(r => r.backReference.isDefined && !reAddedHere(r)) + .map(_.backReference.get).toSeq + val cdfNoBackrefRemoves: Seq[RemoveFile] = + commitRemovesBuf.iterator.filter(_.backReference.isEmpty).toSeq + + // ---- Commit-shape invariants, checked once at construction. ---- + // True if `add` re-commits a file already live before this commit (its key is in the pre-commit + // live set, or it carries a back reference to an existing leaf slot) and is not a replace (a + // replace changes the DV, a distinct key). Such a re-add is only valid with dataChange=false. + private def isReCommittedLiveAdd(add: AddFile): Boolean = { + val preCommitLiveKeysContainAddFile = preCommitLiveKeys.contains( + add.toUniqueFileActionTuple(tableRoot, useDeletionVectorObjectIdentity)) + !replacedPaths.contains(add.path) && + (add.backReference.isDefined || preCommitLiveKeysContainAddFile) + } + val reCommittedLiveAdd: Option[AddFile] = commitAddsBuf.find(isReCommittedLiveAdd) + if (dataChange) { + // (1) A data-changing commit must not re-add a file already live before it: a same-key re-add + // must carry dataChange=false. + reCommittedLiveAdd.foreach { add => + AMTUtils.invariantCheckWithLogging( + checkInvariant = false, + opTypeSuffix = AMTUsageLogs.ALERT_DATA_CHANGE_READD, + message = s"Re-adding already-live file ${add.path} with dataChange=true is not allowed; " + + "a same-key re-add must use dataChange=false.", + data = Map("filePath" -> add.path, "dataChange" -> dataChange)) + } + } else if (commitRemovesBuf.nonEmpty) { + // (2) A dataChange=false commit that also removes files is a compaction (OPTIMIZE/REORG): + // Its adds are freshly written files, so none may re-add a file already live before it. + reCommittedLiveAdd.foreach { add => + AMTUtils.invariantCheckWithLogging( + checkInvariant = false, + opTypeSuffix = AMTUsageLogs.ALERT_COMPACTION_READD, + message = "A dataChange=false commit with removes must not re-add an already-live file, " + + s"but ${add.path} is re-added.", + data = Map( + "filePath" -> add.path, + "dataChange" -> dataChange, + "removeCount" -> commitRemovesBuf.size)) + } + } else { + // (3) A dataChange=false commit with no removes is a metadata refresh (e.g. stats/tags + // update): + // every add must re-commit a file already live before it, under the same (path, dv) key. + commitAddsBuf.find(a => !isReCommittedLiveAdd(a)).foreach { add => + AMTUtils.invariantCheckWithLogging( + checkInvariant = false, + opTypeSuffix = AMTUsageLogs.ALERT_METADATA_REFRESH_DATA_CHANGE_FALSE_ADDS_NEW_FILE, + message = "A dataChange=false commit with no removes must re-add only already-live " + + s"files, but ${add.path} is a new file.", + data = Map( + "filePath" -> add.path, + "dataChange" -> dataChange, + "removeCount" -> commitRemovesBuf.size)) + } + } +} + +private case class MDVAndCDFPositions( + newMDVPositionsByLeaf: Map[String, Set[Int]], + deleteCDFPositionByLeaf: Map[String, Set[Int]], + replaceCDFPositionByLeaf: Map[String, Set[Int]]) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/amt/actions.scala b/spark/src/main/scala/org/apache/spark/sql/delta/amt/actions.scala index 0792d054a8c..ad7cb40c325 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/amt/actions.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/amt/actions.scala @@ -16,17 +16,17 @@ package org.apache.spark.sql.delta.amt -import java.util.UUID - -import scala.util.Try - import org.apache.spark.sql.delta.DeltaColumnMapping -import org.apache.spark.sql.delta.actions.{AddFile, DeletionVectorDescriptor} -import org.apache.spark.sql.delta.stats.DeltaStatistics +import org.apache.spark.sql.delta.actions.{Action, AddFile, DeletionVectorDescriptor, Metadata, Protocol} import org.apache.spark.sql.delta.storage.dv.DeletionVectorStore import com.fasterxml.jackson.annotation.JsonIgnore +import com.fasterxml.jackson.core.JsonParser +import com.fasterxml.jackson.databind.{DeserializationContext, JsonDeserializer} +import com.fasterxml.jackson.databind.annotation.JsonDeserialize import org.apache.hadoop.fs.{FileStatus, Path} +import org.apache.spark.paths.SparkPath +import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.execution.datasources.parquet.ParquetUtils import org.apache.spark.sql.types.{MetadataBuilder, StructField, StructType} @@ -52,6 +52,9 @@ import org.apache.spark.sql.types.{MetadataBuilder, StructField, StructType} * @param split_offsets Row-group split offsets (Iceberg field 132); Delta does not generate or * consume these, but the field is kept in the schema to carry it forward * for tables written by both Delta and Iceberg. + * @param tags Delta [[AddFile]] tags. They have no Iceberg V4 slot, so the field is a Delta-private + * extension carried on DATA entries only, so a file's tags survive a manifest round + * trip. */ case class AMTSingleAction( content_type: Int, // ID: 134, required. @@ -59,16 +62,17 @@ case class AMTSingleAction( location: String, // ID: 100, required. file_format: String, // ID: 101, required ("parquet" for now). tracking: Tracking, // ID: 147, required. - deletion_vector: Option[DeletionVector], // ID: 148, optional (only when content_type=0). + deletion_vector: Option[DeletionVector], // ID: 148, optional (only when content_type=0). spec_id: Option[Int], // ID: 141, optional. - partition: Partition, // ID: 102, required. + partition: Option[Map[String, String]], // ID: 102, optional. sort_order_id: Option[Int], // ID: 140, optional (only when content_type=0). record_count: Long, // ID: 103, required. file_size_in_bytes: Long, // ID: 104, required. - content_stats: Option[ContentStats], // ID: 146, optional. + content_stats: Option[String], // ID: 146, optional. manifest_info: Option[ManifestInfo], // ID: 150, required for DATA_MANIFEST. key_metadata: Option[Array[Byte]], // ID: 131, optional. - split_offsets: Option[Seq[Long]] // ID: 132, optional. + split_offsets: Option[Seq[Long]], // ID: 132, optional. + tags: Option[Map[String, String]] // Delta-private id (only when content_type=0). ) { AMTSingleAction.validate(this) @@ -92,6 +96,7 @@ case class AMTSingleAction( content_stats = content_stats, key_metadata = key_metadata, split_offsets = split_offsets, + tags = tags, format_version = format_version) case AMTSingleAction.ContentType.Type.DataManifest => DataManifestEntry( @@ -165,6 +170,9 @@ object AMTSingleAction { require(action.content_type == ContentType.Type.Data || action.sort_order_id.isEmpty, s"sort_order_id must be null when content_type != ${ContentType.Type.Data}; " + s"got content_type=${action.content_type}.") + require(action.content_type == ContentType.Type.Data || action.tags.isEmpty, + s"tags must be null when content_type != ${ContentType.Type.Data}; " + + s"got content_type=${action.content_type}.") if (ContentType.isRootOnly(action.content_type)) { (action.tracking.sequence_number, action.tracking.file_sequence_number) match { case (Some(a), Some(b)) => require(a == b, @@ -193,6 +201,20 @@ object AMTSingleAction { private def optional(id: Long, name: String): AMTFieldSpec = AMTFieldSpec(name, id, required = false) + /** + * Field ids for the Delta-private `tags` map. Delta's [[AddFile]] tags have no Iceberg V4 slot, + * so they are stamped from the top of the id space, just below Iceberg's reserved band. The V4 + * spec forbids field ids greater than `Int.MaxValue - 200` (2147483447), reserving that range for + * metadata columns (`_file`, `_row_id`, ...); anchoring `tags` at `Int.MaxValue - 300` keeps it a + * legal, non-reserved id that does not collide with any other id this schema assigns in practice. + * The one theoretical exception is `content_stats` (ids grow as 10000 + 200 * columnId): a table + * would need more than ~10.7M stats-collected columns for those ids to reach this band, which is + * not realistic. + */ + private[amt] val TagsFieldId: Long = Int.MaxValue.toLong - 300L + private val TAGS_KEY_FIELD_ID: Long = TagsFieldId + 1L + private val TAGS_VALUE_FIELD_ID: Long = TagsFieldId + 2L + /** Iceberg V4 field specs for the top-level [[AMTSingleAction]] fields. */ private val topLevelFields: Seq[AMTFieldSpec] = Seq( required(134L, "content_type"), @@ -209,7 +231,8 @@ object AMTSingleAction { optional(146L, "content_stats"), optional(150L, "manifest_info"), optional(131L, "key_metadata"), - optional(132L, "split_offsets")) + optional(132L, "split_offsets"), + optional(TagsFieldId, "tags")) /** Iceberg V4 field specs for the scalar fields of the nested [[Tracking]] struct. */ private val trackingFields: Seq[AMTFieldSpec] = Seq( @@ -235,10 +258,12 @@ object AMTSingleAction { required(505L, "existing_files_count"), required(506L, "deleted_files_count"), required(520L, "replaced_files_count"), + required(524L, "modified_files_count"), required(512L, "added_rows_count"), required(513L, "existing_rows_count"), required(514L, "deleted_rows_count"), required(521L, "replaced_rows_count"), + required(525L, "modified_rows_count"), required(516L, "min_sequence_number"), optional(522L, "dv"), optional(523L, "dv_cardinality")) @@ -251,8 +276,6 @@ object AMTSingleAction { /** Iceberg V4 field id for the `split_offsets` list element. */ private val SPLIT_OFFSETS_ELEMENT_FIELD_ID: Long = 133L - private val PARTITION_VALUES_KEY_PLACEHOLDER_ID: Long = 100002L - private val PARTITION_VALUES_VALUE_PLACEHOLDER_ID: Long = 100003L /** Top-level specs by name, for lookup during stamping. */ private val topLevelFieldByName: Map[String, AMTFieldSpec] = @@ -260,36 +283,33 @@ object AMTSingleAction { /** * Test-only Parquet field-id map for an AMT checkpoint schema, keyed by dotted path. Keys are the - * top-level field name (`content_type`), a nested-struct scalar (`tracking.status`), a list - * element (`split_offsets.element`), or a map key/value (`partition.values.key`, - * `partition.values.value`). + * top-level field name (`content_type`), a nested-struct scalar (`tracking.status`), or a list + * element (`split_offsets.element`). */ private[amt] val allFieldIdByName: Map[String, Int] = { - val topLevel = topLevelFields.map(f => f.name -> f.id.toInt) + val topLevel = topLevelFields.filterNot(_.name == "partition").map(f => f.name -> f.id.toInt) val nested = nestedStructFields.flatMap { case (parent, children) => children.map(f => s"$parent.${f.name}" -> f.id.toInt) } val nestedContainers = Map( "split_offsets.element" -> SPLIT_OFFSETS_ELEMENT_FIELD_ID.toInt, - "partition.values.key" -> PARTITION_VALUES_KEY_PLACEHOLDER_ID.toInt, - "partition.values.value" -> PARTITION_VALUES_VALUE_PLACEHOLDER_ID.toInt) + "tags.key" -> TAGS_KEY_FIELD_ID.toInt, + "tags.value" -> TAGS_VALUE_FIELD_ID.toInt) (topLevel ++ nested ++ nestedContainers).toMap } - /** - * The persisted [[AMTSingleAction]] schema with Iceberg field ids stamped onto its Parquet - * schema. Everything here is static, so it is computed once. - */ - lazy val schemaWithFieldId: StructType = { + private lazy val staticStampedSchema: StructType = { import org.apache.spark.sql.delta.implicits._ - val base = amtSingleActionEncoder.schema - StructType(base.map(stampFieldSpec)) + StructType(amtSingleActionEncoder.schema.map(stampFieldSpec)) } /** * Applies the Iceberg [[AMTFieldSpec]] to a single top-level [[AMTSingleAction]] field: * stamps the field id (and any nested element/map ids) and sets nullability from * `spec.required`. + * + * `partition` keeps the encoder's data type here; [[persistedSchema]] substitutes the table's + * typed partition struct, or drops the field, once the partition schema is known. */ private def stampFieldSpec(field: StructField): StructField = { val spec = topLevelFieldByName.getOrElse(field.name, @@ -298,27 +318,26 @@ object AMTSingleAction { val builder = new MetadataBuilder() .withMetadata(field.metadata) .putLong(ParquetUtils.FIELD_ID_METADATA_KEY, spec.id) - // Attach nested (element) ids for list-bearing fields. - // DeltaParquetWriteSupport reads these off the list field itself, keyed `.element`. - listElementFieldId(field.name).foreach { elementId => + // Attach nested (list-element / map key-value) ids for container-typed fields. + // DeltaParquetWriteSupport reads these off the field itself, keyed by the relative path + // (`.element` for a list, `.key` / `.value` for a map). + val nestedIds = nestedFieldIds(field.name) + if (nestedIds.nonEmpty) { + val nestedBuilder = new MetadataBuilder() + nestedIds.foreach { case (relativePath, id) => nestedBuilder.putLong(relativePath, id) } builder.putMetadata( - DeltaColumnMapping.PARQUET_FIELD_NESTED_IDS_METADATA_KEY, - new MetadataBuilder() - .putLong( - Seq(field.name, DeltaColumnMapping.PARQUET_LIST_ELEMENT_FIELD_NAME).mkString("."), - elementId) - .build()) + DeltaColumnMapping.PARQUET_FIELD_NESTED_IDS_METADATA_KEY, nestedBuilder.build()) } - // Rewrite the field's data type: apply nested-struct field specs (id + nullability). + // Rewrite the field's data type by applying nested-struct field specs (id + nullability). val stampedDataType = field.dataType match { case struct: StructType => nestedStructFields.get(field.name) match { case Some(specs) => stampNestedStructFieldSpecs(field.name, struct, specs) case None => - assert(field.name == "partition" || field.name == "content_stats", + assert(field.name == "content_stats", s"Unexpected struct-typed AMTSingleAction field '${field.name}': expected a " + - "nested-id struct, partition, or content_stats.") - if (field.name == "partition") stampPartitionValuesNestedIds(struct) else struct + "nested-id struct or content_stats.") + struct } case other => assert(!nestedStructFields.contains(field.name), @@ -334,10 +353,19 @@ object AMTSingleAction { metadata = builder.build()) } - /** List-element field id for a top-level list field, or None if not a list field. */ - private def listElementFieldId(fieldName: String): Option[Long] = fieldName match { - case "split_offsets" => Some(SPLIT_OFFSETS_ELEMENT_FIELD_ID) - case _ => None + /** + * Nested (list-element / map key-value) field ids for a top-level container field. + */ + private def nestedFieldIds(fieldName: String): Seq[(String, Long)] = fieldName match { + case "split_offsets" => + Seq( + s"$fieldName.${DeltaColumnMapping.PARQUET_LIST_ELEMENT_FIELD_NAME}" -> + SPLIT_OFFSETS_ELEMENT_FIELD_ID) + case "tags" => + Seq( + s"$fieldName.${DeltaColumnMapping.PARQUET_MAP_KEY_FIELD_NAME}" -> TAGS_KEY_FIELD_ID, + s"$fieldName.${DeltaColumnMapping.PARQUET_MAP_VALUE_FIELD_NAME}" -> TAGS_VALUE_FIELD_ID) + case _ => Seq.empty } /** @@ -363,25 +391,24 @@ object AMTSingleAction { }) } - /** Stamps placeholder map key/value ids onto the interim `partition.values` field. */ - private def stampPartitionValuesNestedIds(struct: StructType): StructType = - StructType(struct.map { field => - if (field.name == "values") { - val nestedIds = new MetadataBuilder() - .putLong( - Seq("values", DeltaColumnMapping.PARQUET_MAP_KEY_FIELD_NAME).mkString("."), - PARTITION_VALUES_KEY_PLACEHOLDER_ID) - .putLong( - Seq("values", DeltaColumnMapping.PARQUET_MAP_VALUE_FIELD_NAME).mkString("."), - PARTITION_VALUES_VALUE_PLACEHOLDER_ID) - .build() - field.copy(metadata = new MetadataBuilder() - .withMetadata(field.metadata) - .putMetadata(DeltaColumnMapping.PARQUET_FIELD_NESTED_IDS_METADATA_KEY, nestedIds) - .build()) - } else { - field - } + /** + * The [[AMTSingleAction]] schema as written to disk, which differs from the encoder's in-memory + * schema: every mapped field carries its Iceberg field id, required/optional is driven by the + * field spec rather than the encoder, and the two per-table fields take their table-dependent + * shape -- `partition` holds the typed partition struct (absent entirely for an unpartitioned + * table), and `content_stats` holds the typed per-column statistics struct. + */ + def persistedSchema(metadata: Metadata, protocol: Protocol): StructType = + StructType(staticStampedSchema.flatMap { + case field if field.name == "partition" => + if (metadata.partitionSchema.isEmpty) None + else Some(field.copy( + dataType = AMTPartitionValues.persistedSchema(metadata.partitionSchema))) + case field if field.name == "content_stats" => + val contentStatsSchema = AMTContentStats.persistedSchema(metadata, protocol) + if (contentStatsSchema.isEmpty) None + else Some(field.copy(dataType = contentStatsSchema)) + case field => Some(field) }) } @@ -418,6 +445,7 @@ sealed trait AMTAction { * @param split_offsets Row-group split offsets (Iceberg field 132); Delta does not generate or * consume these, but the field is kept in the schema to carry it forward * for tables written by both Delta and Iceberg. + * @param tags Delta [[AddFile]] tags, preserved verbatim across the manifest round trip. * @param format_version Iceberg writer format version; 4 for V4. */ case class DataEntry( @@ -426,13 +454,14 @@ case class DataEntry( tracking: Tracking, record_count: Long, file_size_in_bytes: Long, - partition: Partition = Partition(), + partition: Option[Map[String, String]] = None, deletion_vector: Option[DeletionVector] = None, spec_id: Option[Int] = None, sort_order_id: Option[Int] = None, - content_stats: Option[ContentStats] = None, + content_stats: Option[String] = None, key_metadata: Option[Array[Byte]] = None, split_offsets: Option[Seq[Long]] = None, + tags: Option[Map[String, String]] = None, format_version: Int = AMTSingleAction.FormatVersionV4) extends AMTAction { @@ -453,47 +482,179 @@ case class DataEntry( content_stats = content_stats, manifest_info = None, key_metadata = key_metadata, - split_offsets = split_offsets) + split_offsets = split_offsets, + tags = tags) def toAddFile(tableRoot: Path): AddFile = { val dv = deletion_vector.map(DeletionVector.toDescriptor(_, tableRoot)).orNull - // `record_count` (Iceberg field 103) and the Delta `numRecords` statistic are both the physical - // row count (total records in the file, including DV-deleted rows), so store it directly. - val stats = s"""{"${DeltaStatistics.NUM_RECORDS}":$record_count}""" + val stats = AMTContentStats.toStatsJson(content_stats, record_count) AddFile( - path = location, - partitionValues = partition.values.getOrElse(Map.empty), + // RFC-2396 encoding used by Delta. + path = SparkPath.fromPathString(location).urlEncoded, + partitionValues = partition.getOrElse(Map.empty), size = file_size_in_bytes, modificationTime = 0L, dataChange = false, stats = stats, deletionVector = dv, baseRowId = tracking.first_row_id, - defaultRowCommitVersion = tracking.sequence_number) + defaultRowCommitVersion = tracking.file_sequence_number, + tags = tags.orNull, + amtPassthrough = AMTPassthrough.fromDataEntry(this)) } } object DataEntry { /** Creates [[DataEntry]] from AddFile. */ - def fromAddFile(add: AddFile, tracking: Tracking, tableRoot: Path): DataEntry = + def fromAddFile(add: AddFile, tracking: Tracking, tableRoot: Path): DataEntry = { + val passthrough = add.amtPassthrough DataEntry( - location = add.path, + // RFC-2396 to Iceberg raw path conversion. + location = AMTUtils.relativizeLocation( + tableRoot.toString, SparkPath.fromUrlString(add.path).toPath.toString), file_format = AMTSingleAction.FileFormatParquet, // Round-trip the AddFile's row-tracking fields through the Iceberg tracking envelope so a // rowTracking-enabled table can reconstruct them on read. tracking = tracking.copy( first_row_id = add.baseRowId, - sequence_number = add.defaultRowCommitVersion), + sequence_number = add.defaultRowCommitVersion, + file_sequence_number = add.effectiveFileSequenceNumber), // Iceberg field 103 is the physical record count of the file, not the live/logical // count after deletes; throw rather than guess when the AddFile carries no stats. record_count = add.numPhysicalRecords.getOrElse( throw new IllegalArgumentException( s"Cannot build AMT entry: AddFile has no record count (missing stats): ${add.path}.")), file_size_in_bytes = add.size, - partition = Partition(Option(add.partitionValues).filter(_.nonEmpty)), + partition = Option(add.partitionValues).filter(_.nonEmpty), deletion_vector = Option(add.deletionVector).map(DeletionVector.fromDescriptor(_, tableRoot)), - content_stats = None) + spec_id = passthrough.flatMap(_.spec_id), + sort_order_id = passthrough.flatMap(_.sort_order_id), + key_metadata = passthrough.flatMap(_.key_metadata), + split_offsets = passthrough.flatMap(_.split_offsets), + tags = Option(add.tags).map(_.toMap).filter(_.nonEmpty), + content_stats = AMTContentStats.fromStatsJson(add.stats)) + } +} + +/** + * Deserializes split_offsets from commit-log JSON as a Seq[Long]. + */ +private[amt] class AMTSplitOffsetsDeserializer extends JsonDeserializer[Seq[Long]] { + override def deserialize(p: JsonParser, ctxt: DeserializationContext): Seq[Long] = + p.readValueAs(classOf[Array[Long]]).toSeq +} + +/** + * The AMT-native fields of a [[DataEntry]] that should be carried by [[AddFile]]. + */ +case class AMTPassthrough( + @JsonDeserialize(contentAs = classOf[java.lang.Integer]) + spec_id: Option[Int] = None, + @JsonDeserialize(contentAs = classOf[java.lang.Integer]) + sort_order_id: Option[Int] = None, + key_metadata: Option[Array[Byte]] = None, + @JsonDeserialize(contentUsing = classOf[AMTSplitOffsetsDeserializer]) + split_offsets: Option[Seq[Long]] = None) { + + // override equals and hashCode to compare by content for some fields. + override def equals(obj: Any): Boolean = obj match { + case that: AMTPassthrough => + spec_id == that.spec_id && + sort_order_id == that.sort_order_id && + split_offsets == that.split_offsets && + ((key_metadata, that.key_metadata) match { + case (Some(a), Some(b)) => java.util.Arrays.equals(a, b) + case (None, None) => true + case _ => false + }) + case _ => false + } + + override def hashCode(): Int = { + val keyMetadataHash = key_metadata.map(b => java.util.Arrays.hashCode(b)).getOrElse(0) + (spec_id, sort_order_id, keyMetadataHash, split_offsets).hashCode() + } +} + +object AMTPassthrough { + /** Name of the `amtPassthrough` field on the AddFile schema. */ + final val FIELD_NAME: String = "amtPassthrough" + + /** The struct type of the `amtPassthrough` field on the AddFile schema. */ + final lazy val STRUCT_TYPE: StructType = + Action.addFileSchema(FIELD_NAME).dataType.asInstanceOf[StructType] + + /** + * Positions of the `amtPassthrough` struct within an [[InternalRow]]. + */ + case class RowIndices( + structIndex: Int, + numFields: Int, + specId: Int, + sortOrderId: Int, + keyMetadata: Int, + splitOffsets: Int) + + object RowIndices { + /** + * Resolve the positions against `schema`, the schema of an `add`-shaped row, or `None` when + * `schema` does not project `amtPassthrough` at all. + */ + def resolve(schema: StructType): Option[RowIndices] = + schema.getFieldIndex(FIELD_NAME).map { structIndex => + val passthroughSchema = schema(FIELD_NAME).dataType.asInstanceOf[StructType] + RowIndices( + structIndex = structIndex, + numFields = passthroughSchema.fields.length, + specId = passthroughSchema.fieldIndex("spec_id"), + sortOrderId = passthroughSchema.fieldIndex("sort_order_id"), + keyMetadata = passthroughSchema.fieldIndex("key_metadata"), + splitOffsets = passthroughSchema.fieldIndex("split_offsets")) + } + } + + /** + * Read the [[AMTPassthrough]] out of `row` at the pre-resolved `indices`, or `None` when the row + * carries no AMT-native fields. + */ + def fromRow(row: InternalRow, indices: RowIndices): Option[AMTPassthrough] = { + if (row.isNullAt(indices.structIndex)) { + None + } else { + val struct = row.getStruct(indices.structIndex, indices.numFields) + Some(AMTPassthrough( + spec_id = + if (struct.isNullAt(indices.specId)) None else Some(struct.getInt(indices.specId)), + sort_order_id = + if (struct.isNullAt(indices.sortOrderId)) None + else Some(struct.getInt(indices.sortOrderId)), + key_metadata = + if (struct.isNullAt(indices.keyMetadata)) None + else Some(struct.getBinary(indices.keyMetadata)), + split_offsets = + if (struct.isNullAt(indices.splitOffsets)) None + else Some(struct.getArray(indices.splitOffsets).toLongArray().toSeq))) + } + } + + /** + * Build the [[AMTPassthrough]] carried on a Delta [[AddFile]] from `entry`, or `None` when the + * entry has no AMT-native fields to carry. + */ + def fromDataEntry(entry: DataEntry): Option[AMTPassthrough] = { + require( + entry.file_format == AMTSingleAction.FileFormatParquet && + entry.format_version == AMTSingleAction.FormatVersionV4, + s"amtPassthrough only supports parquet/v4. got " + + s"${entry.file_format}/${entry.format_version}.") + val passthrough = AMTPassthrough( + spec_id = entry.spec_id, + sort_order_id = entry.sort_order_id, + key_metadata = entry.key_metadata, + split_offsets = entry.split_offsets) + if (passthrough == AMTPassthrough()) None else Some(passthrough) + } } /** @@ -521,9 +682,9 @@ case class DataManifestEntry( record_count: Long, file_size_in_bytes: Long, manifest_info: ManifestInfo, - partition: Partition = Partition(), + partition: Option[Map[String, String]] = None, spec_id: Option[Int] = None, - content_stats: Option[ContentStats] = None, + content_stats: Option[String] = None, key_metadata: Option[Array[Byte]] = None, split_offsets: Option[Seq[Long]] = None, format_version: Int = AMTSingleAction.FormatVersionV4) @@ -546,7 +707,8 @@ case class DataManifestEntry( content_stats = content_stats, manifest_info = Some(manifest_info), key_metadata = key_metadata, - split_offsets = split_offsets) + split_offsets = split_offsets, + tags = None) /** Absolute [[Path]] to the referenced leaf manifest, resolving `location` against the root. */ @JsonIgnore @@ -572,16 +734,22 @@ case class DataManifestEntry( * populated pair is malformed and rejected. */ @JsonIgnore - def manifestDV: Option[(Array[Byte], Long)] = - (manifest_info.dv, manifest_info.dv_cardinality) match { - case (Some(dvBytes), Some(cardinality)) => Some((dvBytes, cardinality)) - case (None, None) => None - case _ => - throw new IllegalStateException( - s"Malformed manifest DV on leaf $location: dv and dv_cardinality must both be set or " + - s"both unset (dv.isDefined=${manifest_info.dv.isDefined}, " + - s"dv_cardinality=${manifest_info.dv_cardinality}).") + def manifestDV: Option[(Array[Byte], Long)] = { + AMTUtils.invariantCheckWithLogging( + checkInvariant = manifest_info.dv.isDefined == manifest_info.dv_cardinality.isDefined, + opTypeSuffix = AMTUsageLogs.ALERT_MALFORMED_MANIFEST_DV, + message = + s"Malformed manifest DV on leaf $location: dv and dv_cardinality must both be set or " + + s"both unset (dv.isDefined=${manifest_info.dv.isDefined}, " + + s"dv_cardinality=${manifest_info.dv_cardinality}).", + data = Map( + "leafLocation" -> location, + "dvIsDefined" -> manifest_info.dv.isDefined, + "dvCardinalityIsDefined" -> manifest_info.dv_cardinality.isDefined)) + manifest_info.dv.flatMap { dv => + manifest_info.dv_cardinality.map(cardinality => (dv, cardinality)) } + } } /** @@ -611,6 +779,25 @@ case class Tracking( s"Unsupported tracking status: $status.") } +/** + * A leaf manifest may be written before the commit version is assigned, so the tracking fields that + * depend on the commit could be written null. The root `DATA_MANIFEST` entry pointing at that + * leaf is written at commit time with explicit values, and a reader recovers each leaf entry's + * tracking by filling its nulls from that parent. This lets a leaf be written once and reused + * across commit retries, since only the root has to be rewritten. + * + * Every `status` named below is the child [[DataEntry]]'s status, not the parent + * `DATA_MANIFEST` entry's status. The inheritable fields are: + * + * - `sequence_number` (3): inherited when null and the status is `ADDED`. + * - `file_sequence_number` (4): inherited when null and the status is `ADDED`. + * - `first_row_id` (142): inherited when null and the status is `ADDED`, but not by simply + * copying the parent's value down. An entry takes the parent's `first_row_id` plus the summed + * `record_count` of the preceding `ADDED` entries in the same leaf that were themselves null. + * + * The `ADDED` restriction keeps the inherited fields correct. Leaf entries with any other status + * must materialize all three values instead of inheriting them. + */ object Tracking { /** * Closed set of `status` values, matching Iceberg V4 integer codes. @@ -628,27 +815,131 @@ object Tracking { val Replaced: Int = 3 val Modified: Int = 4 val all: Set[Int] = Set(Existing, Added, Deleted, Replaced, Modified) + + /** + * Statuses that keep an entry in the active table state, for both entry kinds: on a + * [[DataEntry]] the data file is still part of the table, and on a [[DataManifestEntry]] the + * leaf may still hold live [[DataEntry]]s, so a scan has to read it. + */ + val liveEntryStatuses: Set[Int] = Set(Existing, Added, Modified) + + /** The spec name of a `status` code, for error messages. */ + def nameOf(status: Int): String = status match { + case Existing => "EXISTING" + case Added => "ADDED" + case Deleted => "DELETED" + case Replaced => "REPLACED" + case Modified => "MODIFIED" + case other => s"UNKNOWN($other)" + } } + + /** + * Fills the null tracking fields of a leaf entry from the root `DATA_MANIFEST` entry that + * references its leaf. + * + * @param childTracking The leaf entry's tracking, as read off disk. + * @param parentTracking The inheritable values of the root entry referencing this leaf. + * @param prefixSumRecordCountForNullFirstRowId The summed `record_count` of the entries that + * physically precede this one in the same leaf and + * also carry a null `first_row_id`. + * @param childEntryLocationForLogging The child entry's `location`, used only to describe a + * malformed entry. + */ + def resolve( + childTracking: Tracking, + parentTracking: InheritableTracking, + prefixSumRecordCountForNullFirstRowId: Long, + childEntryLocationForLogging: String): Tracking = + childTracking.copy( + sequence_number = inheritTrackingValueOnAdded( + fieldName = "sequence_number", + fieldValueInChild = childTracking.sequence_number, + inheritedFieldValue = parentTracking.sequence_number, + dataEntryStatus = childTracking.status, + childEntryLocationForLogging = childEntryLocationForLogging), + file_sequence_number = inheritTrackingValueOnAdded( + fieldName = "file_sequence_number", + fieldValueInChild = childTracking.file_sequence_number, + inheritedFieldValue = parentTracking.file_sequence_number, + dataEntryStatus = childTracking.status, + childEntryLocationForLogging = childEntryLocationForLogging), + first_row_id = inheritTrackingValueOnAdded( + fieldName = "first_row_id", + fieldValueInChild = childTracking.first_row_id, + inheritedFieldValue = + parentTracking.first_row_id + prefixSumRecordCountForNullFirstRowId, + dataEntryStatus = childTracking.status, + childEntryLocationForLogging = childEntryLocationForLogging)) + + /** + * Resolves an inheritable tracking field. Only an `ADDED` child may inherit a missing value; + * every other status must materialize the field in the leaf. + */ + private def inheritTrackingValueOnAdded( + fieldName: String, + fieldValueInChild: Option[Long], + inheritedFieldValue: Long, + dataEntryStatus: Int, + childEntryLocationForLogging: String): Option[Long] = + fieldValueInChild.orElse { + if (dataEntryStatus == Status.Added) { + Some(inheritedFieldValue) + } else { + throw new IllegalStateException( + s"Malformed AMT data entry '$childEntryLocationForLogging': " + + s"tracking.$fieldName is null " + + s"but the entry's status is ${Status.nameOf(dataEntryStatus)}. Only ADDED entries " + + s"inherit $fieldName from the root DATA_MANIFEST entry; every other status must " + + "materialize it.") + } + } } /** - * Partition-values carrier for [[AMTSingleAction]] (Iceberg V4 `partition`, field 102). - * - * Iceberg models `partition` as a per-table dynamic struct (one typed field per - * partition column); that shape cannot be expressed statically here, so this PR carries - * Delta's raw `AddFile.partitionValues`. The struct is present at field 102 (the spec - * marks it required); binary interop with a strict V4 reader that expects the typed - * per-column struct is deferred. + * The subset of a root `DATA_MANIFEST` entry's [[Tracking]] that the entries of the leaf it + * points at can inherit. * + * Projecting the parent down to these values keeps the per-leaf parent map small enough + * to capture in a scan closure. * - * @param values Raw Delta partition values (column name to value); None when unpartitioned. + * @param sequence_number Parent's required data sequence number. + * @param file_sequence_number Parent's required file sequence number. + * @param first_row_id Required row id where the leaf's row-id assignment starts. */ -case class Partition(values: Option[Map[String, String]] = None) +case class InheritableTracking( + sequence_number: Long, + file_sequence_number: Long, + first_row_id: Long) + +object InheritableTracking { + /** Projects the required inheritable fields of a root `DATA_MANIFEST` entry. */ + def apply(parent: DataManifestEntry): InheritableTracking = InheritableTracking( + sequence_number = requiredParentValue(parent, "sequence_number", + parent.tracking.sequence_number), + file_sequence_number = requiredParentValue(parent, "file_sequence_number", + parent.tracking.file_sequence_number), + first_row_id = requiredParentValue(parent, "first_row_id", parent.tracking.first_row_id)) + + private def requiredParentValue( + parent: DataManifestEntry, + fieldName: String, + fieldValue: Option[Long]): Long = { + val message = + s"tracking.$fieldName must not be null in AMT root DATA_MANIFEST entries." + AMTUtils.invariantCheckWithLogging( + checkInvariant = fieldValue.isDefined, + opTypeSuffix = "amt.malformedDataManifestTracking", + message = message, + data = Map("leafLocation" -> parent.location, "fieldName" -> fieldName)) + fieldValue.get + } +} /** * Pointer to a deletion-vector blob, mirroring the Iceberg V4 `deletion_vector` struct. * - * @param location Absolute path of the file holding the DV blob. + * @param location Path of the file holding the DV blob. Relative to table root if possible. * @param offset Byte offset where the DV content starts within that file. * @param size_in_bytes Total on-disk DV size = raw bitmap + length + checksum framing. * @param cardinality Number of positions the DV marks deleted. @@ -660,7 +951,6 @@ case class DeletionVector( cardinality: Long) // ID: 156, required. object DeletionVector { - /** Maps a Delta on-disk [[DeletionVectorDescriptor]] onto the AMT sub-struct; rejects inline. */ def fromDescriptor(dv: DeletionVectorDescriptor, tableRoot: Path): DeletionVector = { require(dv.isOnDisk, @@ -668,55 +958,44 @@ object DeletionVector { val offset = dv.offset.getOrElse( throw new IllegalArgumentException( s"On-disk deletion vector is missing an offset: ${dv.pathOrInlineDv}.")) + val location = dv.normalizedTableRelativeObjectFile(tableRoot) match { + case (DeletionVectorDescriptor.RELATIVE_DV_MARKER, relativePath) => relativePath + // A p-DV still has to be URL decoded. + case (DeletionVectorDescriptor.PATH_DV_MARKER, _) => dv.absolutePath(tableRoot).toString + case (other, _) => + throw new IllegalArgumentException( + s"Unsupported on-disk deletion vector storage type: $other.") + } DeletionVector( - location = dv.absolutePath(tableRoot).toString, + location = location, offset = offset.toLong, size_in_bytes = DeletionVectorStore.getTotalSizeOfDVFieldsInFile(dv.sizeInBytes).toLong, cardinality = dv.cardinality) } - /** Rebuilds the Delta [[DeletionVectorDescriptor]] from the AMT sub-struct. */ - def toDescriptor(dv: DeletionVector, tableRoot: Path): DeletionVectorDescriptor = { - val rawSize = dv.size_in_bytes.toInt - - (DeletionVectorStore.getTotalSizeOfDVFieldsInFile(0)) - recoverRelativeUuid(dv.location, tableRoot) match { - case Some((id, randomPrefix)) => - DeletionVectorDescriptor.onDiskWithRelativePath( - id = id, - randomPrefix = randomPrefix, - sizeInBytes = rawSize, - cardinality = dv.cardinality, - offset = Some(dv.offset.toInt)) - case None => - DeletionVectorDescriptor.onDiskWithAbsolutePath( - path = dv.location, - sizeInBytes = rawSize, - cardinality = dv.cardinality, - offset = Some(dv.offset.toInt)) - } - } - /** - * If `location` is a Delta-written DV blob under `tableRoot` (a `deletion_vector_.bin` file - * directly under it or under one random-prefix directory), returns its UUID and prefix so the `u` - * descriptor can be rebuilt; None otherwise (treated as an absolute `p`). + * Rebuilds the Delta [[DeletionVectorDescriptor]] from the AMT sub-struct. */ - private def recoverRelativeUuid(location: String, tableRoot: Path): Option[(UUID, String)] = { - val path = new Path(location) - val parent = path.getParent - val uuid = - Try(DeletionVectorDescriptor.getUUIDFromDeletionVectorFileName(path.getName)).toOption - val rootStr = tableRoot.toString - uuid.flatMap { id => - if (parent.toString == rootStr) { - // /deletion_vector_.bin -- no random prefix. - Some((id, "")) - } else if (parent.getParent.toString == rootStr) { - // //deletion_vector_.bin -- one random-prefix directory. - Some((id, parent.getName)) - } else { - None - } + def toDescriptor(dv: DeletionVector, tableRoot: Path): DeletionVectorDescriptor = { + val rawSize = dv.size_in_bytes.toInt - + DeletionVectorStore.getTotalSizeOfDVFieldsInFile(0) + if (AMTUtils.isAbsoluteLocation(dv.location)) { + // AMT stored paths are unencoded. In case it has to be read as a p-DV, + // it must be URL encoded. + val storedPath = DeletionVectorStore.unescapedStringToPath(dv.location) + require(storedPath.isAbsolute) + DeletionVectorDescriptor.onDiskWithAbsolutePath( + path = DeletionVectorStore.pathToEscapedString(storedPath), + sizeInBytes = rawSize, + cardinality = dv.cardinality, + offset = Some(dv.offset.toInt)) + } else { + // No encoding if read as an r-DV. + DeletionVectorDescriptor.createRelativePathDVDescriptor( + relativePath = dv.location, + sizeInBytes = rawSize, + cardinality = dv.cardinality, + offset = Some(dv.offset.toInt)) } } } @@ -734,11 +1013,13 @@ object DeletionVector { * @param existing_files_count Count of EXISTING file entries. * @param deleted_files_count Count of DELETED file entries. * @param replaced_files_count Count of REPLACED file entries. + * @param modified_files_count Count of MODIFIED file entries. * @param added_rows_count Rows across ADDED files. * @param existing_rows_count Rows across EXISTING files. * @param deleted_rows_count Rows across DELETED files. * @param replaced_rows_count Rows across REPLACED files. - * @param min_sequence_number Minimum data sequence number across the manifest's entries. + * @param modified_rows_count Rows across MODIFIED files. + * @param min_sequence_number Minimum data sequence number across the manifest's live entries. * @param dv Inline manifest deletion-vector bitmap over leaf row positions. * @param dv_cardinality Number of positions the inline manifest DV marks. */ @@ -747,27 +1028,19 @@ case class ManifestInfo( existing_files_count: Int, // ID: 505, required. deleted_files_count: Int, // ID: 506, required. replaced_files_count: Int, // ID: 520, required. + modified_files_count: Int, // ID: 524, required. added_rows_count: Long, // ID: 512, required. existing_rows_count: Long, // ID: 513, required. deleted_rows_count: Long, // ID: 514, required. replaced_rows_count: Long, // ID: 521, required. + modified_rows_count: Long, // ID: 525, required. min_sequence_number: Long, // ID: 516, required. dv: Option[Array[Byte]], // ID: 522, optional (inline manifest DV bitmap). - dv_cardinality: Option[Long]) // ID: 523, optional. + dv_cardinality: Option[Long]) { // ID: 523, optional. -/** - * Column-level statistics carrier. Iceberg V4 leaves the inner shape - * up to the caller (it depends on the table schema). A future PR introduces - * the Delta-stats mapping. - * - * The single nullable `raw_stats` field is deliberate: an empty case class encodes to an - * empty Parquet group, which the Parquet data source rejects - * (`EMPTY_SCHEMA_NOT_SUPPORTED_FOR_DATASOURCE`). One nullable column keeps - * `AMTSingleAction` Parquet-writable without committing to the final stats shape; M1 - * writers leave it `None`. - * - * - * @param raw_stats Column-stats payload; M1 writers leave it None. - */ -case class ContentStats(raw_stats: Option[Array[Byte]] = None) + /** Live (non-tombstone) file entries: ADDED, EXISTING and MODIFIED are all live. */ + def liveFilesCount: Int = added_files_count + existing_files_count + modified_files_count + /** Tombstone file entries: DELETED and REPLACED. */ + def tombstoneFilesCount: Int = deleted_files_count + replaced_files_count +} diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/catalog/AbstractDeltaCatalog.scala b/spark/src/main/scala/org/apache/spark/sql/delta/catalog/AbstractDeltaCatalog.scala index 9818c229a23..b57d9425ef3 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/catalog/AbstractDeltaCatalog.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/catalog/AbstractDeltaCatalog.scala @@ -28,7 +28,7 @@ import scala.collection.mutable import io.delta.storage.commit.uccommitcoordinator.UCCommitCoordinatorClient.UC_TABLE_ID_KEY import io.delta.storage.commit.uccommitcoordinator.UCCommitCoordinatorClient.UC_TABLE_ID_KEY_OLD import org.apache.spark.sql.delta.skipping.clustering.ClusteredTableUtils -import org.apache.spark.sql.delta.skipping.clustering.temp.{ClusterBy, ClusterBySpec} +import org.apache.spark.sql.delta.skipping.clustering.temp.{ClusterBy, ClusterBySpec, ReplacePartitionedByWithClusterBy} import org.apache.spark.sql.delta.skipping.clustering.temp.{ClusterByTransform => TempClusterByTransform} import org.apache.spark.sql.delta.{ColumnWithDefaultExprUtils, DeltaConfigs, DeltaErrors, DeltaTableUtils} import org.apache.spark.sql.delta.{DeltaOptions, IdentityColumn} @@ -1181,6 +1181,8 @@ class AbstractDeltaCatalog extends DelegatingCatalogExtension case deltaTable: DeltaTableV2 => deltaTable case _ if changes.exists(_.isInstanceOf[ClusterBy]) => throw DeltaErrors.alterClusterByNotOnDeltaTableException() + case _ if changes.exists(_.isInstanceOf[ReplacePartitionedByWithClusterBy]) => + throw DeltaErrors.alterClusterByNotOnDeltaTableException() case _ if changes.exists(_.isInstanceOf[SyncIdentity]) => throw DeltaErrors.identityColumnAlterNonDeltaFormatError() case _ => return super.alterTable(ident, changes: _*) @@ -1403,6 +1405,15 @@ class AbstractDeltaCatalog extends DelegatingCatalogExtension AlterTableClusterByDeltaCommand( table, c.clusteringColumns.map(_.fieldNames().toSeq).toSeq).run(spark) } + + case (t, replace) if t == classOf[ReplacePartitionedByWithClusterBy] => + replace.asInstanceOf[Seq[ReplacePartitionedByWithClusterBy]].foreach { r => + if (r.clusteringColumns.nonEmpty) { + validateClusterBySpec(Some(ClusterBySpec(r.clusteringColumns.toSeq)), table.schema()) + } + AlterTableReplacePartitionedByWithClusterByDeltaCommand( + table, r.clusteringColumns.map(_.fieldNames().toSeq).toSeq).run(spark) + } } if (columnUpdates.nonEmpty) { diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/catalog/UCDeltaCatalogClientImpl.scala b/spark/src/main/scala/org/apache/spark/sql/delta/catalog/UCDeltaCatalogClientImpl.scala index 4b0b133ba35..478d2f0e274 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/catalog/UCDeltaCatalogClientImpl.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/catalog/UCDeltaCatalogClientImpl.scala @@ -50,6 +50,7 @@ import org.apache.spark.sql.connector.catalog.{CatalogV2Util, Identifier, Table, import org.apache.spark.sql.delta.{CatalogOwnedTableFeature, ClusteringTableFeature, DeltaConfigs, DeltaErrors, MaterializedRowCommitVersion, MaterializedRowId, TableFeature} import org.apache.spark.sql.delta.actions.{Action, AddFile, CommitInfo, DomainMetadata, Metadata, Protocol, RemoveFile, TableFeatureProtocolUtils} import org.apache.spark.sql.delta.actions.TableFeatureProtocolUtils.FEATURE_PROP_SUPPORTED +import org.apache.spark.sql.delta.CatalogManagedTableMaintenanceOperation.ALLOWED_OPERATIONS_PROPERTY import org.apache.spark.sql.delta.coordinatedcommits.UCTokenBasedRestClientFactory import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.IcebergConstants @@ -617,12 +618,22 @@ private[catalog] class UCDeltaCatalogClientImpl( iceberg.getConvertedDeltaTimestamp ) }.getOrElse(Map.empty[String, String]) + val clientMaintenanceOperations = info.getClientMaintenanceOperations.asScala + val mergedStorageProperties = { + val merged = tableConfig ++ info.getStorageProperties.asScala.toMap ++ uniformProps + // Only the catalog's list grants permission. A user-defined table property must not. + if (clientMaintenanceOperations.isEmpty) { + merged - ALLOWED_OPERATIONS_PROPERTY + } else { + merged + (ALLOWED_OPERATIONS_PROPERTY -> clientMaintenanceOperations.mkString(",")) + } + } // Match UCSingleCatalog's V1Table shape: pack tableConfig + credentials + UniForm into // `storage.properties`, leave `catalogTable.properties` empty. Required for // downstream streaming/routing compatibility. val storage = CatalogStorageFormat.empty.copy( locationUri = Some(new URI(info.getLocation)), - properties = tableConfig ++ info.getStorageProperties.asScala.toMap ++ uniformProps) + properties = mergedStorageProperties) val catalogTable = CatalogTable( identifier = TableIdentifier(ident.name(), ident.namespace().headOption, Some(catalogName)), tableType = fromUcTableType(info.getTableType), diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/commands/AlterTableReplacePartitionedByWithClusterByDeltaCommand.scala b/spark/src/main/scala/org/apache/spark/sql/delta/commands/AlterTableReplacePartitionedByWithClusterByDeltaCommand.scala new file mode 100644 index 00000000000..00fc22335ff --- /dev/null +++ b/spark/src/main/scala/org/apache/spark/sql/delta/commands/AlterTableReplacePartitionedByWithClusterByDeltaCommand.scala @@ -0,0 +1,241 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.commands + +import java.util.Locale + +import scala.collection.JavaConverters._ +import scala.util.Try + +import org.apache.spark.sql.delta.skipping.clustering.ClusteredTableUtils +import org.apache.spark.sql.delta.skipping.clustering.temp.ClusterBySpec +import org.apache.spark.sql.delta._ +import org.apache.spark.sql.delta.actions.Action +import org.apache.spark.sql.delta.catalog.DeltaTableV2 +import org.apache.spark.sql.delta.sources.DeltaSQLConf +import org.apache.spark.sql.delta.stats.DeltaStatistics +import org.apache.spark.sql.delta.util.{DeltaFileOperations, JsonUtils} +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.node.{DecimalNode, LongNode, ObjectNode, TextNode} +import org.apache.parquet.format.converter.ParquetMetadataConverter + +import org.apache.spark.sql.{Row, SparkSession} +import org.apache.spark.sql.connector.expressions.FieldReference +import org.apache.spark.sql.execution.command.LeafRunnableCommand +import org.apache.spark.sql.execution.datasources.parquet.ParquetFooterReaderShims +import org.apache.spark.sql.types._ +import org.apache.spark.util.SerializableConfiguration + +/** + * Command that replaces the partitioning of a Delta table with clustering, without rewriting + * any data file: + * - ALTER TABLE .. REPLACE PARTITIONED BY WITH CLUSTER BY (col1, col2, ...) + * - ALTER TABLE .. REPLACE PARTITIONED BY WITH CLUSTER BY NONE + * + * The conversion is a single metadata-only commit which + * 1. clears the partition columns in the table metadata, + * 2. re-adds every active file with empty `partitionValues` and `dataChange = false`, keeping its + * path, deletion vector and row tracking fields. Per-file min/max/nullCount statistics are + * synthesized for the former partition columns from the (constant) partition values so that + * data skipping on these columns keeps working, and + * 3. when clustering columns are given, enables clustering and records the clustering columns. + * + * After the conversion the former partition columns are read from the data files, so they must be + * physically stored in every data file (see `delta.writePartitionColumnsToParquet` and the + * `materializePartitionColumns` table feature). This is verified by reading the Parquet footers + * before committing. + * + * The commit is done with [[OptimisticTransaction.commitLarge]], which fails if any other commit + * happened since the transaction started. Hence no file can be added concurrently between the + * verification and the commit. + */ +case class AlterTableReplacePartitionedByWithClusterByDeltaCommand( + table: DeltaTableV2, + clusteringColumns: Seq[Seq[String]]) + extends LeafRunnableCommand with AlterDeltaTableCommand { + + override def run(sparkSession: SparkSession): Seq[Row] = { + val deltaLog = table.deltaLog + recordDeltaOperation(deltaLog, "delta.ddl.alter.replacePartitionedByWithClusterBy") { + val txn = startTransaction() + val snapshot = txn.snapshot + val oldMetadata = txn.metadata + if (oldMetadata.partitionColumns.isEmpty) { + throw DeltaErrors.replacePartitionedByOnUnpartitionedTableException() + } + ClusteredTableUtils.validateNumClusteringColumns(clusteringColumns, Some(deltaLog)) + + val physicalPartitionSchema = oldMetadata.physicalPartitionSchema + if (sparkSession.sessionState.conf.getConf( + DeltaSQLConf.DELTA_REPLACE_PARTITIONED_BY_VERIFY_MATERIALIZED_PARTITION_COLUMNS)) { + ReplacePartitionedByUtils.verifyPartitionColumnsMaterialized( + sparkSession, deltaLog, snapshot, oldMetadata.partitionColumns, + physicalPartitionSchema.fieldNames.toSeq) + } + + val newLogicalClusteringColumns = clusteringColumns.map(FieldReference(_).toString) + val baseConfiguration = oldMetadata.configuration + val newConfiguration = + if (clusteringColumns.nonEmpty && !ClusteredTableUtils.isSupported(txn.protocol)) { + baseConfiguration ++ ClusteredTableUtils.getTableFeatureProperties(baseConfiguration) + } else { + baseConfiguration + } + txn.updateMetadata(oldMetadata.copy(partitionColumns = Nil, configuration = newConfiguration)) + + val domainMetadata = if (clusteringColumns.nonEmpty) { + ClusteredTableUtils.validateClusteringColumnsInStatsSchema( + txn.protocol, txn.metadata, ClusterBySpec(clusteringColumns)) + ClusteredTableUtils.getClusteringDomainMetadataForAlterTableClusterBy( + newLogicalClusteringColumns, txn) + } else { + Nil + } + + val stringPrefixLength = sparkSession.sessionState.conf.getConf( + DeltaSQLConf.DATA_SKIPPING_STRING_PREFIX_LENGTH) + val partitionFields = physicalPartitionSchema.fields.toSeq + import org.apache.spark.sql.delta.implicits._ + val newAddFiles = snapshot.allFiles.mapPartitions { files => + files.map { file => + file.copy( + partitionValues = Map.empty, + dataChange = false, + stats = ReplacePartitionedByUtils.addPartitionValuesToStats( + file.stats, file.partitionValues, partitionFields, stringPrefixLength)) + } + } + + val actions: Iterator[Action] = + newAddFiles.toLocalIterator().asScala ++ domainMetadata.iterator + val newProtocolOpt = + if (txn.protocol != snapshot.protocol) Some(txn.protocol) else None + txn.commitLarge( + sparkSession, + actions, + newProtocolOpt, + DeltaOperations.ReplacePartitionedByWithClusterBy( + oldMetadata.partitionColumns.mkString(","), + newLogicalClusteringColumns.mkString(",")), + context = Map.empty, + metrics = Map.empty, + dataChange = Some(false)) + } + Seq.empty[Row] + } +} + +object ReplacePartitionedByUtils { + + private val NUM_EXAMPLE_FILES = 5 + + /** + * Verifies that every active data file of the snapshot physically stores all the given + * partition columns, by reading the Parquet footers of the files. + */ + def verifyPartitionColumnsMaterialized( + spark: SparkSession, + deltaLog: DeltaLog, + snapshot: Snapshot, + logicalPartitionColumns: Seq[String], + physicalPartitionColumns: Seq[String]): Unit = { + val broadcastConf = spark.sparkContext.broadcast( + new SerializableConfiguration(deltaLog.newDeltaHadoopConf())) + val dataRootDir = deltaLog.dataPath.toString + val requiredColumns = physicalPartitionColumns.map(_.toLowerCase(Locale.ROOT)) + + import org.apache.spark.sql.delta.implicits._ + val filesMissingColumns = snapshot.allFiles.select("path").as[String].mapPartitions { paths => + val conf = broadcastConf.value.value + paths.filter { relativePath => + val path = DeltaFileOperations.absolutePath(dataRootDir, relativePath) + val status = path.getFileSystem(conf).getFileStatus(path) + val footer = ParquetFooterReaderShims.readParquetFooter( + conf, status, ParquetMetadataConverter.SKIP_ROW_GROUPS) + val fileColumns = footer.getFileMetaData.getSchema.getFields.asScala + .map(_.getName.toLowerCase(Locale.ROOT)).toSet + !requiredColumns.forall(fileColumns.contains) + } + } + + val examples = filesMissingColumns.take(NUM_EXAMPLE_FILES) + if (examples.nonEmpty) { + throw DeltaErrors.replacePartitionedByPartitionColumnsNotMaterializedException( + logicalPartitionColumns, filesMissingColumns.count(), examples.toSeq) + } + } + + /** + * Adds min/max/nullCount statistics for the given partition columns to the stats JSON of a + * file, using the file's partition values. Every row in a file has the same value for a + * partition column, so these statistics are exact. Statistics that already exist are kept, and + * no statistics are added if the file has none. + * + * Min/max values are only synthesized for types whose JSON stats representation is the same as + * the partition value representation; other types only get a null count. + */ + def addPartitionValuesToStats( + stats: String, + partitionValues: Map[String, String], + partitionFields: Seq[StructField], + stringPrefixLength: Int): String = { + if (stats == null || stats.isEmpty) return stats + val root = Try(JsonUtils.mapper.readTree(stats)).toOption match { + case Some(o: ObjectNode) => o + case _ => return stats + } + val numRecords = root.get(DeltaStatistics.NUM_RECORDS) + if (numRecords == null || !numRecords.canConvertToLong) return stats + + def objectField(name: String): Option[ObjectNode] = root.get(name) match { + case null => Some(root.putObject(name)) + case o: ObjectNode => Some(o) + case _ => None + } + val minValues = objectField(DeltaStatistics.MIN) + val maxValues = objectField(DeltaStatistics.MAX) + val nullCounts = objectField(DeltaStatistics.NULL_COUNT) + + partitionFields.foreach { field => + val name = field.name + partitionValues.getOrElse(name, null) match { + case null => + nullCounts.filterNot(_.has(name)).foreach(_.put(name, numRecords.asLong)) + case value => + nullCounts.filterNot(_.has(name)).foreach(_.put(name, 0L)) + toStatsValue(value, field.dataType, stringPrefixLength).foreach { v => + minValues.filterNot(_.has(name)).foreach(_.set[JsonNode](name, v)) + maxValues.filterNot(_.has(name)).foreach(_.set[JsonNode](name, v)) + } + } + } + JsonUtils.mapper.writeValueAsString(root) + } + + private def toStatsValue(value: String, dataType: DataType, stringPrefixLength: Int) + : Option[JsonNode] = dataType match { + // Longer strings would need to be truncated the same way as collected stats. + case StringType if value.length <= stringPrefixLength => Some(TextNode.valueOf(value)) + case ByteType | ShortType | IntegerType | LongType => + Try(value.trim.toLong).toOption.map(LongNode.valueOf) + case _: DecimalType => + Try(new java.math.BigDecimal(value.trim)).toOption.map(DecimalNode.valueOf) + case DateType => + Try(java.time.LocalDate.parse(value.trim)).toOption.map(d => TextNode.valueOf(d.toString)) + case _ => None + } +} diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/commands/CloneTableBase.scala b/spark/src/main/scala/org/apache/spark/sql/delta/commands/CloneTableBase.scala index eb06047a7ac..5f7752234bd 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/commands/CloneTableBase.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/commands/CloneTableBase.scala @@ -229,9 +229,10 @@ abstract class CloneTableBase( addFileIter.map { fileToCopy => val copiedFile = fileToCopy.copy(dataChange = dataChangeInFileAction) // CLONE does not preserve Row IDs and Commit Versions, nor the source table's AMT - // back reference. + // back reference and passthrough. copiedFile.copy( - baseRowId = None, defaultRowCommitVersion = None, backReference = None) + baseRowId = None, defaultRowCommitVersion = None, backReference = None, + amtPassthrough = None) } sourceTable.snapshot.foreach { sourceSnapshot => // Handle DomainMetadata for cloning a table. @@ -259,6 +260,11 @@ abstract class CloneTableBase( } } + // Every file action above is stamped with `dataChangeInFileAction`, so the commit changes + // data exactly when that is true and there is at least one file action. + val fileActionCount = addedFileCount + val commitDataChange = dataChangeInFileAction && fileActionCount > 0 + recordDeltaOperation( destinationTable, s"delta.${deltaOperation.name.toLowerCase()}.commit") { txn.commitLarge( @@ -267,7 +273,8 @@ abstract class CloneTableBase( Some(newProtocol), deltaOperation, context, - commitOpMetrics.mapValues(_.toString()).toMap) + commitOpMetrics.mapValues(_.toString()).toMap, + dataChange = Some(commitDataChange)) } val cloneLogData = getOperationMetricsForEventRecord(opMetrics) ++ Map( diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/commands/ConvertToDeltaCommand.scala b/spark/src/main/scala/org/apache/spark/sql/delta/commands/ConvertToDeltaCommand.scala index 99e85aefcdf..64f374d7d29 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/commands/ConvertToDeltaCommand.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/commands/ConvertToDeltaCommand.scala @@ -399,19 +399,24 @@ abstract class ConvertToDeltaCommandBase( checkConversionIsAllowed(txn, targetTable) val numFiles = targetTable.numFiles - val addFilesIter = createDeltaActions(spark, manifest, partitionFields, txn, fs) + val addFilesIter = createDeltaActions(spark, manifest, partitionFields, txn, fs).buffered val transactionMetrics = Map[String, String]( "numConvertedFiles" -> numFiles.toString ) metrics("numConvertedFiles") += numFiles sendDriverMetrics(spark, metrics) + val convertsAnyFile = addFilesIter.hasNext + if (convertsAnyFile) { + assert(addFilesIter.head.dataChange, "CONVERT must emit AddFiles with dataChange = true") + } txn.commitLarge( spark, addFilesIter, Some(txn.protocol), getOperation(numFiles, convertProperties, targetTable.format), getContext, - transactionMetrics) + transactionMetrics, + dataChange = Some(convertsAnyFile)) } finally { manifest.close() } diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/commands/CreateDeltaTableCommand.scala b/spark/src/main/scala/org/apache/spark/sql/delta/commands/CreateDeltaTableCommand.scala index b32f0ea2690..2d1e63ee317 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/commands/CreateDeltaTableCommand.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/commands/CreateDeltaTableCommand.scala @@ -17,6 +17,7 @@ package org.apache.spark.sql.delta.commands // scalastyle:off import.ordering.noEmptyLine +import java.util.Locale import java.util.concurrent.TimeUnit import scala.util.Try @@ -48,7 +49,7 @@ import org.apache.spark.sql.execution.command.{LeafRunnableCommand, RunnableComm import org.apache.spark.sql.execution.metric.SQLMetric import org.apache.spark.sql.execution.metric.SQLMetrics.createMetric import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.types.StructType +import org.apache.spark.sql.types.{ArrayType, DataType, MapType, StructType} import org.apache.spark.util.Utils /** @@ -834,6 +835,22 @@ case class CreateDeltaTableCommand( // Unity Catalog table id stored in `io.unitycatalog.tableId`. newMetadata = newMetadata.copy(id = txn.snapshot.metadata.id) } + + // Carry over table and column comments from the old table when the new DDL + // does not explicitly specify them. + if (sparkSession.conf.get(DeltaSQLConf.RETAIN_COMMENTS_DURING_REPLACE_TABLE)) { + // Table comments + if (newMetadata.description == null) { + newMetadata = newMetadata.copy(description = txn.metadata.description) + } + // Schema / column comments, only when overwriteSchema=true is not set + if (!options.canOverwriteSchema) { + val updatedSchema = CreateDeltaTableCommand.carryOverStructTypeComments( + txn.metadata.schema, newMetadata.schema) + newMetadata = newMetadata.copy(schemaString = updatedSchema.json) + } + } + txn.updateMetadataForNewTableInReplace(newMetadata) } } @@ -879,6 +896,61 @@ case class CreateDeltaTableCommand( } } +object CreateDeltaTableCommand { + /** + * Carries over comments from the old schema to the new schema for all fields matched by name + * (case-insensitively), when the new field does not already have an explicit comment. This + * applies to every top-level column as well as recursively through nested struct type. + */ + private[delta] def carryOverStructTypeComments( + oldStruct: StructType, + newStruct: StructType): StructType = { + // Match field names case-insensitively. + def normalizeName(name: String): String = name.toLowerCase(Locale.ROOT) + val oldFieldsByName = oldStruct.fields.map(field => normalizeName(field.name) -> field).toMap + StructType(newStruct.fields.map { newField => + val matched = oldFieldsByName.get(normalizeName(newField.name)) + // Keep the new field's comment when it has one; otherwise carry over the old field's. + val fieldWithComment = if (newField.metadata.contains("comment")) { + newField + } else { + matched.flatMap(_.getComment()) match { + case Some(c) => newField.withComment(c) + case None => newField + } + } + // Recurse into nested types to carry over comments on inner struct fields + val newDataType = matched + .map(field => + carryOverDataTypeComments(field.dataType, fieldWithComment.dataType) + ) + .getOrElse(fieldWithComment.dataType) + fieldWithComment.copy(dataType = newDataType) + }) + } + + /** + * Recursively carries over column comments through nested DataTypes. + * Handles StructType, ArrayType, and MapType at arbitrary nesting depth. + */ + private[delta] def carryOverDataTypeComments( + oldType: DataType, + newType: DataType): DataType = { + (oldType, newType) match { + case (oldStruct: StructType, newStruct: StructType) => + carryOverStructTypeComments(oldStruct, newStruct) + case (ArrayType(oldElem, _), ArrayType(newElem, nullable)) => + ArrayType(carryOverDataTypeComments(oldElem, newElem), nullable) + case (MapType(oldK, oldV, _), MapType(newK, newV, nullable)) => + MapType(carryOverDataTypeComments(oldK, newK), + carryOverDataTypeComments(oldV, newV), + nullable + ) + case _ => newType + } + } +} + // isCreate is true for Create and CreateOrReplace modes. It is false for Replace mode. object TableCreationModes { sealed trait CreationMode { diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/commands/DMLWithDeletionVectorsHelper.scala b/spark/src/main/scala/org/apache/spark/sql/delta/commands/DMLWithDeletionVectorsHelper.scala index 54eb681afb9..3d984aaa222 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/commands/DMLWithDeletionVectorsHelper.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/commands/DMLWithDeletionVectorsHelper.scala @@ -691,7 +691,7 @@ object DeletionVectorWriter extends DeltaLogging { DeletionVectorDescriptor.EMPTY } else { val dvRange = ctx.writer.write(bitmapData) - DeletionVectorDescriptor.onDiskWithRelativePath( + DeletionVectorDescriptor.onDiskWithUuidRelativePath( id = ctx.fileId, randomPrefix = ctx.prefix, sizeInBytes = bitmapData.length, diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/commands/DeleteCommand.scala b/spark/src/main/scala/org/apache/spark/sql/delta/commands/DeleteCommand.scala index 6cd61bf0961..234062a12bb 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/commands/DeleteCommand.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/commands/DeleteCommand.scala @@ -121,7 +121,8 @@ case class DeleteCommand( override def innerChildren: Seq[QueryPlan[_]] = Seq(target) - override val output: Seq[Attribute] = Seq(AttributeReference("num_affected_rows", LongType)()) + override val output: Seq[Attribute] = + Seq(AttributeReference("num_affected_rows", LongType, nullable = false)()) override lazy val metrics = createMetrics diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/commands/DeltaCommand.scala b/spark/src/main/scala/org/apache/spark/sql/delta/commands/DeltaCommand.scala index 229465e6ad9..ca7d30e924d 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/commands/DeltaCommand.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/commands/DeltaCommand.scala @@ -432,10 +432,7 @@ trait DeltaCommand extends DeltaLogging with DeltaCommandInvariants { // check that both session configs are set numOptions = txnVersion.size + txnAppId.size if (numOptions != 0 && numOptions != 2) { - throw DeltaErrors.invalidIdempotentWritesOptionsException( - "Both spark.databricks.delta.write.txnAppId and " + - "spark.databricks.delta.write.txnVersion must be specified for " + - "idempotent Delta writes") + throw DeltaErrors.invalidIdempotentWritesMissingSessionConfsException() } fromSessionConf = true } diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/commands/DeltaReorgTableCommand.scala b/spark/src/main/scala/org/apache/spark/sql/delta/commands/DeltaReorgTableCommand.scala index 8ed02f93e01..7a474909e6a 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/commands/DeltaReorgTableCommand.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/commands/DeltaReorgTableCommand.scala @@ -16,7 +16,11 @@ package org.apache.spark.sql.delta.commands -import org.apache.spark.sql.delta.{DeltaColumnMapping, DeltaErrors, SnapshotDescriptor} +import org.apache.spark.sql.delta.{ + CatalogManagedTableMaintenanceOperation, + DeltaColumnMapping, + DeltaErrors, + SnapshotDescriptor} import org.apache.spark.sql.delta.actions.AddFile import org.apache.spark.sql.{Row, SparkSession} @@ -75,9 +79,10 @@ case class DeltaReorgTableCommand( optimizeByReorg(sparkSession) case DeltaReorgTableSpec(DeltaReorgTableMode.UNIFORM_ICEBERG, Some(icebergCompatVersion)) => val table = getDeltaTable(target, "REORG") - if (table.update().isCatalogOwned) { - throw DeltaErrors.operationBlockedOnCatalogManagedTable("REORG") - } + DeltaErrors.checkCatalogManagedTableOperationAllowed( + CatalogManagedTableMaintenanceOperation.DATA_REORGANIZATION, + table.update(), + table.catalogTable) upgradeUniformIcebergCompatVersion(table, sparkSession, icebergCompatVersion) } diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/commands/MergeIntoCommand.scala b/spark/src/main/scala/org/apache/spark/sql/delta/commands/MergeIntoCommand.scala index 3459a2588f1..076c8b5ebf7 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/commands/MergeIntoCommand.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/commands/MergeIntoCommand.scala @@ -77,10 +77,10 @@ case class MergeIntoCommand( with ClassicMergeExecutor { override val output: Seq[Attribute] = Seq( - AttributeReference("num_affected_rows", LongType)(), - AttributeReference("num_updated_rows", LongType)(), - AttributeReference("num_deleted_rows", LongType)(), - AttributeReference("num_inserted_rows", LongType)()) + AttributeReference("num_affected_rows", LongType, nullable = false)(), + AttributeReference("num_updated_rows", LongType, nullable = false)(), + AttributeReference("num_deleted_rows", LongType, nullable = false)(), + AttributeReference("num_inserted_rows", LongType, nullable = false)()) protected def runMerge(spark: SparkSession): Seq[Row] = { recordDeltaOperation(targetDeltaLog, "delta.dml.merge") { diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala b/spark/src/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala index d40f2972043..4f186379eab 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala @@ -46,7 +46,6 @@ import org.apache.spark.sql.execution.metric.SQLMetrics.createMetric import org.apache.spark.sql.types._ import org.apache.spark.util.{SystemClock, ThreadUtils} import org.apache.spark.sql.catalyst.catalog.CatalogTable -import org.apache.spark.sql.delta.actions.InMemoryLogReplay.UniqueFileActionTuple /** Base class defining abstract optimize command */ abstract class OptimizeTableCommandBase extends RunnableCommand with DeltaCommand { @@ -162,9 +161,10 @@ case class OptimizeTableCommand( throw DeltaErrors.notADeltaTableException(table.deltaLog.dataPath.toString) } - if (snapshot.isCatalogOwned) { - throw DeltaErrors.operationBlockedOnCatalogManagedTable("OPTIMIZE") - } + DeltaErrors.checkCatalogManagedTableOperationAllowed( + CatalogManagedTableMaintenanceOperation.DATA_REORGANIZATION, + snapshot, + table.catalogTable) val isClusteredTable = ClusteredTableUtils.isSupported(snapshot.protocol) if (isClusteredTable) { @@ -475,15 +475,22 @@ class OptimizeExecutor( val newPartitionSchema = newTxn.metadata.partitionSchema // Note: When checking if the candidate set is the same, we need to consider (Path, DV) // as the key. - val candidateSetOld = filesToProcess. - map(f => UniqueFileActionTuple(f.pathAsUri, f.getDeletionVectorUniqueId)).toSet + val useObjectIdentity = + FileAction.useDeletionVectorObjectIdentity( + snapshot.metadata, snapshot.protocol, sparkSession) + val candidateSetOld = filesToProcess + .map(f => + f.toUniqueFileActionTuple(snapshot.dataPath, useObjectIdentity)) + .toSet // We specifically don't list the files through the transaction since we are potentially // only processing a subset of them below. If the transaction is still valid, we will // register the files and predicate below val candidateSetNew = newTxn.snapshot.filesForScan(partitionPredicate).files - .map(f => UniqueFileActionTuple(f.pathAsUri, f.getDeletionVectorUniqueId)).toSet + .map(f => + f.toUniqueFileActionTuple(newTxn.snapshot.dataPath, useObjectIdentity)) + .toSet // As long as all of the files that we compacted are still part of the table, // and the partitioning has not changed it is valid to continue to try diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/commands/ReorgTableHelper.scala b/spark/src/main/scala/org/apache/spark/sql/delta/commands/ReorgTableHelper.scala index 440a36537ba..d2ef4404898 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/commands/ReorgTableHelper.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/commands/ReorgTableHelper.scala @@ -113,7 +113,9 @@ trait ReorgTableHelper extends Serializable { import org.apache.spark.sql.delta.implicits._ files.toDF(spark).as[AddFile].mapPartitions { iter => - val sqlConf = SparkSession.active.sessionState.conf + // Runs on executors, where no SparkSession is active; SQLConf.get returns the task's + // read-only conf propagated from the driver. + val sqlConf = SQLConf.get filterParquetFiles( sqlConf, iter.toList, diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/commands/RestoreTableCommand.scala b/spark/src/main/scala/org/apache/spark/sql/delta/commands/RestoreTableCommand.scala index 99124d029e0..f16d39e3019 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/commands/RestoreTableCommand.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/commands/RestoreTableCommand.scala @@ -22,7 +22,7 @@ import scala.collection.JavaConverters._ import scala.util.{Success, Try} import org.apache.spark.sql.delta.{DeltaErrors, DeltaLog, DeltaOperations, DomainMetadataUtils, IdentityColumn, Snapshot} -import org.apache.spark.sql.delta.actions.{AddFile, DeletionVectorDescriptor, RemoveFile} +import org.apache.spark.sql.delta.actions.{AddFile, DeletionVectorDescriptor, FileAction, RemoveFile} import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util.DeltaFileOperations.absolutePath @@ -136,6 +136,22 @@ case class RestoreTableCommand(sourceTable: DeltaTableV2) files.map(file => (file, file.path)) }.toDF("tgtAddFile", "tgtPath") + val dvTableRoot = deltaLog.dataPath + // We only compare current and target snapshots for object identity decision. + // An AMT period in between should be always safe proteced by the blocking checkpoint + // at AMT downgrade. + val useDeletionVectorObjectIdentity = + FileAction.useDeletionVectorObjectIdentity( + snapshotToRestore.metadata, + snapshotToRestore.protocol, + spark + ) || + FileAction.useDeletionVectorObjectIdentity( + latestSnapshot.metadata, + latestSnapshot.protocol, + spark + ) + def addDVsToNormalizedDF( mayHaveDVs: Boolean, dvIdColumnName: String, @@ -144,7 +160,8 @@ case class RestoreTableCommand(sourceTable: DeltaTableV2) if (mayHaveDVs) { normalizedDf.withColumn( dvIdColumnName, - DeletionVectorDescriptor.uniqueIdExpression(dvAccessColumn)) + DeletionVectorDescriptor.uniqueIdExpression( + dvAccessColumn, dvTableRoot, useDeletionVectorObjectIdentity)) } else { normalizedDf.withColumn(dvIdColumnName, lit(null)) } @@ -222,13 +239,17 @@ case class RestoreTableCommand(sourceTable: DeltaTableV2) val actions = addActions ++ removeActions ++ DomainMetadataUtils.handleDomainMetadataForRestoreTable(snapshotToRestore, latestSnapshot) + val restoreChangesData = + metrics.getOrElse(NUM_RESTORED_FILES, 0L) + metrics.getOrElse(NUM_REMOVED_FILES, 0L) > 0 + txn.commitLarge( spark, actions, Some(newProtocol), DeltaOperations.Restore(version, timestamp), Map.empty, - metrics.mapValues(_.toString).toMap) + metrics.mapValues(_.toString).toMap, + dataChange = Some(restoreChangesData)) Seq(Row( metrics.get(TABLE_SIZE_AFTER_RESTORE), diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/commands/UpdateCommand.scala b/spark/src/main/scala/org/apache/spark/sql/delta/commands/UpdateCommand.scala index a790660d761..9e404a4554f 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/commands/UpdateCommand.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/commands/UpdateCommand.scala @@ -65,7 +65,7 @@ case class UpdateCommand( extends LeafRunnableCommand with DeltaCommand { override val output: Seq[Attribute] = { - Seq(AttributeReference("num_affected_rows", LongType)()) + Seq(AttributeReference("num_affected_rows", LongType, nullable = false)()) } override def innerChildren: Seq[QueryPlan[_]] = Seq(target) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/commands/VacuumCommand.scala b/spark/src/main/scala/org/apache/spark/sql/delta/commands/VacuumCommand.scala index 07e4d5bda50..d1da4554abc 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/commands/VacuumCommand.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/commands/VacuumCommand.scala @@ -41,7 +41,6 @@ import org.apache.spark.broadcast.Broadcast import org.apache.spark.internal.MDC import org.apache.spark.paths.SparkPath import org.apache.spark.sql.{Column, DataFrame, Dataset, Encoder, SparkSession} -import org.apache.spark.sql.catalyst.catalog.CatalogTableType import org.apache.spark.sql.execution.metric.SQLMetric import org.apache.spark.sql.execution.metric.SQLMetrics.createMetric import org.apache.spark.sql.functions.{col, count, lit, replace, startswith, substr, sum} @@ -167,16 +166,10 @@ object VacuumCommand extends VacuumCommandImpl with Serializable { val snapshot = table.update() deltaLog.protocolWrite(snapshot.protocol) - if (snapshot.isCatalogOwned) { - table.catalogTable.foreach { catalogTable => - assert( - catalogTable.tableType == CatalogTableType.MANAGED, - s"All Catalog Owned tables should be MANAGED tables, " + - s"but found ${catalogTable.tableType} for table ${catalogTable.identifier}." - ) - } - throw DeltaErrors.operationBlockedOnCatalogManagedTable("VACUUM") - } + DeltaErrors.checkCatalogManagedTableOperationAllowed( + CatalogManagedTableMaintenanceOperation.DATA_CLEANUP, + snapshot, + table.catalogTable) // By default, we will do full vacuum unless LITE vacuum conf is set val isLiteVacuumEnabled = spark.sessionState.conf.getConf(DeltaSQLConf.LITE_VACUUM_ENABLED) @@ -744,7 +737,7 @@ trait VacuumCommandImpl extends DeltaCommand { dv match { case Some(dv) if dv.isOnDisk => - if (dv.isRelative) { + if (dv.isUuidRelative) { // We actually want a relative path here. Some((pathToUrlEncodedString(dv.absolutePath(new Path("."))), dv.sizeInBytes)) } else { @@ -809,38 +802,8 @@ trait VacuumCommandImpl extends DeltaCommand { eligibleEndCommitVersion: Long, relativizeIgnoreError: Option[Boolean]): Dataset[SerializableFileStatus] = { import org.apache.spark.sql.delta.implicits._ - // When coordinated commits are enabled, commit files could be found in _delta_log directory - // as well as in commit directory. We get the delta log files outside of the retention window - // from both the places. - val prefix = listingPrefix(deltaLog.logPath, eligibleStartCommitVersion) - val eligibleDeltaLogFilesFromDeltaLogDirectory = - deltaLog.store.listFrom(prefix, deltaLog.newDeltaHadoopConf) - .collect { case DeltaFile(f, deltaFileVersion) => (f, deltaFileVersion) } - .takeWhile(_._2 <= eligibleEndCommitVersion) - .toSeq - - val fs = deltaLog.logPath.getFileSystem(deltaLog.newDeltaHadoopConf()) - val commitDirPath = FileNames.commitDirPath(deltaLog.logPath) - val updatedStartCommitVersion = - eligibleDeltaLogFilesFromDeltaLogDirectory.lastOption.map(_._2) - .getOrElse(eligibleStartCommitVersion) - val eligibleDeltaLogFilesFromCommitDirectory = if (fs.exists(commitDirPath)) { - deltaLog.store - .listFrom(listingPrefix(commitDirPath, updatedStartCommitVersion), - deltaLog.newDeltaHadoopConf) - .collect { case UnbackfilledDeltaFile(f, deltaFileVersion, _) => (f, deltaFileVersion) } - .takeWhile(_._2 <= eligibleEndCommitVersion) - .toSeq - } else { - Seq.empty - } - - val allDeltaLogFilesOutsideTheRetentionWindow = eligibleDeltaLogFilesFromDeltaLogDirectory ++ - eligibleDeltaLogFilesFromCommitDirectory - val deltaLogFileIndex = DeltaLogFileIndex(DeltaLogFileIndex.COMMIT_FILE_FORMAT, - allDeltaLogFilesOutsideTheRetentionWindow.map(_._1)).get - - val allActions = deltaLog.loadIndex(deltaLogFileIndex).as[SingleAction] + val allActions = new DeltaHistoryManager(deltaLog) + .getAllStagedAndCommittedFileActions(eligibleStartCommitVersion, eligibleEndCommitVersion) val nonCDFFiles = allActions .where("remove IS NOT NULL") .select(col("remove") diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/commands/WriteIntoDelta.scala b/spark/src/main/scala/org/apache/spark/sql/delta/commands/WriteIntoDelta.scala index 0f71bbb9687..234b91fce8b 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/commands/WriteIntoDelta.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/commands/WriteIntoDelta.scala @@ -158,6 +158,12 @@ case class WriteIntoDelta( clusterBySpecOpt } val rearrangeOnly = options.rearrangeOnly + if (rearrangeOnly) { + DeltaErrors.checkCatalogManagedTableOperationAllowed( + CatalogManagedTableMaintenanceOperation.DATA_REORGANIZATION, + txn.snapshot, + txn.catalogTable) + } val charPadding = sparkSession.conf.get(SQLConf.READ_SIDE_CHAR_PADDING) val charAsVarchar = sparkSession.conf.get(SQLConf.CHAR_AS_VARCHAR) val dataSchema = if (!charAsVarchar && charPadding) { @@ -382,10 +388,9 @@ case class WriteIntoDelta( if (!useDynamicPartitionOverwriteMode && options.useNullIntolerantEqualityWithDPO.isDefined) { - throw DeltaErrors.illegalDeltaOptionException( + throw DeltaErrors.illegalDeltaOptionDynamicPartitionOverwriteOnly( name = DeltaOptions.USE_NULL_INTOLERANT_EQUALITY_WITH_DPO, - input = options.useNullIntolerantEqualityWithDPO.get.toString, - explain = "This option should be specified only in Dynamic Partition Overwrite mode.") + input = options.useNullIntolerantEqualityWithDPO.get.toString) } val deletedFiles = if (useDynamicPartitionOverwriteMode) { diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/commands/backfill/RowTrackingBackfillBatch.scala b/spark/src/main/scala/org/apache/spark/sql/delta/commands/backfill/RowTrackingBackfillBatch.scala index 8ff03bc2196..7756c8aaacd 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/commands/backfill/RowTrackingBackfillBatch.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/commands/backfill/RowTrackingBackfillBatch.scala @@ -16,7 +16,7 @@ package org.apache.spark.sql.delta.commands.backfill -import org.apache.spark.sql.delta.{DeltaConfigs, DeltaOperations, OptimisticTransaction, RowTrackingFeature} +import org.apache.spark.sql.delta.{DeltaConfigs, DeltaErrors, DeltaOperations, OptimisticTransaction, RowTrackingFeature} import org.apache.spark.sql.delta.actions.AddFile import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util.{Utils => DeltaUtils} @@ -39,15 +39,15 @@ case class RowTrackingBackfillBatch(filesInBatch: Seq[AddFile]) extends Backfill val ignoreSuspension = DeltaUtils.isTesting && spark.conf.get(ignoreProperty).toBoolean val suspendRowTracking = DeltaConfigs.ROW_TRACKING_SUSPENDED.fromMetaData(metadata) && !ignoreSuspension - if (!isRowTrackingSupported || suspendRowTracking) { - throw new IllegalStateException( - """ - |Cannot run backfill command if row tracking is not supported or - |row ID generation is suspended.""".stripMargin) + if (!isRowTrackingSupported) { + throw DeltaErrors.protocolChangedException(None) + } + if (suspendRowTracking) { + throw DeltaErrors.rowTrackingIllegalPropertyCombination() } val filesToCommit = filesInBatch.map(_.copy(dataChange = false)) - // Base Row IDs are added as part of the OptimisticTransaction.prepareCommit(), so we don't + // Base Row IDs are added as part of OptimisticTransaction.prepareInitialActions(), so we don't // need to do anything here other than recommit the files. // Note: A backfill commit can be empty ,i.e. have no file actions, at commit time due to // files being removed by concurrent conflict resolution. diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/commands/cdc/CDCReader.scala b/spark/src/main/scala/org/apache/spark/sql/delta/commands/cdc/CDCReader.scala index c22315bf131..e67f9c072ba 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/commands/cdc/CDCReader.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/commands/cdc/CDCReader.scala @@ -324,6 +324,7 @@ trait CDCReaderImpl extends CDCReaderBase { .flatMap(_.inCommitTimestamp) .map(ict => new Timestamp(ict)) .getOrElse(nonICTTimestampsByVersion.get(v).orNull) + val changesDataFunc = CommitInfo.fileActionChangesData(commitInfo) // When `isStreaming` = `true` the [CommitInfo] action is only used for passing the // in-commit timestamp to this method. We should filter them out. commitInfo = if (isStreaming) None else commitInfo @@ -344,8 +345,8 @@ trait CDCReaderImpl extends CDCReaderBase { } else { // Otherwise, we take the AddFile and RemoveFile actions with dataChange = true and // infer CDC from them. - val addActions = actions.collect { case a: AddFile if a.dataChange => a } - val removeActions = actions.collect { case r: RemoveFile if r.dataChange => r } + val addActions = actions.collect { case a: AddFile if changesDataFunc(a) => a } + val removeActions = actions.collect { case r: RemoveFile if changesDataFunc(r) => r } numAddFiles += addActions.size numRemoveFiles += removeActions.size totalBytes += addActions.map(_.size).sum @@ -669,7 +670,7 @@ trait CDCReaderImpl extends CDCReaderBase { // but CDCReader use CaseInsensitiveStringMap vs. CaseInsensitiveMap used by DataFrameReader. def toBoolean(input: String, name: String): Boolean = { Try(input.toBoolean).toOption.getOrElse { - throw DeltaErrors.illegalDeltaOptionException(name, input, "must be 'true' or 'false'") + throw DeltaErrors.illegalDeltaOptionMustBeBoolean(name, input) } } diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/commands/cdc/CDCReaderBase.scala b/spark/src/main/scala/org/apache/spark/sql/delta/commands/cdc/CDCReaderBase.scala index 542310de0da..a91d8a5074b 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/commands/cdc/CDCReaderBase.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/commands/cdc/CDCReaderBase.scala @@ -277,11 +277,9 @@ trait CDCReaderBase extends DeltaLogging { // Non-legacy schema mode options cannot be used with time-travel because the schema to use // will be confusing. if (isTimeTravelQuery && schemaMode != BatchCDFSchemaLegacy) { - throw DeltaErrors.illegalDeltaOptionException( + throw DeltaErrors.illegalDeltaOptionSchemaModeWithTimeTravel( DeltaSQLConf.DELTA_CDF_DEFAULT_SCHEMA_MODE_FOR_COLUMN_MAPPING_TABLE.key, - schemaMode.name, - s"${DeltaSQLConf.DELTA_CDF_DEFAULT_SCHEMA_MODE_FOR_COLUMN_MAPPING_TABLE.key} " + - s"cannot be used with time travel options.") + schemaMode.name) } getConstructedCDCRelation( diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/commands/columnmapping/RemoveColumnMappingCommand.scala b/spark/src/main/scala/org/apache/spark/sql/delta/commands/columnmapping/RemoveColumnMappingCommand.scala index 845d6f111e0..2a47967af28 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/commands/columnmapping/RemoveColumnMappingCommand.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/commands/columnmapping/RemoveColumnMappingCommand.scala @@ -41,6 +41,11 @@ class RemoveColumnMappingCommand( */ def run(spark: SparkSession, removeColumnMappingTableProperty: Boolean): Unit = { deltaLog.withNewTransaction(catalogOpt) { txn => + DeltaErrors.checkCatalogManagedTableOperationAllowed( + CatalogManagedTableMaintenanceOperation.DATA_REORGANIZATION, + txn.snapshot, + txn.catalogTable) + val originalFiles = txn.filterFiles() val originalData = buildDataFrame(txn, originalFiles) val originalSchema = txn.snapshot.schema diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/commands/convert/ParquetFileManifest.scala b/spark/src/main/scala/org/apache/spark/sql/delta/commands/convert/ParquetFileManifest.scala index 02e8005ff58..ee76437de40 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/commands/convert/ParquetFileManifest.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/commands/convert/ParquetFileManifest.scala @@ -26,6 +26,7 @@ import org.apache.hadoop.fs.Path import org.apache.spark.sql.{Dataset, SparkSession} import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.execution.datasources.parquet.{ParquetFileFormat, ParquetToSparkSchemaConverter} +import org.apache.spark.sql.functions.col import org.apache.spark.sql.types.StructType import org.apache.spark.util.SerializableConfiguration @@ -45,12 +46,32 @@ class ManualListingFileManifest( .where("!isDir") } + // Set when rebalancing caches the raw listing (see allFiles); released in close(). + private var listingToUnpersist: Option[Dataset[SerializableFileStatus]] = None + override lazy val allFiles: Dataset[ConvertTargetFile] = { import org.apache.spark.sql.delta.implicits._ val conf = spark.sparkContext.broadcast(serializableConf) val fetchConfig = parquetSchemaFetchConfig - val files = doList().mapPartitions { iter => + // recursiveListDirs parcels files by top-level directory, so a large partition dir becomes a + // single skewed task reading all its footers. Range-partition by path before reading footers: + // this spreads the footer reads across tasks (by file count) and processes them in path order, + // so schema merging (first-appearance column order) is deterministic. + val listed = doList() + val balanced = + if (spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_CONVERT_REBALANCE_FILE_LISTING)) { + // Cache the listing so repartitionByRange's range-bound sampling does not re-run the + // (potentially expensive) recursive directory listing. + val cachedListing = listed.cache() + listingToUnpersist = Some(cachedListing) + cachedListing + .repartitionByRange(spark.sparkContext.defaultParallelism, col("path")) + .sortWithinPartitions(col("path")) + } else { + listed + } + val files = balanced.mapPartitions { iter => val fileStatuses = iter.toSeq val pathToStatusMapping = fileStatuses.map { fileStatus => fileStatus.path -> fileStatus @@ -70,11 +91,16 @@ class ManualListingFileManifest( override lazy val parquetSchema: Option[StructType] = { recordDeltaOperationForTablePath(basePath, "delta.convert.schemaInference") { + // When rebalancing is on, allFiles is already path-ordered (range-partitioned + sorted by + // path), so schema merging is deterministic without a separate sort here. Some(ConvertUtils.mergeSchemasInParallel(spark, partitionSchema, allFiles)) } } - override def close(): Unit = allFiles.unpersist() + override def close(): Unit = { + allFiles.unpersist() + listingToUnpersist.foreach(_.unpersist()) + } } /** A file manifest generated through listing partition paths from Metastore catalog. */ diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/commands/merge/MergeIntoMaterializeSource.scala b/spark/src/main/scala/org/apache/spark/sql/delta/commands/merge/MergeIntoMaterializeSource.scala index 9296c2e7f8a..b81c054c5ca 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/commands/merge/MergeIntoMaterializeSource.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/commands/merge/MergeIntoMaterializeSource.scala @@ -191,7 +191,8 @@ trait MergeIntoMaterializeSource extends DeltaLogging with DeltaSparkPlanUtils { case s: SparkException if materializedSourceRDD.nonEmpty && s.getErrorClass == "CHECKPOINT_RDD_BLOCK_ID_NOT_FOUND" && - s.getMessageParameters.get("rddBlockId").contains(s"rdd_${materializedSourceRDD.get.id}") => + Option(s.getMessageParameters.get("rddBlockId")) + .exists(_.contains(s"rdd_${materializedSourceRDD.get.id}")) => logWarning(log"Materialized ${MDC(DeltaLogKeys.OPERATION, operation)} source RDD block " + log"lost. ${MDC(DeltaLogKeys.OPERATION, operation)} needs to be restarted. " + log"This was attempt number ${MDC(DeltaLogKeys.ATTEMPT, attempt)}.") diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/coordinatedcommits/CoordinatedCommitsUtils.scala b/spark/src/main/scala/org/apache/spark/sql/delta/coordinatedcommits/CoordinatedCommitsUtils.scala index 7789efc5160..1af2f90a35a 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/coordinatedcommits/CoordinatedCommitsUtils.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/coordinatedcommits/CoordinatedCommitsUtils.scala @@ -28,7 +28,7 @@ import org.apache.spark.sql.delta.commands.CloneTableCommand import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.sources.DeltaSQLConf -import org.apache.spark.sql.delta.util.{FileNames, JsonUtils} +import org.apache.spark.sql.delta.util.{DeltaCommitFileProvider, FileNames, JsonUtils} import org.apache.spark.sql.delta.util.FileNames.{BackfilledDeltaFile, CompactedDeltaFile, DeltaFile, UnbackfilledDeltaFile} import io.delta.storage.LogStore import io.delta.storage.commit.{CommitCoordinatorClient, GetCommitsResponse => JGetCommitsResponse, TableIdentifier} @@ -68,11 +68,11 @@ object CatalogOwnedTableUtils extends DeltaLogging { txn.updateProtocol(protocol = p) // Force a metadata update to trigger ICT (In-Commit Timestamp) enablement. // CatalogOwnedTableFeature requires ICT to be enabled in the metadata, but - // updateMetadataAndProtocolWithRequiredFeatures in prepareCommit only runs + // updateMetadataAndProtocolWithRequiredFeatures in prepareInitialActions only runs // when there's an explicit metadata change (metadataChanges.headOption is non-empty). // Since we're only updating the protocol here without changing metadata content, // we need to explicitly call updateMetadata to ensure metadataChanges is non-empty - // during prepareCommit, which will then enable ICT in the metadata. + // during prepareInitialActions, which will then enable ICT in the metadata. // Without this, generateInCommitTimestampForFirstCommitAttempt would return None, // causing UCCommitCoordinatorClient to fail with DELTA_MISSING_COMMIT_TIMESTAMP. txn.updateMetadata(proposedNewMetadata = txn.metadata) @@ -540,6 +540,38 @@ object CoordinatedCommitsUtils extends DeltaLogging { } } + /** + * Ensures that commit files are backfilled up to the current version in the snapshot. + * + * This method checks if there are any un-backfilled versions up to the current version and + * triggers the backfilling process using the commit-coordinator. It verifies that the delta file + * for the current version exists after the backfilling process. + * + * @throws IllegalStateException + * if the delta file for the current version is not found after backfilling. + */ + def ensureCommitFilesBackfilled( + version: Long, + deltaLog: DeltaLog, + tableCommitCoordinatorClient: TableCommitCoordinatorClient, + deltaCommitFileProvider: DeltaCommitFileProvider, + catalogTableOpt: Option[CatalogTable]): Unit = { + val minUnbackfilledVersion = deltaCommitFileProvider.minUnbackfilledVersion + if (minUnbackfilledVersion <= version) { + val hadoopConf = deltaLog.newDeltaHadoopConf() + tableCommitCoordinatorClient.backfillToVersion( + catalogTableOpt.map(_.identifier), + version, + lastKnownBackfilledVersion = Some(minUnbackfilledVersion - 1)) + val fs = deltaLog.logPath.getFileSystem(hadoopConf) + val expectedBackfilledDeltaFile = FileNames.unsafeDeltaFile(deltaLog.logPath, version) + if (!fs.exists(expectedBackfilledDeltaFile)) { + throw new IllegalStateException("Backfilling of commit files failed. " + + s"Expected delta file $expectedBackfilledDeltaFile not found.") + } + } + } + /** * Returns an iterator of commit files starting from startVersion. * If the iterator is consumed beyond what the file system listing shows, this method do a @@ -738,10 +770,8 @@ object CoordinatedCommitsUtils extends DeltaLogging { * Returns true if the snapshot is backed by unbackfilled commits. */ def unbackfilledCommitsPresent(snapshot: Snapshot): Boolean = { - snapshot.logSegment.deltas.exists { - case FileNames.UnbackfilledDeltaFile(_, _, _) => true - case _ => false - } && !snapshot.allCommitsBackfilled + // The unbackfilled commits from the segment might be stale, so we consult the snapshot as well. + snapshot.logSegment.unbackfilledDeltas.nonEmpty && !snapshot.allCommitsBackfilled } /** @@ -758,9 +788,8 @@ object CoordinatedCommitsUtils extends DeltaLogging { // as backfills are managed by commit-coordinators. return } - val unbackfilledFilesAndVersions = snapshot.logSegment.deltas.collect { - case UnbackfilledDeltaFile(unbackfilledDeltaFile, version, _) => - (unbackfilledDeltaFile, version) + val unbackfilledFilesAndVersions = snapshot.logSegment.unbackfilledDeltas.map { + case (file, version, _) => (file, version) } if (unbackfilledFilesAndVersions.isEmpty) return // Coordinated commits are disabled on the table but the table still has un-backfilled files. diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/deletionvectors/ManifestBitmap.scala b/spark/src/main/scala/org/apache/spark/sql/delta/deletionvectors/ManifestBitmap.scala new file mode 100644 index 00000000000..0c0d1002d9d --- /dev/null +++ b/spark/src/main/scala/org/apache/spark/sql/delta/deletionvectors/ManifestBitmap.scala @@ -0,0 +1,147 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.deletionvectors + +import scala.collection.Seq + +import org.apache.spark.sql.delta.RowIndexFilterProvider +import org.apache.hadoop.fs.Path + +import org.apache.spark.util.Utils + +/** + * An immutable bitmap of unsigned positions for the AMT manifest deletion vectors: the masking + * MDV in `manifest_info.dv` and the CDF `tracking.deleted_positions` / + * `tracking.replaced_positions` bitmaps. + */ +trait ManifestBitmap { + /** Returns `true` if `value` is set. */ + def contains(value: Int): Boolean + + /** Returns the number of set positions. */ + def cardinality: Long + + /** Returns `true` if no position is set. */ + def isEmpty: Boolean + + /** Returns a mutable copy that can be updated and converted back to an immutable bitmap. */ + def toMutable: MutableManifestBitmap + + /** Serializes to the on-disk byte form carried in `manifest_info.dv` / the CDF bitmaps. */ + def serializeAsByteArray(): Array[Byte] + + /** The set positions in ascending order, for tests only. */ + final def toArrayForTesting: Array[Long] = { + assert(Utils.isTesting) + toArrayForTestingImpl + } + + protected def toArrayForTestingImpl: Array[Long] + + /** Builds a [[RowIndexFilterProvider]] representing this bitmap. */ + def toRowIndexFilterProvider(tableRoot: Path): RowIndexFilterProvider +} + +/** A mutable copy of a [[ManifestBitmap]]. */ +trait MutableManifestBitmap { + /** Adds `value` to the bitmap. */ + def add(value: Int): Unit + + /** Returns an immutable snapshot of the current bitmap. */ + def toManifestBitmap: ManifestBitmap +} + +object ManifestBitmap { + /** Returns a bitmap of the given positions. */ + def fromPositions(values: Seq[Int]): ManifestBitmap = + RoaringManifestBitmap.fromPositions(values) + + /** Deserializes a bitmap previously produced by [[ManifestBitmap.serializeAsByteArray]]. */ + def fromSerializedByteArray(bytes: Array[Byte]): ManifestBitmap = + RoaringManifestBitmap.fromSerializedByteArray(bytes) +} + +/** + * A [[ManifestBitmap]] backed by a [[RoaringBitmapArray]] in the portable format. + */ +final class RoaringManifestBitmap private ( + private val serializedBytesOrPositions: Either[Array[Byte], Vector[Int]]) + extends ManifestBitmap { + + // Materialized on first bit-level use: decoded from the on-disk bytes, or built from the + // positions. + private lazy val roaringBitmapArray: RoaringBitmapArray = + serializedBytesOrPositions match { + case Left(bytes) => RoaringBitmapArray.readFrom(bytes) + case Right(positions) => RoaringBitmapArray(positions.map(_.toLong): _*) + } + + override def contains(value: Int): Boolean = roaringBitmapArray.contains(value.toLong) + + override def cardinality: Long = roaringBitmapArray.cardinality + + override def isEmpty: Boolean = roaringBitmapArray.isEmpty + + override def toMutable: MutableManifestBitmap = + MutableRoaringManifestBitmap.fromSerializedByteArray(serializeAsByteArray()) + + override protected def toArrayForTestingImpl: Array[Long] = roaringBitmapArray.toArray + + override def serializeAsByteArray(): Array[Byte] = + serializedBytesOrPositions match { + // Defensive copy: Don't expose internal buffer for a caller to mutate. + case Left(bytes) => bytes.clone() + case Right(_) => roaringBitmapArray.serializeAsByteArray(RoaringBitmapArrayFormat.Portable) + } + + override def toRowIndexFilterProvider(tableRoot: Path): RowIndexFilterProvider = { + // Capture serialized bytes so the scan task owns a self-contained provider. + val bytes = serializeAsByteArray() + RoaringManifestBitmap.Provider(bytes) + } +} + +object RoaringManifestBitmap { + private final case class Provider(serializedBytes: Array[Byte]) + extends RowIndexFilterProvider { + override def retrieve( + _hadoopConf: org.apache.hadoop.conf.Configuration): DropMarkedRowsFilter = + new DropMarkedRowsFilter(RoaringBitmapArray.readFrom(serializedBytes)) + } + + def fromPositions(values: Seq[Int]): RoaringManifestBitmap = + new RoaringManifestBitmap(Right(values.toVector)) + + def fromSerializedByteArray(serializedBytes: Array[Byte]): RoaringManifestBitmap = + new RoaringManifestBitmap(Left(serializedBytes.clone())) +} + +private final class MutableRoaringManifestBitmap private ( + private val roaringBitmapArray: RoaringBitmapArray) + extends MutableManifestBitmap { + + override def add(value: Int): Unit = roaringBitmapArray.add(value.toLong) + + override def toManifestBitmap: ManifestBitmap = + RoaringManifestBitmap.fromSerializedByteArray( + roaringBitmapArray.serializeAsByteArray(RoaringBitmapArrayFormat.Portable)) +} + +private object MutableRoaringManifestBitmap { + def fromSerializedByteArray(serializedBytes: Array[Byte]): MutableRoaringManifestBitmap = + new MutableRoaringManifestBitmap(RoaringBitmapArray.readFrom(serializedBytes)) +} diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/deletionvectors/MutableMumblingBitmap.scala b/spark/src/main/scala/org/apache/spark/sql/delta/deletionvectors/MutableMumblingBitmap.scala new file mode 100644 index 00000000000..06ef0dc07ac --- /dev/null +++ b/spark/src/main/scala/org/apache/spark/sql/delta/deletionvectors/MutableMumblingBitmap.scala @@ -0,0 +1,76 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.deletionvectors + +import scala.collection.mutable + +import org.apache.spark.sql.delta.deletionvectors.mumbling.{MumblingBitmap, MumblingBitmapWriter} + +/** + * A mutable Mumbling bitmap that retains an immutable base bitmap and records additions + * separately. + * + * Creating an immutable snapshot merges the overlay one container at a time, without materializing + * the existing set positions. + */ +final class MutableMumblingBitmap(private val base: MumblingBitmap) { + + private val additionContainers = mutable.LongMap.empty[Array[Byte]] + private var additionCardinality = 0 + private var maxAdditionContainer = -1 + + /** Adds `value` to the bitmap. */ + def add(value: Long): Unit = { + require( + value >= 0 && value < MumblingBitmapWriter.MAX_POSITION_EXCLUSIVE, + s"Position out of range for a Mumbling bitmap: $value " + + s"(must be in [0, ${MumblingBitmapWriter.MAX_POSITION_EXCLUSIVE}))") + val position = value.toInt + val containerIndex = position >>> 8 + val block = additionContainers.getOrElseUpdate( + containerIndex.toLong, new Array[Byte](MutableMumblingBitmap.ContainerBytes)) + val positionInContainer = position & 0xFF + val byteIndex = positionInContainer >>> 3 + val mask = 1 << (7 - (positionInContainer & 0x7)) + if ((block(byteIndex) & mask) == 0) { + block(byteIndex) = (block(byteIndex) | mask).toByte + additionCardinality += 1 + maxAdditionContainer = math.max(maxAdditionContainer, containerIndex) + } + } + + private def additionContainerBlocks: Array[Array[Byte]] = { + val blocks = new Array[Array[Byte]](maxAdditionContainer + 1) + additionContainers.foreach { case (containerIndex, block) => + blocks(containerIndex.toInt) = block + } + blocks + } + + /** Returns an immutable snapshot of the current bitmap. */ + def toMumblingBitmap: MumblingBitmap = { + if (additionCardinality == 0) { + base + } else { + MumblingBitmapWriter.merge(base, additionContainerBlocks) + } + } +} + +object MutableMumblingBitmap { + private val ContainerBytes = 32 +} diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/files/DeltaSourceSnapshot.scala b/spark/src/main/scala/org/apache/spark/sql/delta/files/DeltaSourceSnapshot.scala index af112229250..d2277838f55 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/files/DeltaSourceSnapshot.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/files/DeltaSourceSnapshot.scala @@ -18,7 +18,7 @@ package org.apache.spark.sql.delta.files import scala.collection.JavaConverters._ -import org.apache.spark.sql.delta.{DeltaLog, DeltaTableUtils, Snapshot} +import org.apache.spark.sql.delta.{DeltaLogUtils, DeltaTableUtils, Snapshot} import org.apache.spark.sql.delta.actions.SingleAction import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.sources.IndexedFile @@ -77,7 +77,7 @@ class DeltaSourceSnapshot( .withColumn("isLast", lit(false)) .withColumn("shouldSkip", lit(false)) - DeltaLog.filterFileList( + DeltaLogUtils.filterFileList( snapshot.metadata.partitionSchema, initialFiles, partitionFilters, diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/files/TahoeFileIndex.scala b/spark/src/main/scala/org/apache/spark/sql/delta/files/TahoeFileIndex.scala index 8a66ef58b1f..fccc678726f 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/files/TahoeFileIndex.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/files/TahoeFileIndex.scala @@ -90,7 +90,7 @@ abstract class TahoeFileIndex( * set by the partition and data filters. Essentially, this is a collection of all the files * associated with the identified partitions. */ - def listPartitionsAsAddFiles( + private[delta] def listPartitionsAsAddFiles( partitionFilters: Seq[Expression], dataFilters: Seq[Expression]): (Seq[(InternalRow, Seq[AddFile])], Seq[AddFile]) = { val matchedFiles = matchingFiles(partitionFilters, dataFilters) @@ -142,7 +142,7 @@ abstract class TahoeFileIndex( FileStatusWithMetadata(fs, metadata.toMap) } - def makePartitionDirectories( + private[delta] def makePartitionDirectories( partitionValuesToFiles: Seq[(InternalRow, Seq[AddFile])]): Seq[PartitionDirectory] = { val timeZone = spark.sessionState.conf.sessionLocalTimeZone partitionValuesToFiles.map { diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/hooks/AutoCompact.scala b/spark/src/main/scala/org/apache/spark/sql/delta/hooks/AutoCompact.scala index c978c9396ee..05f50e50665 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/hooks/AutoCompact.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/hooks/AutoCompact.scala @@ -98,6 +98,15 @@ trait AutoCompactBase extends PostCommitHook with DeltaLogging { } override def run(spark: SparkSession, txn: CommittedTransaction): Unit = { + try { + DeltaErrors.checkCatalogManagedTableOperationAllowed( + CatalogManagedTableMaintenanceOperation.DATA_REORGANIZATION, + txn.postCommitSnapshot, + txn.catalogTable) + } catch { + case _: DeltaUnsupportedOperationException => return + } + val conf = spark.sessionState.conf val autoCompactTypeOpt = getAutoCompactType(conf, txn.postCommitSnapshot.metadata) // Skip Auto Compact if current transaction is not qualified or the table is not qualified diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/hooks/CheckpointHook.scala b/spark/src/main/scala/org/apache/spark/sql/delta/hooks/CheckpointHook.scala index b37dbd4539b..4be0b8ee405 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/hooks/CheckpointHook.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/hooks/CheckpointHook.scala @@ -16,7 +16,8 @@ package org.apache.spark.sql.delta.hooks -import org.apache.spark.sql.delta.{AdaptiveMetadataTableFeature, CommittedTransaction, DeltaOperations} +import org.apache.spark.sql.delta.CommittedTransaction +import org.apache.spark.sql.delta.amt.AMTUtils import org.apache.spark.sql.SparkSession @@ -27,37 +28,23 @@ object CheckpointHook extends PostCommitHook { override def run(spark: SparkSession, txn: CommittedTransaction): Unit = { if (!txn.maintenanceOperation.shouldCheckpoint) return - // AMT tables checkpoint by rewriting their manifest tree via a follow-up OPTIMIZE CHECKPOINT - // commit rather than a standalone checkpoint file. - if (txn.postCommitSnapshot.protocol.isFeatureSupported(AdaptiveMetadataTableFeature)) { - val triggerMode = txn.maintenanceOperation.amtTriggerModeOpt.getOrElse { - throw new IllegalStateException( - "An AMT table scheduled a checkpoint but carries no AMTTriggerMode.") + val snapshotToCheckpoint = + if (AMTUtils.amtEnabled(txn.postCommitSnapshot)) { + txn.postCommitSnapshot + } else { + // Two txns writing multi-part checkpoints at the same version has the risk of corruption. + // So make sure the txn write the checkpoint only for the version it created. + val cp = txn.postCommitSnapshot.checkpointProvider + txn.deltaLog.getSnapshotAt( + txn.committedVersion, + lastCheckpointHint = None, + lastCheckpointProvider = Some(cp), + catalogTableOpt = txn.catalogTable, + enforceTimeTravelWithinDeletedFileRetention = false) } - writeAMTCheckpoint( - txn, incremental = triggerMode.isIncremental, triggerName = triggerMode.name) - return - } - - // Since the postCommitSnapshot isn't guaranteed to match committedVersion, we have to - // explicitly checkpoint the snapshot at the committedVersion. - val cp = txn.postCommitSnapshot.checkpointProvider - val snapshotToCheckpoint = txn.deltaLog.getSnapshotAt( - txn.committedVersion, - lastCheckpointHint = None, - lastCheckpointProvider = Some(cp), - catalogTableOpt = txn.catalogTable, - enforceTimeTravelWithinDeletedFileRetention = false) - txn.deltaLog.checkpoint(snapshotToCheckpoint, txn.catalogTable) - } - - /** - * Attempts a Manifest Commit on the table. - */ - private def writeAMTCheckpoint( - txn: CommittedTransaction, incremental: Boolean, triggerName: String): Unit = { - val checkpointTxn = - txn.deltaLog.startTransaction(txn.catalogTable, Some(txn.postCommitSnapshot)) - checkpointTxn.commit(Seq.empty, DeltaOperations.OptimizeCheckpoint(incremental, triggerName)) + txn.deltaLog.checkpoint( + snapshotToCheckpoint, + txn.catalogTable, + amtTriggerModeOpt = txn.maintenanceOperation.amtTriggerModeOpt) } } diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/hooks/GenerateSymlinkManifest.scala b/spark/src/main/scala/org/apache/spark/sql/delta/hooks/GenerateSymlinkManifest.scala index 74d7ec45ca6..ce093b64c12 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/hooks/GenerateSymlinkManifest.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/hooks/GenerateSymlinkManifest.scala @@ -72,6 +72,15 @@ trait GenerateSymlinkManifestImpl extends PostCommitHook with DeltaLogging with override val name: String = "Generate Symlink Format Manifest" override def run(spark: SparkSession, txn: CommittedTransaction): Unit = { + try { + DeltaErrors.checkCatalogManagedTableOperationAllowed( + "GENERATE_SYMLINK_FORMAT_MANIFEST", + txn.postCommitSnapshot, + txn.catalogTable) + } catch { + case _: DeltaUnsupportedOperationException => return + } + generateIncrementalManifest(spark, txn, txn.postCommitSnapshot) } @@ -183,6 +192,10 @@ trait GenerateSymlinkManifestImpl extends PostCommitHook with DeltaLogging with deltaLog: DeltaLog, catalogTableOpt: Option[CatalogTable]): Unit = { val snapshot = deltaLog.update(catalogTableOpt = catalogTableOpt) + DeltaErrors.checkCatalogManagedTableOperationAllowed( + "GENERATE_SYMLINK_FORMAT_MANIFEST", + snapshot, + catalogTableOpt) assertTableIsDVFree(spark, snapshot) generateFullManifestWithSnapshot(spark, deltaLog, snapshot) } diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/hooks/IcebergConverterHook.scala b/spark/src/main/scala/org/apache/spark/sql/delta/hooks/IcebergConverterHook.scala index 32e933a4338..ebfc8c19caa 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/hooks/IcebergConverterHook.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/hooks/IcebergConverterHook.scala @@ -19,7 +19,6 @@ package org.apache.spark.sql.delta.hooks import org.apache.spark.sql.delta.{CommittedTransaction, DeltaErrors, UniversalFormat, UniversalFormatConverter} import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.sources.DeltaSQLConf.DELTA_UNIFORM_ICEBERG_SYNC_CONVERT_ENABLED -import org.apache.commons.lang3.exception.ExceptionUtils import org.apache.spark.sql.SparkSession @@ -48,7 +47,7 @@ trait IcebergConverterHook extends PostCommitHook with DeltaLogging { override def handleError(spark: SparkSession, error: Throwable, version: Long): Unit = { logError(error.getMessage, error) throw DeltaErrors.universalFormatConversionFailedException( - version, "iceberg", ExceptionUtils.getMessage(error)) + version, "iceberg", error) } def triggerIcebergConversion( diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/hooks/UpdateCatalog.scala b/spark/src/main/scala/org/apache/spark/sql/delta/hooks/UpdateCatalog.scala index aaacc60aec5..338d52e39ed 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/hooks/UpdateCatalog.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/hooks/UpdateCatalog.scala @@ -358,8 +358,17 @@ object UpdateCatalog { catalog.externalCatalog.alterTableDataSchema(db, tblName, schema) } - // We have to update the properties anyway with the latest version/timestamp information - catalog.alterTable(table.copy(properties = updatedProperties(snapshot) ++ additionalProperties)) + // We have to update the properties anyway with the latest version/timestamp information. + // If RETAIN_COMMENTS_DURING_REPLACE_TABLE is enabled, replace table may carry over table + // comments. UC has to be aware of this. + val updatedTable = table.copy( + properties = updatedProperties(snapshot) ++ additionalProperties) + catalog.alterTable( + if (spark.conf.get(DeltaSQLConf.RETAIN_COMMENTS_DURING_REPLACE_TABLE)) { + updatedTable.copy(comment = Option(snapshot.metadata.description)) + } else { + updatedTable + }) } /** Updates our properties map with the version and timestamp information of the snapshot. */ diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/metering/DeltaLogging.scala b/spark/src/main/scala/org/apache/spark/sql/delta/metering/DeltaLogging.scala index f13cf62b0c3..fe139839960 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/metering/DeltaLogging.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/metering/DeltaLogging.scala @@ -16,6 +16,8 @@ package org.apache.spark.sql.delta.metering +import java.util.function.Supplier + import scala.concurrent.duration._ import scala.util.Try import scala.util.control.NonFatal @@ -153,7 +155,7 @@ trait DeltaLogging check: => Boolean, name: String, msg: String, - deltaLog: DeltaLog = null, + deltaLog: DeltaLoggingProvider = null, data: AnyRef = null, path: Option[Path] = None) : Unit = { @@ -170,11 +172,80 @@ trait DeltaLogging } } + /** + * A variation of [[deltaAssert]] that only logs the assertion error and + * does not add an assertion in test. This is useful for writing negative tests + * that validates the behaviour of executing the code after the [[deltaAssert]] call. + * The assertion in test prevents us from testing the production behaviour. + */ + protected def deltaAssertLogOnly( + check: => Boolean, + name: String, + msg: String, + deltaLog: DeltaLoggingProvider = null, + data: AnyRef = null, + path: Option[Path] = None + ): Unit = { + if (!check) { + recordDeltaEvent( + provider = deltaLog, + opType = s"delta.assertions.$name", + data = data, + path = path + ) + logWarning(msg) + } + } + + /** + * A variation of [[deltaAssert]] that logs the assertion error and throws a throwable. + * We don't need an assertion in test since if it fails we want the same throwable in + * test and in production. + */ + protected def deltaAssertAndThrow( + check: => Boolean, + name: String, + msg: String, + throwable: Throwable, + deltaLog: DeltaLoggingProvider = null, + data: AnyRef = null, + path: Option[Path] = None + ): Unit = { + if (!check) { + recordDeltaEvent( + provider = deltaLog, + opType = s"delta.assertions.$name", + data = data, + path = path + ) + logError(msg) + + throw throwable + } + } + protected def recordFrameProfile[T](group: String, name: String)(thunk: => T): T = { // future work to capture runtime information ... thunk } + + /** Java adapter for a value-returning frame with fixed system-code group and name literals. */ + protected def recordFrameProfileValue[T]( + group: String, + name: String, + body: Supplier[T]): T = { + recordFrameProfile(group, name)(body.get()) + } + + /** Java adapter for a void frame with fixed system-code group and name literals. */ + protected def recordFrameProfileAction( + group: String, + name: String, + body: Runnable): Unit = { + recordFrameProfile(group, name)(body.run()) + } + private def withDmqTag[T](thunk: => T): T = { thunk } diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/schema/SchemaUtils.scala b/spark/src/main/scala/org/apache/spark/sql/delta/schema/SchemaUtils.scala index 6f4d4c0edf5..2fd01ecff30 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/schema/SchemaUtils.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/schema/SchemaUtils.scala @@ -23,7 +23,7 @@ import scala.util.control.NonFatal import org.apache.spark.sql.delta.Relocated._ import org.apache.spark.sql.delta.ClassicColumnConversions._ -import org.apache.spark.sql.delta.{DeltaAnalysisException, DeltaColumnMappingMode, DeltaErrors, DeltaLog, GeneratedColumn, NoMapping, TypeWidening, TypeWideningMode} +import org.apache.spark.sql.delta.{DataTypeChangeViolation, DeltaAnalysisException, DeltaCannotChangeDataTypeException, DeltaColumnMappingMode, DeltaErrors, DeltaLog, GeneratedColumn, NoMapping, TypeWidening, TypeWideningMode} import org.apache.spark.sql.delta.{RowCommitVersion, RowId} import org.apache.spark.sql.delta.ClassicColumnConversions._ import org.apache.spark.sql.delta.actions.Protocol @@ -1118,15 +1118,16 @@ def normalizeColumnNamesInDataType( columnMappingMode: DeltaColumnMappingMode, columnPath: Seq[String] = Nil, failOnAmbiguousChanges: Boolean = false, - allowTypeWidening: Boolean = false): Option[String] = { - def verify(cond: Boolean, err: => String): Unit = { + allowTypeWidening: Boolean = false): Option[DataTypeChangeViolation] = { + def verify(cond: Boolean, violation: => DataTypeChangeViolation): Unit = { if (!cond) { - throw DeltaErrors.cannotChangeDataType(err) + throw DeltaErrors.cannotChangeDataType(violation) } } def verifyNullability(fn: Boolean, tn: Boolean, columnPath: Seq[String]): Unit = { - verify(tn || !fn, s"tightening nullability of ${UnresolvedAttribute(columnPath).name}") + verify(tn || !fn, + DataTypeChangeViolation.TightenNullability(UnresolvedAttribute(columnPath).name)) } def check(fromDt: DataType, toDt: DataType, columnPath: Seq[String]): Unit = { @@ -1155,8 +1156,8 @@ def normalizeColumnNamesInDataType( case None => addingColumns = true verify(toField.nullable, - "adding non-nullable column " + - UnresolvedAttribute(columnPath :+ toField.name).name) + DataTypeChangeViolation.AddNonNullableColumn( + UnresolvedAttribute(columnPath :+ toField.name).name)) } } val columnName = UnresolvedAttribute(columnPath).name @@ -1165,20 +1166,21 @@ def normalizeColumnNamesInDataType( } if (columnMappingMode == NoMapping) { verify(remainingFields.isEmpty, - s"dropping column(s) [${remainingFields.map(_.name).mkString(", ")}]" + - (if (columnPath.nonEmpty) s" from $columnName" else "")) + DataTypeChangeViolation.DropColumns( + remainingFields.toSeq.map(field => + UnresolvedAttribute(columnPath :+ field.name).name))) } case (_: NullType, _) => () case (fromDataType: AtomicType, toDataType: AtomicType) if allowTypeWidening => verify(TypeWidening.isTypeChangeSupported(fromDataType, toDataType), - s"changing data type of ${UnresolvedAttribute(columnPath).name} " + - s"from $fromDataType to $toDataType") + DataTypeChangeViolation.ChangeDataType( + UnresolvedAttribute(columnPath).name, fromDataType, toDataType)) case (fromDataType, toDataType) => verify(fromDataType == toDataType, - s"changing data type of ${UnresolvedAttribute(columnPath).name} " + - s"from $fromDataType to $toDataType") + DataTypeChangeViolation.ChangeDataType( + UnresolvedAttribute(columnPath).name, fromDataType, toDataType)) } } @@ -1186,8 +1188,8 @@ def normalizeColumnNamesInDataType( check(from, to, columnPath) None } catch { - case e: AnalysisException => - Some(e.message) + case e: DeltaCannotChangeDataTypeException => + Some(e.violation) } } diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/skipping/clustering/temp/AlterTableClusterBy.scala b/spark/src/main/scala/org/apache/spark/sql/delta/skipping/clustering/temp/AlterTableClusterBy.scala index 405446d41b6..8b06cf4f3e1 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/skipping/clustering/temp/AlterTableClusterBy.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/skipping/clustering/temp/AlterTableClusterBy.scala @@ -37,3 +37,23 @@ case class AlterTableClusterBy( /** A TableChange to alter clustering columns for a table. */ case class ClusterBy(clusteringColumns: Seq[NamedReference]) extends TableChange {} + +/** + * The logical plan of the following commands: + * - ALTER TABLE ... REPLACE PARTITIONED BY WITH CLUSTER BY (col1, col2, ...) + * - ALTER TABLE ... REPLACE PARTITIONED BY WITH CLUSTER BY NONE + * + * Converts a partitioned table into a clustered table (or an unpartitioned, unclustered table + * with NONE) without rewriting data files. + */ +case class AlterTableReplacePartitionedByWithClusterBy( + table: LogicalPlan, clusterBySpec: Option[ClusterBySpec]) extends AlterTableCommand { + override def changes: Seq[TableChange] = + Seq(ReplacePartitionedByWithClusterBy(clusterBySpec.map(_.columnNames).getOrElse(Seq.empty))) + + protected def withNewChildInternal(newChild: LogicalPlan): LogicalPlan = copy(table = newChild) +} + +/** A TableChange to replace the partitioning of a table with the given clustering columns. */ +case class ReplacePartitionedByWithClusterBy(clusteringColumns: Seq[NamedReference]) + extends TableChange {} diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala b/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala index c2b62fc6976..ceed56b9cc3 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala @@ -93,6 +93,14 @@ trait DeltaSQLConfBase extends DeltaSQLConfUtils { .booleanConf .createOptional + val TEST_BARRIER_ENABLED = + buildConf("testBarrier.enabled") + .internal() + .doc("If true, tests are allowed to use TestBarrier via DeltaTestBarrier. " + + "This allows test to pause and release a Delta code path deterministically.") + .booleanConf + .createWithDefault(false) + val DELTA_COLLECT_STATS = buildConf("stats.collect") .internal() @@ -200,6 +208,24 @@ trait DeltaSQLConfBase extends DeltaSQLConfUtils { .checkValue(n => n >= 0, "must not be negative.") .createWithDefault(2) + val DELTA_SNAPSHOT_FILESYSTEM_LISTING_FILTER_STAGED_COMMITS_ENABLED = + buildConf("snapshot.filesystemListing.filterStagedCommits.enabled") + .internal() + .doc("When true, raw filesystem listings accept only backfilled Delta commit files.") + .booleanConf + .createWithDefault(true) + + val DELTA_COMMIT_INCONSISTENT_LIST_MAX_RETRIES = + buildConf("commit.inconsistentList.maxRetries") + .internal() + .doc("How many times to retry fetching the log segment after a commit when the listing " + + "returns a version lower than the committed version. The listing can be stale for a " + + "few seconds after a commit (in case of list-after-write storage inconsistency), and " + + "each retry waits with exponential backoff, capped at 30 seconds, before re-listing.") + .intConf + .checkValue(n => n >= 0 && n < 10, "must be between 0 (inclusive) and 10 (exclusive).") + .createWithDefault(3) + val DELTA_SNAPSHOT_CACHE_STORAGE_LEVEL = buildConf("snapshotCache.storageLevel") .internal() @@ -217,6 +243,17 @@ trait DeltaSQLConfBase extends DeltaSQLConfUtils { .checkValue(_ >= 0, "must be non-negative") .createWithDefault(500000L) + val DELTA_LOG_SEGMENT_DELTAS_TO_STRING_LIMIT = + buildConf("logSegment.deltasToStringLimit") + .internal() + .doc("Maximum number of delta files rendered when a LogSegment is turned into a string " + + "for logging or error messages. A LogSegment can hold a very large number of delta " + + "files, and rendering all of them can materialize a huge string and OOM the driver, so " + + "the remainder is elided once this limit is exceeded. Set to -1 to disable truncation.") + .longConf + .checkValue(_ >= -1, "must be -1 (truncation disabled) or non-negative") + .createWithDefault(5000L) + val DELTA_PARTITION_COLUMN_CHECK_ENABLED = buildConf("partitionColumnValidity.enabled") .internal() @@ -445,6 +482,17 @@ trait DeltaSQLConfBase extends DeltaSQLConfUtils { .intConf .createWithDefault(1000000) + val DELTA_CONVERT_REBALANCE_FILE_LISTING = + buildConf("convert.rebalanceFileListing") + .internal() + .doc("When true, CONVERT TO DELTA rebalances the recursively-listed files " + + "across tasks (by file count) before reading Parquet footers for schema inference. " + + "Files are processed in path order for deterministic schema merging. " + + "recursiveListDirs otherwise parcels files by top-level directory, so a single large " + + "partition directory becomes one skewed task that reads all its footers alone.") + .booleanConf + .createWithDefault(false) + val DELTA_CONVERT_METADATA_CHECK_ENABLED = buildConf("convert.metadataCheck.enabled") .doc( @@ -528,6 +576,17 @@ trait DeltaSQLConfBase extends DeltaSQLConfUtils { .booleanConf .createWithDefault(false) + val DELTA_COMMIT_IDEMPOTENCY_CHECK_VALIDATE_PREPARED_ACTIONS_ENABLED = + buildConf("commit.idempotencyCheck.validatePreparedActions.enabled") + .internal() + .doc("When enabled, on an idempotent self-commit (the write landed but the response was " + + "lost, detected on a later retry), validate that the prepared actions we finalize with " + + "match the actions read back from the landed commit, ignoring CommitInfo.version. This " + + "is a correctness check with a performance cost, intended to be removed once the " + + "idempotent self-commit finalization path is proven.") + .booleanConf + .createWithDefault(true) + val FEATURE_ENABLEMENT_CONFLICT_RESOLUTION_ENABLED = buildConf("featureEnablement.conflictResolution.enabled") .internal() @@ -601,6 +660,26 @@ trait DeltaSQLConfBase extends DeltaSQLConfUtils { .checkValue(_ > 0, "fullRewriteCheckpointIntervalMultiplier must be positive.") .createWithDefault(5) + val AMT_CONFLICT_CHECKING_MAX_FULL_REGENERATE_RETRIES = + buildConf("amt.conflictChecking.maxFullRegenerateRetries") + .internal() + .doc("Maximum number of times the AMT checkpoint path regenerates a losing full AMT " + + "OPTIMIZE checkpoint against a refreshed snapshot after a concurrent winner invalidated " + + "its base tree, before surfacing the conflict.") + .intConf + .checkValue(_ >= 0, "maxFullRegenerateRetries must be non-negative.") + .createWithDefault(5) + + val AMT_SNAPSHOT_DISCOVERY_ASYNC_COMMIT_INFO_READ_ENABLED = + buildConf("amt.snapshotDiscovery.asyncCommitInfoRead.enabled") + .internal() + .doc("When enabled, an async CommitInfo read will be kicked off during snapshot creation " + + "in parallel with the CRC read. Enabling it could cause slight performance overhead on " + + "non-AMT tables when CRC is absent, and extra checkpoint threadpool contention. " + + "Disabling it could result in missing file actions in snapshot discovery for AMT tables.") + .booleanConf + .createWithDefault(DeltaUtils.isTesting) + val UNSUPPORTED_TESTING_FEATURES_ENABLED = buildConf("tableFeatures.dev.unsupportedTableFeatures.enabled") .internal() @@ -1601,6 +1680,22 @@ trait DeltaSQLConfBase extends DeltaSQLConfUtils { .booleanConf .createWithDefault(true) + val DELTA_READ_SET_TRANSACTIONS_FROM_CRC = + buildConf("setTransactionsInCrc.useForReads") + .internal() + .doc("When enabled, Delta will use the setTransactions from CRC (if available) to speed up" + + " Snapshot.setTransactions API.") + .booleanConf + .createWithDefault(true) + + val FAST_QUERY_PATH_ENABLED = + buildConf("fastQueryPath.enabled") + .doc("If enabled, analysis and data skipping on Delta tables will go through a fast path " + + "that does minimal amount of work. " + + "Many snapshot fields are retrieved from the checksum if available.") + .booleanConf + .createWithDefault(true) + val DELTA_MAX_SET_TRANSACTIONS_IN_CRC = buildConf("setTransactionsInCrc.maxAllowed") .internal() @@ -1897,6 +1992,14 @@ trait DeltaSQLConfBase extends DeltaSQLConfUtils { .booleanConf .createWithDefault(true) + val REPLACEWHERE_LITERAL_STRING_PREDICATES_ENABLED = + buildConf("replaceWhere.literalStringPredicates.enabled") + .internal() + .doc("When enabled, Delta overwrite filters preserve literal string operands. " + + "When disabled, startsWith, endsWith, and contains use the legacy LIKE translation.") + .booleanConf + .createWithDefault(true) + val REPLACEWHERE_DATACOLUMNS_ENABLED = buildConf("replaceWhere.dataColumns.enabled") .doc( @@ -2141,6 +2244,20 @@ trait DeltaSQLConfBase extends DeltaSQLConfUtils { .booleanConf .createWithDefault(true) + object GeneratedColumnPartitionFilterInferenceMode extends + DeltaBreakingChangeEnum(GENERATED_COLUMN_PARTITION_FILTER_INFERENCE_MODE) + + val GENERATED_COLUMN_PARTITION_FILTER_INFERENCE_MODE = + buildConf("generatedColumn.partitionFilterInference.mode") + .internal() + .doc("Controls telemetry and suppression for generated-column partition filter " + + "inferences. LOG_ONLY records candidate signatures while retaining all inferred " + + "filters. ASSERT also suppresses candidates covered by the current safety policy.") + .stringConf + .transform(_.toUpperCase(Locale.ROOT)) + .checkValues(DeltaBreakingChangeEnum.validValues) + .createWithDefault(DeltaBreakingChangeEnum.LOG_ONLY) + val GENERATED_COLUMN_ALLOW_NULLABLE = buildConf("generatedColumn.allowNullableIngest.enabled") .internal() @@ -2169,23 +2286,26 @@ trait DeltaSQLConfBase extends DeltaSQLConfUtils { .checkValues(GeneratedColumnValidateOnWriteMode.values.map(_.toString)) .createWithDefault(GeneratedColumnValidateOnWriteMode.LOG_ONLY.toString) - sealed abstract class ConsistentDataChangeValidationMode(val name: String) { + sealed abstract class DataChangeValidationMode(val name: String) { override def toString: String = name } - object ConsistentDataChangeValidationMode { + object DataChangeValidationMode { /** Skip the validation entirely. */ - case object OFF extends ConsistentDataChangeValidationMode("off") + case object OFF extends DataChangeValidationMode("off") /** Record a Delta event on violation but do not throw. */ - case object LOG extends ConsistentDataChangeValidationMode("log") + case object LOG extends DataChangeValidationMode("log") /** Throw an exception on violation. */ - case object FATAL extends ConsistentDataChangeValidationMode("fatal") + case object FATAL extends DataChangeValidationMode("fatal") - val values: Seq[ConsistentDataChangeValidationMode] = Seq(OFF, LOG, FATAL) - private val byName: Map[String, ConsistentDataChangeValidationMode] = + val values: Seq[DataChangeValidationMode] = Seq(OFF, LOG, FATAL) + private val byName: Map[String, DataChangeValidationMode] = values.map(m => m.name -> m).toMap - def fromConf(conf: SQLConf): ConsistentDataChangeValidationMode = + def consistentDataChangeMode(conf: SQLConf): DataChangeValidationMode = byName(conf.getConf(DELTA_COMMIT_VALIDATE_CONSISTENT_DATA_CHANGE_MODE)) + + def expectedDataChangeMode(conf: SQLConf): DataChangeValidationMode = + byName(conf.getConf(DELTA_COMMIT_VALIDATE_EXPECTED_DATA_CHANGE_MODE)) } val DELTA_COMMIT_VALIDATE_CONSISTENT_DATA_CHANGE_MODE = @@ -2200,8 +2320,35 @@ trait DeltaSQLConfBase extends DeltaSQLConfUtils { |""".stripMargin) .stringConf .transform(_.toLowerCase(Locale.ROOT)) - .checkValues(ConsistentDataChangeValidationMode.values.map(_.name).toSet) - .createWithDefault(ConsistentDataChangeValidationMode.LOG.name) + .checkValues(DataChangeValidationMode.values.map(_.name).toSet) + .createWithDefault(DataChangeValidationMode.LOG.name) + + val DELTA_COMMIT_VALIDATE_EXPECTED_DATA_CHANGE_MODE = + buildConf("commitValidation.expectedDataChange.mode") + .internal() + .doc(""" + |Controls validation that every FileAction an operation commits carries the dataChange + |value the operation declares via DeltaOperations.Operation.expectedFileDataChange + |(operations that leave it unset are not validated). + | - off: Skip the validation entirely. + | - log: Record a Delta event on violation but do not throw. + | - fatal: Throw an exception on violation. + |""".stripMargin) + .stringConf + .transform(_.toLowerCase(Locale.ROOT)) + .checkValues(DataChangeValidationMode.values.map(_.name).toSet) + .createWithDefault(DataChangeValidationMode.LOG.name) + + val DELTA_COMMIT_INFO_DATA_CHANGE_READ_ENABLED = + buildConf("commitInfo.dataChange.read.enabled") + .internal() + .doc(""" + |When enabled, readers that need to know whether a commit changed data read the + |commit-level dataChange recorded in its CommitInfo instead of scanning the commit's + |file actions. + |""".stripMargin) + .booleanConf + .createWithDefault(false) object ValidateCheckConstraintsMode extends Enumeration { val OFF, LOG_ONLY, ASSERT = Value @@ -2473,6 +2620,14 @@ trait DeltaSQLConfBase extends DeltaSQLConfUtils { .booleanConf .createWithDefault(true) + val DELTA_DROP_STATS_COLUMNS_ESCAPE_NAMES = + buildConf("stats.dropStatsColumns.escapeNames") + .internal() + .doc("Whether to properly escape surviving data skipping stats column names after dropping " + + "a column.") + .booleanConf + .createWithDefault(true) + val DELTA_ALTER_TABLE_DROP_COLUMN_ENABLED = buildConf("alterTable.dropColumn.enabled") .internal() @@ -2958,6 +3113,31 @@ trait DeltaSQLConfBase extends DeltaSQLConfUtils { .booleanConf .createWithDefault(true) + val DELETION_VECTORS_USE_OBJECT_IDENTITY_FOR_NON_AMT = + buildConf("deletionVectors.useObjectIdentityForNonAMTTables") + .internal() + .doc( + """When true, Delta compares deletion vectors by their normalized object identity instead + |of their legacy descriptor identity for non-AMT tables (AMT tables always use object + |identity). The legacy identity is based on the serialized descriptor fields, + |so equivalent `u`, `r`, and in-table `p` descriptors compare as different DVs. + |The object identity is based on the table-relative DV object location and offset + |when possible, so those equivalent descriptors compare as the same DV. + |""".stripMargin) + .booleanConf + .createWithDefault(false) + + val DELETION_VECTORS_USE_OBJECT_IDENTITY_FOR_INCREMENTAL_CRC = + buildConf("deletionVectors.useObjectIdentityForIncrementalCRCComputation") + .internal() + .doc( + """Kill-switch for reconciling deletion vectors by their normalized object identity when + |incrementally computing the checksum (CRC). When false, deletion vectors fall back to + |their legacy descriptor identity. + |""".stripMargin) + .booleanConf + .createWithDefault(true) + val DELETION_VECTOR_PACKING_TARGET_SIZE = buildConf("deletionVectors.packing.targetSize") .internal() @@ -3037,6 +3217,18 @@ trait DeltaSQLConfBase extends DeltaSQLConfUtils { .booleanConf .createWithDefault(true) + val RETAIN_COMMENTS_DURING_REPLACE_TABLE = + buildConf("retainCommentsDuringReplace") + .internal() + .doc( + """ + |If enabled, replacing a table (CREATE OR REPLACE TABLE, REPLACE TABLE, or the + |DataFrameWriterV2 replace()/createOrReplace() APIs) retains table and column comments + |from the old table when the new DDL does not explicitly specify them. + |""".stripMargin) + .booleanConf + .createWithDefault(true) + val ALLOW_COLUMN_MAPPING_REMOVAL = buildConf("columnMapping.allowRemoval") .internal() @@ -3253,6 +3445,16 @@ trait DeltaSQLConfBase extends DeltaSQLConfUtils { ) .createWithDefault(4) + val DELTA_REPLACE_PARTITIONED_BY_VERIFY_MATERIALIZED_PARTITION_COLUMNS = + buildConf("alterTable.replacePartitionedBy.verifyMaterializedPartitionColumns") + .internal() + .doc("""When true, ALTER TABLE ... REPLACE PARTITIONED BY WITH CLUSTER BY reads the Parquet + |footer of every data file to verify that the partition columns are physically stored in + |the file before dropping the partitioning. Disabling this check can make partition column + |values unreadable if a data file does not contain them.""".stripMargin) + .booleanConf + .createWithDefault(true) + val DELTA_LOG_CACHE_SIZE = buildConf("delta.log.cacheSize") .internal() .doc("The maximum number of DeltaLog instances to cache in memory.") @@ -3327,6 +3529,16 @@ trait DeltaSQLConfBase extends DeltaSQLConfUtils { .booleanConf .createWithDefault(false) + val DELTA_SHARING_STREAMING_CONVERT_STARTING_TIMESTAMP_TO_VERSION = + buildConf("spark.sql.delta.sharing.streamingConvertStartingTimestampToVersion") + .doc("When true, a Delta Sharing streaming query converts startingTimestamp to a version " + + "on the sharing server, passing that version to the wrapped DeltaSource. When false, the " + + "wrapped DeltaSource resolves the timestamp again on the local delta log, where an empty " + + "version range fails with DELTA_TIMESTAMP_GREATER_THAN_COMMIT.") + .internal() + .booleanConf + .createWithDefault(false) + val DELTA_SHARING_ENABLE_AUTO_RESOLVE_FOR_CDF = buildConf("spark.sql.delta.sharing.enableAutoResolveForCdf") .doc("When true, Delta Sharing CDF queries without an explicit responseFormat will " + @@ -3622,13 +3834,6 @@ trait DeltaSQLConfBase extends DeltaSQLConfUtils { .booleanConf .createWithDefault(true) - val DELTA_STREAMING_INITIAL_SNAPSHOT_MAX_FILES = - buildConf("streaming.initialSnapshotMaxFiles") - .internal() - .doc("Maximum number of files allowed in initial snapshot for V2 streaming.") - .intConf - .createWithDefault(100000) - val DELTA_STREAMING_USE_DISTRIBUTED_INITIAL_SNAPSHOT = buildConf("streaming.distributedInitialSnapshot") .internal() diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSource.scala b/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSource.scala index b757428bb4b..e6a2e40ad85 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSource.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSource.scala @@ -1192,24 +1192,10 @@ case class DeltaSource( startOffsetOption: Option[DeltaSourceOffset], endOffset: DeltaSourceOffset): (Long, Long, Boolean) = { val (startVersion, startIndex, isInitialSnapshot) = if (startOffsetOption.isEmpty) { - getStartingVersion match { - case Some(v) => - (v, DeltaSourceOffset.BASE_INDEX, false) - - case None => - if (endOffset.isInitialSnapshot) { - (endOffset.reservoirVersion, DeltaSourceOffset.BASE_INDEX, true) - } else { - assert( - endOffset.reservoirVersion > 0, s"invalid reservoirVersion in endOffset: $endOffset") - // Load from snapshot `endOffset.reservoirVersion - 1L` so that `index` in `endOffset` - // is still valid. - // It's OK to use the previous version as the updated initial snapshot, even if the - // initial snapshot might have been different from the last time when this starting - // offset was computed. - (endOffset.reservoirVersion - 1L, DeltaSourceOffset.BASE_INDEX, true) - } - } + val startingVersion = getStartingVersion + (DeltaStreamUtils.resolveFirstBatchStartVersion(endOffset, startingVersion), + DeltaSourceOffset.BASE_INDEX, + startingVersion.isEmpty) } else { val startOffset = startOffsetOption.get if (!startOffset.isInitialSnapshot) { @@ -1329,6 +1315,11 @@ object DeltaSource extends DeltaLogging { bytesToTake -= bytes } + /** Returns whether an atomic group of files fits within the admission limits. */ + protected def hasCapacityFor(files: Int, bytes: Long): Boolean = { + filesToTake - files >= 0 && bytesToTake - bytes >= 0 + } + /** * This overloaded method checks if all the FileActions for a commit can be accommodated by * the rate limit. @@ -1344,8 +1335,7 @@ object DeltaSource extends DeltaLogging { // else check if all of the files together satisfy the limit, only then admit val bytesInFiles = getSize(admittableFiles) val shouldAdmit = !commitProcessedInBatch || - (filesToTake - admittableFiles.size >= 0 && bytesToTake - bytesInFiles >= 0) - + hasCapacityFor(admittableFiles.size, bytesInFiles) commitProcessedInBatch = true take(files = admittableFiles.size, bytes = bytesInFiles) shouldAdmit diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSourceCDCSupport.scala b/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSourceCDCSupport.scala index c9cdf889203..7ed4e93b98e 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSourceCDCSupport.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSourceCDCSupport.scala @@ -145,7 +145,13 @@ trait DeltaSourceCDCSupport { self: DeltaSource => // This is to avoid returning `update_preimage` and `update_postimage` in separate // batches. if (admissionControl.admit(filteredFiles)) { - filteredFiles.toIterator + // Emit the BEGIN/END index markers with the CDC files (as the AddFile/RemoveFile + // branch does) so buildOffsetFromIndexedFile rolls the offset to (version + 1, + // BASE_INDEX); an AddCDCFile window-top would otherwise park and stall the stream. + fileActions + .filter(f => f.cdc != null || hasNoFileActionAndStartOrEndIndex(f)) + .filter(isValidIndexedFile(_, fromVersion, fromIndex, endOffset)) + .toIterator } else { Iterator() } @@ -431,11 +437,13 @@ trait DeltaSourceCDCSupport { self: DeltaSource => protocolAction, commitInfoAction) } else { + val changesDataFunc = + CommitInfo.fileActionChangesData(actions.collectFirst { case ci: CommitInfo => ci }) (actions.filter { case a: AddFile => - a.dataChange + changesDataFunc(a) case r: RemoveFile => - r.dataChange + changesDataFunc(r) case m: Metadata => checkAndCacheMetadata(m) false diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSourceUtils.scala b/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSourceUtils.scala index 226b44e8a50..f5bf4d02067 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSourceUtils.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSourceUtils.scala @@ -21,6 +21,7 @@ import java.util.Locale import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute import org.apache.spark.sql.catalyst.expressions import org.apache.spark.sql.catalyst.expressions.Expression +import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.sources import org.apache.spark.sql.sources.Filter @@ -94,14 +95,29 @@ object DeltaSourceUtils { case sources.Or(filter1, filter2) => expressions.Or(translateFilters(Array(filter1)), translateFilters(Array(filter2))) case sources.StringStartsWith(attribute, value) => - new expressions.Like( - UnresolvedAttribute(attribute), expressions.Literal.create(s"${value}%")) + if (SQLConf.get.getConf(DeltaSQLConf.REPLACEWHERE_LITERAL_STRING_PREDICATES_ENABLED)) { + expressions.StartsWith( + UnresolvedAttribute(attribute), expressions.Literal.create(value)) + } else { + new expressions.Like( + UnresolvedAttribute(attribute), expressions.Literal.create(s"${value}%")) + } case sources.StringEndsWith(attribute, value) => - new expressions.Like( - UnresolvedAttribute(attribute), expressions.Literal.create(s"%${value}")) + if (SQLConf.get.getConf(DeltaSQLConf.REPLACEWHERE_LITERAL_STRING_PREDICATES_ENABLED)) { + expressions.EndsWith( + UnresolvedAttribute(attribute), expressions.Literal.create(value)) + } else { + new expressions.Like( + UnresolvedAttribute(attribute), expressions.Literal.create(s"%${value}")) + } case sources.StringContains(attribute, value) => - new expressions.Like( - UnresolvedAttribute(attribute), expressions.Literal.create(s"%${value}%")) + if (SQLConf.get.getConf(DeltaSQLConf.REPLACEWHERE_LITERAL_STRING_PREDICATES_ENABLED)) { + expressions.Contains( + UnresolvedAttribute(attribute), expressions.Literal.create(value)) + } else { + new expressions.Like( + UnresolvedAttribute(attribute), expressions.Literal.create(s"%${value}%")) + } case sources.AlwaysTrue() => expressions.Literal.TrueLiteral case sources.AlwaysFalse() => expressions.Literal.FalseLiteral }.reduceOption(expressions.And).getOrElse(expressions.Literal.TrueLiteral) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaStreamUtils.scala b/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaStreamUtils.scala index 114eceda778..8c7e6dac0a3 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaStreamUtils.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaStreamUtils.scala @@ -304,4 +304,40 @@ object DeltaStreamUtils { } } } + + /** + * Resolves the version from which the first batch should scan when no start offset is available. + * + * A configured starting version (including one resolved from `startingTimestamp`) takes + * precedence. Without one, an initial-snapshot end offset uses its snapshot version. An + * incremental end offset uses the immediately preceding version as an updated initial snapshot, + * allowing the commit-local index in the end offset to remain valid. + * + * @param endOffset inclusive end offset of the first planned batch + * @param startingVersion starting version resolved from the source options, if configured + * @return version from which the first batch should scan + */ + def resolveFirstBatchStartVersion( + endOffset: DeltaSourceOffset, + startingVersion: Option[Long]): Long = { + startingVersion match { + case Some(v) => + v + + case None => + if (endOffset.isInitialSnapshot) { + endOffset.reservoirVersion + } else { + assert( + endOffset.reservoirVersion > 0, + s"invalid reservoirVersion in endOffset: $endOffset") + // Load from snapshot `endOffset.reservoirVersion - 1L` so that `index` in `endOffset` + // is still valid. + // It's OK to use the previous version as the updated initial snapshot, even if the + // initial snapshot might have been different from the last time when this starting + // offset was computed. + endOffset.reservoirVersion - 1L + } + } + } } diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/stats/DataSkippingReader.scala b/spark/src/main/scala/org/apache/spark/sql/delta/stats/DataSkippingReader.scala index 574e0e3008e..5ed63e67b82 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/stats/DataSkippingReader.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/stats/DataSkippingReader.scala @@ -18,6 +18,7 @@ package org.apache.spark.sql.delta.stats // scalastyle:off import.ordering.noEmptyLine import java.io.Closeable +import java.util.concurrent.TimeUnit.NANOSECONDS import scala.collection.mutable.ArrayBuffer @@ -654,7 +655,7 @@ trait DataSkippingReaderBase * of the statistics need to be consistent across all files. */ override def filesForScan(filters: Seq[Expression], keepNumRecords: Boolean): DeltaScan = { - val startTime = System.currentTimeMillis() + val startTimeNs = System.nanoTime() if (filters == Seq(TrueLiteral) || filters.isEmpty || schema.isEmpty) { recordDeltaOperation(snapshotToScan, "delta.skipping.none") { // When there are no filters we can just return allFiles with no extra processing @@ -673,6 +674,7 @@ trait DataSkippingReaderBase rows = rowCount, files = numOfFilesIfKnown, logicalRows = logicalRowCount) + val durationNs = System.nanoTime() - startTimeNs return DeltaScan( version = version, files = files, @@ -685,7 +687,8 @@ trait DataSkippingReaderBase partitionLikeDataFilters = ExpressionSet(Nil), rewrittenPartitionLikeDataFilters = Set.empty, unusedFilters = ExpressionSet(Nil), - scanDurationMs = System.currentTimeMillis() - startTime, + scanDurationMs = NANOSECONDS.toMillis(durationNs), + scanDurationNs = Some(durationNs), dataSkippingType = getCorrectDataSkippingType(DeltaDataSkippingType.noSkippingV1) ) } @@ -713,6 +716,7 @@ trait DataSkippingReaderBase // When there are only partition filters we can scan allFiles // rather than withStats and thus we skip data skipping information. val (files, scanSize) = filterOnPartitions(partitionFilters, keepNumRecords) + val durationNs = System.nanoTime() - startTimeNs DeltaScan( version = version, files = files, @@ -725,7 +729,8 @@ trait DataSkippingReaderBase partitionLikeDataFilters = ExpressionSet(Nil), rewrittenPartitionLikeDataFilters = Set.empty, unusedFilters = ExpressionSet(ineligibleFilters), - scanDurationMs = System.currentTimeMillis() - startTime, + scanDurationMs = NANOSECONDS.toMillis(durationNs), + scanDurationNs = Some(durationNs), dataSkippingType = getCorrectDataSkippingType(DeltaDataSkippingType.partitionFilteringOnlyV1) ) @@ -805,6 +810,7 @@ trait DataSkippingReaderBase getDataSkippedFiles(finalPartitionFilters, finalSkippingFilters, keepNumRecords) } + val durationNs = System.nanoTime() - startTimeNs DeltaScan( version = version, files = files, @@ -817,7 +823,8 @@ trait DataSkippingReaderBase partitionLikeDataFilters = ExpressionSet(partitionLikeFilters.map(_._1)), rewrittenPartitionLikeDataFilters = partitionLikeFilters.map(_._2.expr.expr).toSet, unusedFilters = ExpressionSet(unusedFilters.map(_._1) ++ ineligibleFilters), - scanDurationMs = System.currentTimeMillis() - startTime, + scanDurationMs = NANOSECONDS.toMillis(durationNs), + scanDurationNs = Some(durationNs), dataSkippingType = getCorrectDataSkippingType(dataSkippingType) ) } @@ -830,7 +837,7 @@ trait DataSkippingReaderBase */ override def filesForScan(limit: Long, partitionFilters: Seq[Expression]): DeltaScan = recordDeltaOperation(snapshotToScan, "delta.skipping.filteredLimit") { - val startTime = System.currentTimeMillis() + val startTimeNs = System.nanoTime() val finalPartitionFilters = constructPartitionFilters(partitionFilters) val scan = { @@ -851,6 +858,7 @@ trait DataSkippingReaderBase scan.numLogicalRecords ) + val durationNs = System.nanoTime() - startTimeNs DeltaScan( version = version, files = scan.files, @@ -863,7 +871,8 @@ trait DataSkippingReaderBase partitionLikeDataFilters = ExpressionSet(Nil), rewrittenPartitionLikeDataFilters = Set.empty, unusedFilters = ExpressionSet(Nil), - scanDurationMs = System.currentTimeMillis() - startTime, + scanDurationMs = NANOSECONDS.toMillis(durationNs), + scanDurationNs = Some(durationNs), dataSkippingType = DeltaDataSkippingType.filteredLimit ) } diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/stats/DeltaScan.scala b/spark/src/main/scala/org/apache/spark/sql/delta/stats/DeltaScan.scala index ba5cd8cddcf..92fd15c7ace 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/stats/DeltaScan.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/stats/DeltaScan.scala @@ -93,6 +93,7 @@ case class DeltaScan( val rewrittenPartitionLikeDataFilters: Set[Expression], val unusedFilters: ExpressionSet, val scanDurationMs: Long, + val scanDurationNs: Option[Long] = None, val dataSkippingType: DeltaDataSkippingType) { assert(version == scannedSnapshot.version) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/stats/PrepareDeltaScan.scala b/spark/src/main/scala/org/apache/spark/sql/delta/stats/PrepareDeltaScan.scala index 7100c0149a9..4ba6ffc9547 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/stats/PrepareDeltaScan.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/stats/PrepareDeltaScan.scala @@ -121,7 +121,10 @@ trait PrepareDeltaScanBase extends Rule[LogicalPlan] // If we trigger limit push down, the filters must be partition filters. Since // there are no data filters, we don't need to apply Generated Columns // optimization. See `DeltaTableScan` for more details. - return scanGenerator.filesForScan(limitOpt.get, filters) + return recordFrameProfile( + "Delta", "PrepareDeltaScan.filesForScan.limitAndFilters") { + scanGenerator.filesForScan(limitOpt.get, filters) + } } val filtersForScan = if (!GeneratedColumn.partitionFilterOptimizationEnabled(spark)) { @@ -131,7 +134,9 @@ trait PrepareDeltaScanBase extends Rule[LogicalPlan] spark, scanGenerator.snapshotToScan, filters, delta) filters ++ generatedPartitionFilters } - scanGenerator.filesForScan(filtersForScan) + return recordFrameProfile("Delta", "PrepareDeltaScan.filesForScan.filters") { + scanGenerator.filesForScan(filtersForScan) + } } } diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/stats/StatisticsCollection.scala b/spark/src/main/scala/org/apache/spark/sql/delta/stats/StatisticsCollection.scala index 2144fc23220..5d53f442daf 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/stats/StatisticsCollection.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/stats/StatisticsCollection.scala @@ -630,6 +630,7 @@ object StatisticsCollection extends DeltaCommand { val deltaStatsColumnSpec = configuredDeltaStatsColumnSpec(metadata) deltaStatsColumnSpec.deltaStatsColumnNamesOpt.map { deltaColumnsNames => val droppedColumnSet = columnsToDrop.toSet + val escapeNames = SQLConf.get.getConf(DeltaSQLConf.DELTA_DROP_STATS_COLUMNS_ESCAPE_NAMES) val deltaStatsColumnStr = deltaColumnsNames .map(_.nameParts) .filterNot { attributeNameParts => @@ -640,7 +641,13 @@ object StatisticsCollection extends DeltaCommand { commonPrefix == droppedColumnParts.size }.nonEmpty } - .map(columnParts => UnresolvedAttribute(columnParts).name) + .map { columnParts => + if (escapeNames) { + UnresolvedAttribute(columnParts).sql + } else { + UnresolvedAttribute(columnParts).name + } + } .mkString(",") Map(DeltaConfigs.DATA_SKIPPING_STATS_COLUMNS.key -> deltaStatsColumnStr) }.getOrElse(Map.empty[String, String]) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/util/DeltaCommitFileProvider.scala b/spark/src/main/scala/org/apache/spark/sql/delta/util/DeltaCommitFileProvider.scala index b6554abd0dc..62268f29f8c 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/util/DeltaCommitFileProvider.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/util/DeltaCommitFileProvider.scala @@ -16,6 +16,7 @@ package org.apache.spark.sql.delta.util +import org.apache.spark.sql.delta.LogSegment import org.apache.spark.sql.delta.Snapshot import org.apache.spark.sql.delta.util.FileNames._ import org.apache.hadoop.fs.Path @@ -78,9 +79,13 @@ case class DeltaCommitFileProvider( object DeltaCommitFileProvider { def apply(snapshot: Snapshot): DeltaCommitFileProvider = { - val uuids = snapshot.logSegment.deltas - .collect { case UnbackfilledDeltaFile(_, version, uuid) => version -> uuid } - .toMap - new DeltaCommitFileProvider(snapshot.path.toString, snapshot.version, uuids) + apply(snapshot.logPath, snapshot.logSegment) + } + + def apply(logPath: Path, logSegment: LogSegment): DeltaCommitFileProvider = { + val uuids = logSegment.unbackfilledDeltas.map { + case (_, version, uuid) => version -> uuid + }.toMap + new DeltaCommitFileProvider(logPath.toString, logSegment.version, uuids) } } diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/util/DeltaFileSystemOptions.scala b/spark/src/main/scala/org/apache/spark/sql/delta/util/DeltaFileSystemOptions.scala new file mode 100644 index 00000000000..8149177158e --- /dev/null +++ b/spark/src/main/scala/org/apache/spark/sql/delta/util/DeltaFileSystemOptions.scala @@ -0,0 +1,68 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.delta.util + +import org.apache.spark.sql.delta.DeltaTableUtils +import org.apache.spark.sql.delta.sources.DeltaSQLConf + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.catalog.CatalogTable + +/** + * Shared helpers for deriving Hadoop filesystem options from DataFrame options and catalog table + * storage properties. + */ +object DeltaFileSystemOptions { + + /** + * Retains only entries whose key starts with a recognised Hadoop filesystem + * prefix ([[DeltaTableUtils.validDeltaTableHadoopPrefixes]]). + */ + private[delta] def filterHadoopOptions(options: Map[String, String]): Map[String, String] = { + options.filter { case (k, _) => + DeltaTableUtils.validDeltaTableHadoopPrefixes.exists(k.startsWith) + } + } + + /** + * Extracts file-system-relevant storage properties from a catalog table. + */ + def extractCatalogTableFsOptions(catalogTableOpt: Option[CatalogTable]): Map[String, String] = { + catalogTableOpt + .map(ct => filterHadoopOptions(ct.storage.properties)) + .getOrElse(Map.empty) + } + + /** + * Constructs filesystem options by combining catalog table storage properties with DataFrame + * reader/writer options. Only keys matching valid Hadoop prefixes are picked up so that parquet + * or json options are not passed to the code that reads Delta transaction logs. + * + * Gated by [[DeltaSQLConf.LOAD_FILE_SYSTEM_CONFIGS_FROM_DATAFRAME_OPTIONS]]. + */ + def buildFsOptions( + spark: SparkSession, + options: Map[String, String], + catalogTableOpt: Option[CatalogTable] = None): Map[String, String] = { + val catalogStorageProps = extractCatalogTableFsOptions(catalogTableOpt) + if (spark.sessionState.conf.getConf( + DeltaSQLConf.LOAD_FILE_SYSTEM_CONFIGS_FROM_DATAFRAME_OPTIONS)) { + catalogStorageProps ++ filterHadoopOptions(options) + } else { + catalogStorageProps + } + } +} diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/util/DeltaTestBarrier.scala b/spark/src/main/scala/org/apache/spark/sql/delta/util/DeltaTestBarrier.scala new file mode 100644 index 00000000000..0bccf5b7069 --- /dev/null +++ b/spark/src/main/scala/org/apache/spark/sql/delta/util/DeltaTestBarrier.scala @@ -0,0 +1,52 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.util + +import com.databricks.spark.util.{TestBarrier, TestBarrierRegistry} +import org.apache.spark.sql.delta.sources.DeltaSQLConf + +import org.apache.spark.sql.internal.SQLConf + +/** Entry point for labeled barriers used by Delta tests. */ +object DeltaTestBarrier { + /** Creates and registers a barrier for `label`. */ + def createBarrier(conf: SQLConf, label: String, hitCount: Int = 1): TestBarrier = { + assert(conf.getConf(DeltaSQLConf.TEST_BARRIER_ENABLED)) + TestBarrierRegistry.createBarrier(label, hitCount, Utils.isTesting) + } + + /** Blocks the current thread if the barrier for `label` is enabled. */ + def waitIfEnabled(label: String): Unit = + TestBarrierRegistry.waitIfEnabled(label, Utils.isTesting) + + /** Prevents new threads from blocking without releasing current waiters. */ + def disableBarrierWithoutReleasing(label: String): Unit = + TestBarrierRegistry.disableBarrierWithoutReleasing(label) + + /** Records hits and destroys the barrier once its target count is reached. */ + def hitBarrier(label: String, times: Int = 1): Unit = + TestBarrierRegistry.hitBarrier(label, times) + + /** Enables the barrier registered for `label`. */ + def enableBarrier(label: String): Unit = TestBarrierRegistry.enableBarrier(label) + + /** Removes the barrier for `label` and releases all blocked threads. */ + def destroyBarrier(label: String): Unit = TestBarrierRegistry.destroyBarrier(label) + + /** Returns whether a barrier is registered for `label`. */ + def isBarrierEnabled(label: String): Boolean = TestBarrierRegistry.isBarrierEnabled(label) +} diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/util/PartitionUtils.scala b/spark/src/main/scala/org/apache/spark/sql/delta/util/PartitionUtils.scala index 3a7241486ad..959b77807cd 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/util/PartitionUtils.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/util/PartitionUtils.scala @@ -56,10 +56,11 @@ import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.analysis._ import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec -import org.apache.spark.sql.catalyst.expressions.{Attribute, Cast, Literal} +import org.apache.spark.sql.catalyst.expressions.{Attribute, Cast, DateFormatClass, Expression, Literal} import org.apache.spark.sql.catalyst.types.DataTypeUtils import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap import org.apache.spark.sql.types._ +import org.apache.spark.util.Utils /** * This file is forked from [[org.apache.spark.sql.execution.datasources.PartitioningUtils]]. @@ -120,7 +121,9 @@ object PartitionSpec { private[delta] object PartitionUtils { lazy val timestampPartitionPattern = s"yyyy-MM-dd HH:mm:ss${precisionMatchPatterns(6)}" - lazy val utcFormatter = TimestampFormatter(s"yyyy-MM-dd'T'HH:mm:ss.SSSSSSz", ZoneId.of("Z")) + private val utcZoneId: ZoneId = ZoneId.of("Z") + private val utcTimestampPartitionFormat = s"yyyy-MM-dd'T'HH:mm:ss.SSSSSSz" + lazy val utcFormatter = TimestampFormatter(utcTimestampPartitionFormat, utcZoneId) private def precisionMatchPatterns(maxDigits: Int): String = (maxDigits to 1 by -1) @@ -760,6 +763,13 @@ private[delta] object PartitionUtils { /** * Converts a typed literal to a normalized string representation for correct comparisons. + * This must be kept in sync with [[expressionToNormalizedString]]. + * + * TODO(v4amt): make this delegate to [[expressionToNormalizedString]] so the two cannot drift. + * The duplication is what makes drift possible at all: a partition value written through the + * literal path and read back through the expression path has to render identically, and only + * the two implementations agreeing enforces that today. + * * @param literal The Literal to convert to a string. Must have the correct type set. * @param timeZoneId Optional timezone ID for timestamp types. * @param useUtcNormalizedTimestamp If true, formats timestamp types into UTC ISO 8601 format. @@ -776,7 +786,18 @@ private[delta] object PartitionUtils { literal.dataType match { case TimestampType if useUtcNormalizedTimestamp => // Format timestamp in UTC ISO 8601 format: "2000-01-01T12:00:00.000000Z" - utcFormatter.format(literal.value.asInstanceOf[Long]) + val normalized = utcFormatter.format(literal.value.asInstanceOf[Long]) + if (Utils.isTesting) { + val fromExpression = expressionToNormalizedString( + value = literal, + dataType = literal.dataType, + timeZoneId = timeZoneId, + useUtcNormalizedTimestamp = useUtcNormalizedTimestamp).eval() + assert(Option(fromExpression).map(_.toString).orNull == normalized, + s"literalToNormalizedString and expressionToNormalizedString disagree on " + + s"${literal.dataType}: '$normalized' vs '$fromExpression'.") + } + normalized case _ => // All other types can safely be converted to a string. @@ -785,6 +806,34 @@ private[delta] object PartitionUtils { } } + /** + * Converts a typed value to a normalized string representation for correct comparisons. + * This must be kept in sync with [[literalToNormalizedString]]. + * @param value The value expression to convert to a string. + * @param dataType The data type of the value. + * @param timeZoneId Optional timezone ID for timestamp types. + * @param useUtcNormalizedTimestamp If true, formats timestamp types into UTC ISO 8601 format. + * @return The string representation of the value as an expression. + */ + def expressionToNormalizedString( + value: Expression, + dataType: DataType, + timeZoneId: Option[String] = None, + useUtcNormalizedTimestamp: Boolean = false): Expression = dataType match { + // TODO(v4amt): follow the Iceberg rule for timestamps that carry a zone. Per the spec such a + // value "represents a point in time: values are stored as UTC and do not retain a source time + // zone", so `2017-11-16 17:10:34 PST` and `2017-11-17 01:10:34 UTC` are the same value. The + // UTC normalization below gives that for a TimestampType read through this path, but the rule + // has to hold for every path that compares partition values, not just this one. + case TimestampType if useUtcNormalizedTimestamp => + // Format timestamp in UTC ISO 8601 format: "2000-01-01T12:00:00.000000Z" + DateFormatClass(value, Literal(utcTimestampPartitionFormat), Some(utcZoneId.getId)) + + case _ => + // All other types can safely be converted to a string. + Cast(value, StringType, timeZoneId, ansiEnabled = false) + } + /** * Parses partition values (strings) to their corresponding Literal with the appropiate type * as defined in the partition schema. diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/util/TransactionHelper.scala b/spark/src/main/scala/org/apache/spark/sql/delta/util/TransactionHelper.scala index ab4f3fe6d39..39df1dc64b6 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/util/TransactionHelper.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/util/TransactionHelper.scala @@ -25,7 +25,7 @@ import org.apache.spark.sql.delta.{CatalogOwnedTableFeature, CommitStats, Commit import org.apache.spark.sql.delta.DeltaOperations.Operation import org.apache.spark.sql.delta.RowId.RowTrackingMetadataDomain import org.apache.spark.sql.delta.actions.{Action, AddCDCFile, AddFile, CommitInfo, DomainMetadata, FileAction, Metadata, Protocol, RemoveFile, SetTransaction} -import org.apache.spark.sql.delta.amt.AMTWriteMetrics +import org.apache.spark.sql.delta.amt.AMTCommitStats import org.apache.spark.sql.delta.coordinatedcommits.{CatalogOwnedTableUtils, TableCommitCoordinatorClient} import org.apache.spark.sql.delta.hooks.PostCommitHook import org.apache.spark.sql.delta.logging.DeltaLogKeys @@ -55,6 +55,12 @@ trait TransactionHelper extends DeltaLogging { /** The Hadoop [[Configuration]] used to access the Delta log. */ def newDeltaHadoopConf(): Configuration = deltaLog.newDeltaHadoopConf() + /** Canonical name of the commit log store class for commit-stats telemetry. */ + protected def commitLogStoreClassName: String = deltaLog.store.getClass.getCanonicalName + + /** Value used for the `TAG_LOG_STORE_CLASS` operation tag. */ + protected[delta] def commitLogStoreClassNameForTag: String = deltaLog.store.getClass.getName + def catalogTable: Option[CatalogTable] def snapshot: Snapshot @@ -378,7 +384,7 @@ trait TransactionHelper extends DeltaLogging { fileSizeHistogramOpt: Option[FileSizeHistogram], commitInfoOpt: Option[CommitInfo], commitSizeBytes: Long, - amtWriteMetricsOpt: Option[AMTWriteMetrics] = None, + amtCommitStatsOpt: Option[AMTCommitStats] = None, isIdempotentRetry: Boolean = false): Unit = { assertStateBeforeFinalization() @@ -425,7 +431,7 @@ trait TransactionHelper extends DeltaLogging { removeFilesHistogram = removeFilesHistogram.map(FileSizeHistogramUtils.compress), numOfDomainMetadatas = numOfDomainMetadatas, txnId = Some(txnId), - amtWriteMetrics = amtWriteMetricsOpt) + amtCommitStats = amtCommitStatsOpt) recordDeltaEvent(deltaLog, DeltaLogging.DELTA_COMMIT_STATS_OPTYPE, data = stats) } diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2TableManager.scala b/spark/src/main/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2TableManager.scala deleted file mode 100644 index 73354dfb7d0..00000000000 --- a/spark/src/main/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2TableManager.scala +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (2021) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.delta.v2.interop - -import org.apache.spark.sql.delta.{DeltaLog, Snapshot} -import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.fs.Path - -import org.apache.spark.util.Clock - -/** - * Interface for accessing and managing a Delta table. Mirrors DeltaLog's role as the entry - * point for all Delta table operations. - * - * Two implementations: - * - [[DeltaLog]] (existing): reads the Delta protocol via log replay - * - KernelTableManager (future): reads the Delta protocol via Kernel - * - * Commands, OptimisticTransaction, and other business logic code against this interface - * so they work with either backend. - */ -trait DeltaV2TableManager { - - /** The path of the Delta log directory. */ - def logPath: Path - - /** The path of the data directory of the Delta table. */ - def dataPath: Path - - /** Clock used for timestamping operations. */ - def clock: Clock - - /** Hadoop configuration options for this table. */ - def options: Map[String, String] - - /** Create a new Hadoop Configuration for this table. */ - def newDeltaHadoopConf(): Configuration - - /** Get the current snapshot of the table, updating if necessary. */ - def snapshot: Snapshot - - /** Get the latest snapshot without checking for updates. */ - def unsafeVolatileSnapshot: Snapshot - - /** Whether the table exists (version >= 0). */ - def tableExists: Boolean - - /** A unique identifier for this table. */ - def tableId: String - - /** Create the log directories if they do not exist. */ - def createLogDirectoriesIfNotExists(): Unit -} diff --git a/spark/src/test/java/org/apache/spark/sql/delta/deletionvectors/mumbling/PFORRandomData.java b/spark/src/test/java/org/apache/spark/sql/delta/deletionvectors/mumbling/PFORRandomData.java new file mode 100644 index 00000000000..ed87cba5a4d --- /dev/null +++ b/spark/src/test/java/org/apache/spark/sql/delta/deletionvectors/mumbling/PFORRandomData.java @@ -0,0 +1,45 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.deletionvectors.mumbling; + +import java.util.Random; + +/** Data generation for {@link PFOREncoding} tests. */ +class PFORRandomData { + + private PFORRandomData() {} + + /** Generates {@code count} values between 0 and {@code maxValue}. */ + static int[] uniform(Random random, int count, int maxValue) { + int[] values = new int[count]; + for (int i = 0; i < count; i++) { + values[i] = random.nextInt(maxValue + 1); + } + + return values; + } + + /** Generates values in [0, 3] with about {@code excPercent} exceptions in [0, 255]. */ + static int[] exceptions(Random random, int count, float excPercent) { + int[] values = new int[count]; + for (int i = 0; i < count; i++) { + values[i] = random.nextFloat() < excPercent ? random.nextInt(256) : random.nextInt(4); + } + + return values; + } +} diff --git a/spark/src/test/java/org/apache/spark/sql/delta/deletionvectors/mumbling/TestMumblingBitmap.java b/spark/src/test/java/org/apache/spark/sql/delta/deletionvectors/mumbling/TestMumblingBitmap.java new file mode 100644 index 00000000000..2404185eb2e --- /dev/null +++ b/spark/src/test/java/org/apache/spark/sql/delta/deletionvectors/mumbling/TestMumblingBitmap.java @@ -0,0 +1,383 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.deletionvectors.mumbling; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.google.common.base.Preconditions; +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.junit.jupiter.api.Test; + +public class TestMumblingBitmap { + + @Test + public void testEmptyBitmap() { + MumblingBitmap bitmap = bitmap(); + assertEquals(0, bitmap.cardinality()); + assertFalse(bitmap.isSet(0)); + assertFalse(bitmap.isSet(255)); + assertFalse(bitmap.isSet(256)); + } + + @Test + public void testInvalidPosition() { + MumblingBitmap bitmap = bitmap(); + assertEquals(0, bitmap.cardinality()); + assertFalse(bitmap.isSet(0)); + + IllegalArgumentException error = + assertThrows(IllegalArgumentException.class, () -> bitmap.isSet(-1)); + assertEquals("Invalid bit position: -1 < 0", error.getMessage()); + } + + @Test + public void testEmptySparseContainer() { + MumblingBitmap bitmap = bitmap(sparse()); + assertEquals(0, bitmap.cardinality()); + assertFalse(bitmap.isSet(0)); + assertFalse(bitmap.isSet(100)); + assertFalse(bitmap.isSet(255)); + } + + @Test + public void testSparseContainerSetPositions() { + MumblingBitmap bitmap = bitmap(sparse(0, 5, 100, 255)); + assertEquals(4, bitmap.cardinality()); + + assertTrue(bitmap.isSet(0)); + assertTrue(bitmap.isSet(5)); + assertTrue(bitmap.isSet(100)); + assertTrue(bitmap.isSet(255)); + assertFalse(bitmap.isSet(1)); + assertFalse(bitmap.isSet(4)); + assertFalse(bitmap.isSet(6)); + assertFalse(bitmap.isSet(99)); + assertFalse(bitmap.isSet(101)); + assertFalse(bitmap.isSet(254)); + assertFalse(bitmap.isSet(256)); + } + + @Test + public void testFullSparseContainer() { + int[] positions = new int[31]; + for (int i = 0; i < positions.length; i += 1) { + positions[i] = i * 8; + } + + MumblingBitmap bitmap = bitmap(sparse(positions)); + assertEquals(31, bitmap.cardinality()); + for (int position : positions) { + assertTrue(bitmap.isSet(position)); + } + assertFalse(bitmap.isSet(1)); + assertFalse(bitmap.isSet(7)); + assertFalse(bitmap.isSet(255)); + } + + @Test + public void testFullDenseContainer() { + byte[] container = new byte[32]; + Arrays.fill(container, (byte) 0xFF); + + MumblingBitmap bitmap = bitmap(dense(container)); + assertEquals(256, bitmap.cardinality()); + for (int i = 0; i < 256; i += 1) { + assertTrue(bitmap.isSet(i)); + } + assertFalse(bitmap.isSet(256)); + } + + @Test + public void testDenseSpecExample1() { + byte[] container = new byte[32]; + Arrays.fill(container, 0, 4, (byte) 0xFF); + MumblingBitmap bitmap = bitmap(dense(container)); + assertEquals(32, bitmap.cardinality()); + + for (int i = 0; i <= 31; i += 1) { + assertTrue(bitmap.isSet(i)); + } + assertFalse(bitmap.isSet(32)); + assertFalse(bitmap.isSet(255)); + } + + @Test + public void testDenseSpecExample2() { + byte[] container = new byte[32]; + Arrays.fill(container, 0, 4, (byte) 0xFF); + container[4] = (byte) 0x80; + MumblingBitmap bitmap = bitmap(dense(container)); + assertEquals(33, bitmap.cardinality()); + + for (int i = 0; i <= 32; i += 1) { + assertTrue(bitmap.isSet(i)); + } + assertFalse(bitmap.isSet(33)); + assertFalse(bitmap.isSet(255)); + } + + @Test + public void testDenseSpecExample3() { + byte[] container = new byte[32]; + container[0] = (byte) 0xFF; + container[1] = (byte) 0xFF; + container[30] = (byte) 0xFF; + container[31] = (byte) 0xFF; + MumblingBitmap bitmap = bitmap(dense(container)); + assertEquals(32, bitmap.cardinality()); + + for (int i = 0; i <= 15; i += 1) { + assertTrue(bitmap.isSet(i)); + } + for (int i = 240; i <= 255; i += 1) { + assertTrue(bitmap.isSet(i)); + } + assertFalse(bitmap.isSet(16)); + assertFalse(bitmap.isSet(239)); + assertFalse(bitmap.isSet(256)); + } + + @Test + public void testDenseSpecExample4() { + byte[] container = new byte[32]; + Arrays.fill(container, (byte) 0xAA); + MumblingBitmap bitmap = bitmap(dense(container)); + assertEquals(128, bitmap.cardinality()); + + for (int i = 0; i < 256; i += 1) { + assertEquals(i % 2 == 0, bitmap.isSet(i)); + } + assertFalse(bitmap.isSet(256)); + } + + @Test + public void testMultipleContainers() { + MumblingBitmap bitmap = bitmap(sparse(5), sparse(), sparse(10)); + assertEquals(2, bitmap.cardinality()); + assertTrue(bitmap.isSet(5)); + assertFalse(bitmap.isSet(256)); + assertTrue(bitmap.isSet(522)); + assertFalse(bitmap.isSet(512)); + assertFalse(bitmap.isSet(4)); + assertFalse(bitmap.isSet(265)); + assertFalse(bitmap.isSet(267)); + } + + @Test + public void testMixedSparseAndDense() { + byte[] denseContainer = new byte[32]; + Arrays.fill(denseContainer, 0, 4, (byte) 0xFF); + MumblingBitmap bitmap = bitmap(dense(denseContainer), sparse(1)); + assertEquals(33, bitmap.cardinality()); + + for (int i = 0; i < 32; i += 1) { + assertTrue(bitmap.isSet(i)); + } + assertFalse(bitmap.isSet(32)); + assertFalse(bitmap.isSet(256)); + assertTrue(bitmap.isSet(257)); + assertFalse(bitmap.isSet(258)); + } + + @Test + public void testMixedSparseAndDenseWithPFORException() { + byte[] denseContainer = new byte[32]; + Arrays.fill(denseContainer, 0, 4, (byte) 0xFF); + + MumblingBitmap bitmap = + bitmap( + sparse(0), + sparse(1), + sparse(2), + sparse(3), + sparse(4), + sparse(5), + sparse(6), + dense(denseContainer), + sparse(8), + sparse(9), + sparse(10), + sparse(11), + sparse(12), + sparse(13), + sparse(14)); + assertEquals(46, bitmap.cardinality()); + + for (int i = 0; i < 15; i += 1) { + if (i != 7) { + assertTrue(bitmap.isSet(256 * i + i)); + } + } + for (int i = 0; i < 32; i += 1) { + assertTrue(bitmap.isSet(256 * 7 + i)); + } + assertFalse(bitmap.isSet(256 * 7 - 1)); + assertFalse(bitmap.isSet(256 * 7 + 32)); + } + + @Test + public void testBufferWithOffset() { + ByteBuffer buffer = build(sparse(42)); + byte[] rawBytes = new byte[buffer.remaining()]; + buffer.get(rawBytes); + + ByteBuffer padded = ByteBuffer.allocate(4 + rawBytes.length); + padded.position(4); + padded.put(rawBytes); + padded.position(4); + + MumblingBitmap bitmap = new MumblingBitmap(padded); + assertEquals(1, bitmap.cardinality()); + assertFalse(bitmap.isSet(41)); + assertTrue(bitmap.isSet(42)); + assertFalse(bitmap.isSet(43)); + } + + @Test + public void testHeadersBeyondMumblingLimitsAreRejected() { + ByteBuffer excessiveContainers = ByteBuffer.wrap(new byte[] {1, 0, 0, 0, 1, 32}); + ByteBuffer excessiveCardinality = ByteBuffer.wrap(new byte[] {1, 1, 0, 32, 0, 0}); + + assertThrows(IllegalStateException.class, () -> new MumblingBitmap(excessiveContainers)); + assertThrows(IllegalStateException.class, () -> new MumblingBitmap(excessiveCardinality)); + } + + @Test + public void testInvalidDescriptorsAreRejected() { + byte[] bytes = {1, 0, 0, 0, 1, 0, 0, 0, 64}; + MumblingBitmap bitmap = new MumblingBitmap(ByteBuffer.wrap(bytes)); + IllegalStateException error = assertThrows(IllegalStateException.class, () -> bitmap.isSet(0)); + assertTrue(error.getMessage().contains("Invalid descriptor")); + } + + @Test + public void testDenseDescriptorLowBitsAreIgnored() { + byte[] bytes = new byte[6 + 3 + 32]; + bytes[0] = 1; + bytes[1] = 1; + bytes[4] = 1; + bytes[8] = 33; + bytes[9] = (byte) 0x80; + + MumblingBitmap bitmap = new MumblingBitmap(ByteBuffer.wrap(bytes)); + assertEquals(1, bitmap.cardinality()); + assertTrue(bitmap.isSet(0)); + assertFalse(bitmap.isSet(1)); + } + + private static Container sparse(int... positions) { + byte[] bytes = new byte[positions.length]; + for (int i = 0; i < positions.length; i += 1) { + Preconditions.checkArgument( + positions[i] < 256, "Invalid position in container: %s", positions[i]); + if (i > 0) { + Preconditions.checkArgument( + positions[i] > positions[i - 1], + "Invalid sparse container: pos %s=%s >= pos %s=%s", + i - 1, + positions[i - 1], + i, + positions[i]); + } + bytes[i] = (byte) positions[i]; + } + return new Container(bytes); + } + + private static Container dense(byte[] container) { + Preconditions.checkArgument(container.length == 32, "Dense container must be 32 bytes"); + return new Container(container); + } + + private static MumblingBitmap bitmap(Container... containers) { + return new MumblingBitmap(build(containers)); + } + + private static ByteBuffer build(Container... containers) { + Preconditions.checkArgument( + containers.length <= 8192, + "Invalid container count (max 8192): %s", + containers.length); + + int[] descriptors = new int[containers.length]; + int cardinality = 0; + int sizeEstimate = 6; + for (int i = 0; i < containers.length; i += 1) { + descriptors[i] = containers[i].descriptor; + cardinality += containers[i].cardinality; + sizeEstimate += containers[i].bytes.length; + } + Preconditions.checkArgument( + cardinality <= 2_097_152, + "Invalid cardinality (max 2,097,152): %s", + cardinality); + + sizeEstimate += PFOREncoding.estimateEncodedSize(containers.length); + ByteBuffer buffer = ByteBuffer.allocate(sizeEstimate); + buffer.put(0, (byte) 1); + buffer.put(1, (byte) (cardinality & 0xFF)); + buffer.put(2, (byte) ((cardinality >>> 8) & 0xFF)); + buffer.put(3, (byte) ((cardinality >>> 16) & 0xFF)); + buffer.put(4, (byte) (containers.length & 0xFF)); + buffer.put(5, (byte) ((containers.length >>> 8) & 0xFF)); + + int descriptorArraySize = + PFOREncoding.encode(descriptors, 0, buffer, 6, descriptors.length); + int containerOffset = 6 + descriptorArraySize; + ByteBuffer containerData = buffer.duplicate(); + containerData.position(containerOffset); + for (Container container : containers) { + containerData.put(container.bytes); + } + buffer.limit(containerData.position()); + return buffer; + } + + private static class Container { + private final byte[] bytes; + private final int descriptor; + private final int cardinality; + + private Container(byte[] bytes) { + this.bytes = bytes; + this.descriptor = bytes.length; + this.cardinality = cardinality(bytes); + } + + private static int cardinality(byte[] bytes) { + if (bytes.length < 32) { + return bytes.length; + } else if (bytes.length == 32) { + int setBits = 0; + for (byte value : bytes) { + setBits += Integer.bitCount(value & 0xFF); + } + Preconditions.checkArgument( + setBits > 31, + "Invalid dense container: %s values should be sparse", + setBits); + return setBits; + } else { + throw new IllegalArgumentException("Invalid container: longer than 32 bytes"); + } + } + } +} diff --git a/spark/src/test/java/org/apache/spark/sql/delta/deletionvectors/mumbling/TestPFOREncoding.java b/spark/src/test/java/org/apache/spark/sql/delta/deletionvectors/mumbling/TestPFOREncoding.java new file mode 100644 index 00000000000..08acbd8468a --- /dev/null +++ b/spark/src/test/java/org/apache/spark/sql/delta/deletionvectors/mumbling/TestPFOREncoding.java @@ -0,0 +1,154 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.deletionvectors.mumbling; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.junit.jupiter.api.Test; + +public class TestPFOREncoding { + + @Test + public void testExample1AllZeros() { + int[] values = new int[256]; + Arrays.fill(values, 0); + + ByteBuffer encoded = PFOREncoding.encode(values, values.length); + assertArrayEquals(bytes(0x00, 0x00, 0x00), toByteArray(encoded)); + assertArrayEquals(values, PFOREncoding.decode(encoded, values.length)); + } + + @Test + public void testExample2AllFives() { + int[] values = new int[51]; + Arrays.fill(values, 5); + + ByteBuffer encoded = PFOREncoding.encode(values, values.length); + assertArrayEquals(bytes(0x00, 0x00, 0x05), toByteArray(encoded)); + assertArrayEquals(values, PFOREncoding.decode(encoded, values.length)); + } + + @Test + public void testExample3SparseExceptions() { + int[] values = {0, 0, 0, 0, 0xFF, 0, 0, 0xFE}; + byte[] expected = + bytes(0x80, 0x02, 0x00, /* offsets */ 0x04, 0x07, /* exceptions */ 0xFF, 0xFE); + + ByteBuffer encoded = PFOREncoding.encode(values, values.length); + assertArrayEquals(expected, toByteArray(encoded)); + assertArrayEquals(values, PFOREncoding.decode(encoded, values.length)); + } + + @Test + public void testExample4TwoBitsNoExceptions() { + int[] values = {6, 7, 8}; + byte[] expected = bytes(0x02, 0x00, 0x06, 0x18); + + ByteBuffer encoded = PFOREncoding.encode(values, values.length); + assertArrayEquals(expected, toByteArray(encoded)); + assertArrayEquals(values, PFOREncoding.decode(encoded, values.length)); + } + + @Test + public void testExample5() { + int[] values = {6, 34, 8, 7}; + byte[] expected = bytes(0x05, 0x00, 0x06, 0x07, 0x04, 0x10); + byte[] fromSpec = bytes(0x32, 0x01, 0x06, 0x09, 0x01, 0xE0); + + ByteBuffer encoded = PFOREncoding.encode(values, values.length); + assertArrayEquals(expected, toByteArray(encoded)); + assertArrayEquals(values, PFOREncoding.decode(encoded, values.length)); + assertArrayEquals(values, PFOREncoding.decode(ByteBuffer.wrap(fromSpec), values.length)); + } + + @Test + public void testEncodeWithValueOffset() { + int[] values = new int[400]; + for (int i = 0; i < values.length; i += 1) { + values[i] = i % 251; + } + + int valueOffset = 5; + int count = 260; + ByteBuffer out = ByteBuffer.allocate(PFOREncoding.estimateEncodedSize(count)); + int bytesWritten = PFOREncoding.encode(values, valueOffset, out, 0, count); + + int[] decoded = new int[count]; + int bytesRead = PFOREncoding.decode(out, 0, decoded, 0, count); + + assertEquals(bytesWritten, bytesRead); + assertArrayEquals(Arrays.copyOfRange(values, valueOffset, valueOffset + count), decoded); + } + + @Test + public void testFullByteRangeUsesRawRepresentation() { + int[] values = new int[256]; + for (int i = 0; i < values.length; i += 1) { + values[i] = i; + } + + ByteBuffer encoded = PFOREncoding.encode(values, values.length); + byte[] actual = toByteArray(encoded); + assertEquals(259, actual.length); + assertArrayEquals(bytes(0x08, 0x00, 0x00), Arrays.copyOfRange(actual, 0, 3)); + for (int i = 0; i < values.length; i += 1) { + assertEquals((byte) values[i], actual[i + 3]); + } + assertArrayEquals(values, PFOREncoding.decode(ByteBuffer.wrap(actual), values.length)); + } + + @Test + public void testByteBufferOffsetsDoNotMovePositionOrLimit() { + int[] values = {3, 3, 4, 7, 8, 129, 130}; + ByteBuffer buffer = ByteBuffer.allocate(64); + buffer.position(7); + buffer.limit(61); + + int positionBeforeEncode = buffer.position(); + int limitBeforeEncode = buffer.limit(); + int bytesWritten = PFOREncoding.encode(values, 0, buffer, 5, values.length); + assertEquals(positionBeforeEncode, buffer.position()); + assertEquals(limitBeforeEncode, buffer.limit()); + + int[] decoded = new int[values.length]; + int positionBeforeDecode = buffer.position(); + int limitBeforeDecode = buffer.limit(); + int bytesRead = PFOREncoding.decode(buffer, 5, decoded, 0, values.length); + assertEquals(bytesWritten, bytesRead); + assertArrayEquals(values, decoded); + assertEquals(positionBeforeDecode, buffer.position()); + assertEquals(limitBeforeDecode, buffer.limit()); + } + + private static byte[] toByteArray(ByteBuffer buffer) { + ByteBuffer copy = buffer.duplicate(); + byte[] result = new byte[copy.remaining()]; + copy.get(result); + return result; + } + + private static byte[] bytes(int... values) { + byte[] result = new byte[values.length]; + for (int i = 0; i < values.length; i += 1) { + result[i] = (byte) values[i]; + } + return result; + } +} diff --git a/spark/src/test/java/org/apache/spark/sql/delta/deletionvectors/mumbling/TestPFOREncodingRandom.java b/spark/src/test/java/org/apache/spark/sql/delta/deletionvectors/mumbling/TestPFOREncodingRandom.java new file mode 100644 index 00000000000..94bcd3263dc --- /dev/null +++ b/spark/src/test/java/org/apache/spark/sql/delta/deletionvectors/mumbling/TestPFOREncodingRandom.java @@ -0,0 +1,142 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.deletionvectors.mumbling; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; + +import java.nio.ByteBuffer; +import java.util.Random; +import java.util.stream.Stream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +/** Randomized round-trip tests for {@link PFOREncoding}. */ +public class TestPFOREncodingRandom { + private static final long SEED = 3546521684L; + + static Stream uniformRandomCases() { + return Stream.of( + Arguments.of("1-bit range, count=1", 1, 1), + Arguments.of("1-bit range, count=7", 7, 1), + Arguments.of("1-bit range, count=8", 8, 1), + Arguments.of("1-bit range, count=9", 9, 1), + Arguments.of("1-bit range, count=256", 256, 1), + Arguments.of("2-bit range, count=7", 7, 3), + Arguments.of("2-bit range, count=8", 8, 3), + Arguments.of("2-bit range, count=9", 9, 3), + Arguments.of("2-bit range, count=100", 100, 3), + Arguments.of("2-bit range, count=256", 256, 3), + Arguments.of("3-bit range, count=7", 7, 7), + Arguments.of("3-bit range, count=24", 24, 7), + Arguments.of("3-bit range, count=256", 256, 7), + Arguments.of("6-bit range, count=7", 7, 32), + Arguments.of("6-bit range, count=8", 8, 32), + Arguments.of("6-bit range, count=63", 63, 32), + Arguments.of("6-bit range, count=64", 64, 32), + Arguments.of("6-bit range, count=256", 256, 32), + Arguments.of("full range, count=1", 1, 255), + Arguments.of("full range, count=7", 7, 255), + Arguments.of("full range, count=8", 8, 255), + Arguments.of("full range, count=9", 9, 255), + Arguments.of("full range, count=15", 15, 255), + Arguments.of("full range, count=16", 16, 255), + Arguments.of("full range, count=100", 100, 255), + Arguments.of("full range, count=255", 255, 255), + Arguments.of("full range, count=256", 256, 255), + Arguments.of("full range, count=257", 257, 255), + Arguments.of("full range, count=512", 512, 255), + Arguments.of("full range, count=513", 513, 255)); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("uniformRandomCases") + public void testUniformRandom(String name, int count, int maxValue) { + assertRoundTrip(PFORRandomData.uniform(new Random(SEED + name.hashCode()), count, maxValue)); + } + + static Stream exceptionCases() { + return Stream.of( + Arguments.of("Exception 10%, count=7", 7, 0.10f), + Arguments.of("Exception 10%, count=8", 8, 0.10f), + Arguments.of("Exception 10%, count=9", 9, 0.10f), + Arguments.of("Exception 10%, count=100", 100, 0.10f), + Arguments.of("Exception 10%, count=256", 256, 0.10f), + Arguments.of("Exception 25%, count=256", 256, 0.25f), + Arguments.of("Exception 50%, count=256", 256, 0.50f), + Arguments.of("Exception 10%, count=512", 512, 0.10f)); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("exceptionCases") + public void testRandomWithExceptions(String name, int count, float excPercent) { + Random random = new Random(SEED + name.hashCode()); + assertRoundTrip(PFORRandomData.exceptions(random, count, excPercent)); + } + + static Stream offsetRangeCases() { + return Stream.of( + Arguments.of("offset min=100 rangeSize=3, count=8", 8, 100, 3), + Arguments.of("offset min=100 rangeSize=3, count=256", 256, 100, 3), + Arguments.of("offset min=50 rangeSize=31, count=64", 64, 50, 31), + Arguments.of("offset min=200 rangeSize=55, count=100", 100, 200, 55), + Arguments.of("offset min=128 rangeSize=127, count=256", 256, 128, 127)); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("offsetRangeCases") + public void testOffsetRandom(String name, int count, int minValue, int rangeSize) { + Random random = new Random(SEED + name.hashCode()); + int[] values = PFORRandomData.uniform(random, count, rangeSize); + for (int i = 0; i < values.length; i += 1) { + values[i] += minValue; + } + + assertRoundTrip(values); + } + + @Test + public void testFullRangeAndChunkBoundaries() { + Random random = new Random(9876L); + int[] lengths = {0, 1, 255, 256, 257, 511, 512, 1000}; + for (int length : lengths) { + assertRoundTrip(PFORRandomData.uniform(random, length, 255)); + } + } + + @Test + public void testRandomWidthsBasesAndExceptionPatterns() { + Random random = new Random(4242L); + for (int iteration = 0; iteration < 500; iteration += 1) { + int length = random.nextInt(600); + int rangeSize = 1 + random.nextInt(256); + int base = random.nextInt(256 - rangeSize + 1); + int[] values = PFORRandomData.uniform(random, length, rangeSize - 1); + for (int i = 0; i < values.length; i += 1) { + values[i] += base; + } + assertRoundTrip(values); + } + } + + private static void assertRoundTrip(int[] values) { + ByteBuffer encoded = PFOREncoding.encode(values, values.length); + int[] decoded = PFOREncoding.decode(encoded, values.length); + assertArrayEquals(values, decoded); + } +} diff --git a/spark/src/test/scala-shims/spark-4.1/org/apache/spark/sql/delta/DeltaGeoSuite.scala b/spark/src/test/scala-shims/spark-4.1/org/apache/spark/sql/delta/DeltaGeoSuite.scala index 84f7d291dbf..6b0585e42a7 100644 --- a/spark/src/test/scala-shims/spark-4.1/org/apache/spark/sql/delta/DeltaGeoSuite.scala +++ b/spark/src/test/scala-shims/spark-4.1/org/apache/spark/sql/delta/DeltaGeoSuite.scala @@ -591,7 +591,7 @@ class DeltaGeoSuite extends QueryTest "'delta.enableDeletionVectors' = 'false'" + ")") } - assert(ex.getErrorClass == "DELTA_UNIVERSAL_FORMAT_VIOLATION", + assert(ex.getErrorClass == "DELTA_UNIVERSAL_FORMAT_VIOLATION.HUDI_UNSUPPORTED_DATA_TYPE", s"Expected DELTA_UNIVERSAL_FORMAT_VIOLATION, got: ${ex.getErrorClass}") assert(ex.getMessage.toLowerCase(java.util.Locale.ROOT).contains("geometry"), s"Expected message to mention GeometryType, got: ${ex.getMessage}") @@ -607,7 +607,7 @@ class DeltaGeoSuite extends QueryTest "'delta.enableDeletionVectors' = 'false'" + ")") } - assert(ex.getErrorClass == "DELTA_UNIVERSAL_FORMAT_VIOLATION", + assert(ex.getErrorClass == "DELTA_UNIVERSAL_FORMAT_VIOLATION.HUDI_UNSUPPORTED_DATA_TYPE", s"Expected DELTA_UNIVERSAL_FORMAT_VIOLATION, got: ${ex.getErrorClass}") assert(ex.getMessage.toLowerCase(java.util.Locale.ROOT).contains("geography"), s"Expected message to mention GeographyType, got: ${ex.getMessage}") @@ -625,7 +625,7 @@ class DeltaGeoSuite extends QueryTest "'delta.enableDeletionVectors' = 'false'" + ")") } - assert(ex.getErrorClass == "DELTA_UNIVERSAL_FORMAT_VIOLATION", + assert(ex.getErrorClass == "DELTA_UNIVERSAL_FORMAT_VIOLATION.HUDI_UNSUPPORTED_DATA_TYPE", s"Expected DELTA_UNIVERSAL_FORMAT_VIOLATION, got: ${ex.getErrorClass}") } } @@ -787,7 +787,7 @@ class DeltaGeoSuite extends QueryTest "'delta.enableDeletionVectors' = 'false', " + "'delta.columnMapping.mode' = 'name')") } - assert(ex.getErrorClass == "DELTA_UNIVERSAL_FORMAT_VIOLATION", + assert(ex.getErrorClass == "DELTA_UNIVERSAL_FORMAT_VIOLATION.HUDI_UNSUPPORTED_DATA_TYPE", s"Unexpected error class: ${ex.getErrorClass}") assert(ex.getMessage.contains("hudi"), s"Expected message to mention hudi, got: ${ex.getMessage}") diff --git a/spark/src/test/scala-shims/spark-4.2/org/apache/spark/sql/delta/DeltaGeoSuite.scala b/spark/src/test/scala-shims/spark-4.2/org/apache/spark/sql/delta/DeltaGeoSuite.scala index 5b0c86c3662..90755664a69 100644 --- a/spark/src/test/scala-shims/spark-4.2/org/apache/spark/sql/delta/DeltaGeoSuite.scala +++ b/spark/src/test/scala-shims/spark-4.2/org/apache/spark/sql/delta/DeltaGeoSuite.scala @@ -606,7 +606,7 @@ class DeltaGeoSuite extends QueryTest "'delta.enableDeletionVectors' = 'false'" + ")") } - assert(ex.getErrorClass == "DELTA_UNIVERSAL_FORMAT_VIOLATION", + assert(ex.getErrorClass == "DELTA_UNIVERSAL_FORMAT_VIOLATION.HUDI_UNSUPPORTED_DATA_TYPE", s"Expected DELTA_UNIVERSAL_FORMAT_VIOLATION, got: ${ex.getErrorClass}") assert(ex.getMessage.toLowerCase(java.util.Locale.ROOT).contains("geometry"), s"Expected message to mention GeometryType, got: ${ex.getMessage}") @@ -622,7 +622,7 @@ class DeltaGeoSuite extends QueryTest "'delta.enableDeletionVectors' = 'false'" + ")") } - assert(ex.getErrorClass == "DELTA_UNIVERSAL_FORMAT_VIOLATION", + assert(ex.getErrorClass == "DELTA_UNIVERSAL_FORMAT_VIOLATION.HUDI_UNSUPPORTED_DATA_TYPE", s"Expected DELTA_UNIVERSAL_FORMAT_VIOLATION, got: ${ex.getErrorClass}") assert(ex.getMessage.toLowerCase(java.util.Locale.ROOT).contains("geography"), s"Expected message to mention GeographyType, got: ${ex.getMessage}") @@ -640,7 +640,7 @@ class DeltaGeoSuite extends QueryTest "'delta.enableDeletionVectors' = 'false'" + ")") } - assert(ex.getErrorClass == "DELTA_UNIVERSAL_FORMAT_VIOLATION", + assert(ex.getErrorClass == "DELTA_UNIVERSAL_FORMAT_VIOLATION.HUDI_UNSUPPORTED_DATA_TYPE", s"Expected DELTA_UNIVERSAL_FORMAT_VIOLATION, got: ${ex.getErrorClass}") } } @@ -1161,7 +1161,7 @@ class DeltaGeoSuite extends QueryTest "'delta.enableDeletionVectors' = 'false', " + "'delta.columnMapping.mode' = 'name')") } - assert(ex.getErrorClass == "DELTA_UNIVERSAL_FORMAT_VIOLATION", + assert(ex.getErrorClass == "DELTA_UNIVERSAL_FORMAT_VIOLATION.HUDI_UNSUPPORTED_DATA_TYPE", s"Unexpected error class: ${ex.getErrorClass}") assert(ex.getMessage.contains("hudi"), s"Expected message to mention hudi, got: ${ex.getMessage}") diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/ActionSerializerSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/ActionSerializerSuite.scala index a543aea7fa6..558a3a98e60 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/ActionSerializerSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/ActionSerializerSuite.scala @@ -21,6 +21,7 @@ import java.util.UUID import org.apache.spark.sql.delta.DeltaOperations.ManualUpdate import org.apache.spark.sql.delta.actions._ import org.apache.spark.sql.delta.actions.TableFeatureProtocolUtils.{TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION} +import org.apache.spark.sql.delta.amt.AMTPassthrough import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest @@ -110,7 +111,7 @@ class ActionSerializerSuite extends QueryTest with SharedSparkSession with Delta size = 1, modificationTime = 2, dataChange = false, - backReference = Some(BackReference(manifest = "metadata/leaf-1.parquet", pos = 7L))) + backReference = Some(BackReference(manifest = "metadata/leaf-1.parquet", pos = 7))) assert(withBackRef.json.contains("\"backReference\":{\"manifest\":" + "\"metadata/leaf-1.parquet\",\"pos\":7}")) assert(Action.fromJson(withBackRef.json) === withBackRef) @@ -125,7 +126,7 @@ class ActionSerializerSuite extends QueryTest with SharedSparkSession with Delta val withBackRef = RemoveFile( path = "a", deletionTimestamp = Some(1L), - backReference = Some(BackReference(manifest = "metadata/leaf-2.parquet", pos = 3L))) + backReference = Some(BackReference(manifest = "metadata/leaf-2.parquet", pos = 3))) assert(withBackRef.json.contains("\"backReference\":{\"manifest\":" + "\"metadata/leaf-2.parquet\",\"pos\":3}")) assert(Action.fromJson(withBackRef.json) === withBackRef) @@ -136,7 +137,7 @@ class ActionSerializerSuite extends QueryTest with SharedSparkSession with Delta } test("AddFile back reference propagates to removeWithTimestamp and copy") { - val backRef = Some(BackReference(manifest = "metadata/leaf-3.parquet", pos = 11L)) + val backRef = Some(BackReference(manifest = "metadata/leaf-3.parquet", pos = 11)) val add = AddFile("a", Map.empty, 1, 2, dataChange = true, backReference = backRef) // removeWithTimestamp -> tombstone inherits the back reference. assert(add.removeWithTimestamp().backReference == backRef) @@ -144,6 +145,40 @@ class ActionSerializerSuite extends QueryTest with SharedSparkSession with Delta assert(add.copy(dataChange = false).backReference == backRef) } + test("AddFile - amtPassthrough json serialization/deserialization") { + val withPassthrough = AddFile( + path = "a", + partitionValues = Map.empty, + size = 1, + modificationTime = 2, + dataChange = false, + amtPassthrough = Some(AMTPassthrough(spec_id = Some(7)))) + assert(withPassthrough.json.contains("\"amtPassthrough\":{\"spec_id\":7}")) + assert(Action.fromJson(withPassthrough.json) === withPassthrough) + + // Absent (default None) -> omitted from the serialized action. + val withoutPassthrough = AddFile("a", Map.empty, 1, 2, dataChange = false) + assert(!withoutPassthrough.json.contains("amtPassthrough")) + assert(Action.fromJson(withoutPassthrough.json).asInstanceOf[AddFile].amtPassthrough.isEmpty) + + // A present-but-empty passthrough keeps the struct, with its own field omitted. + val emptyPassthrough = AddFile( + "a", Map.empty, 1, 2, dataChange = false, amtPassthrough = Some(AMTPassthrough())) + assert(emptyPassthrough.json.contains("\"amtPassthrough\":{}")) + assert(Action.fromJson(emptyPassthrough.json) === emptyPassthrough) + } + + test("AddFile amtPassthrough propagates to copy but not to removeWithTimestamp") { + val passthrough = Some(AMTPassthrough(spec_id = Some(7))) + val add = AddFile("a", Map.empty, 1, 2, dataChange = true, amtPassthrough = passthrough) + // copy (the mechanism the superseding AddFile of removeRows relies on) preserves it. + assert(add.copy(dataChange = false).amtPassthrough == passthrough) + // Unlike backReference, RemoveFile has no amtPassthrough field: the tombstone drops it. The + // surviving AddFile is what carries the AMT-native state forward. + assert(add.removeWithTimestamp().json.contains("\"remove\"")) + assert(!add.removeWithTimestamp().json.contains("amtPassthrough")) + } + // This is the same test as "removefile" in OSS, but due to a Jackson library upgrade the behavior // has diverged between Spark 3.1 and Spark 3.2. // We don't believe this is a practical issue because all extant versions of Delta explicitly @@ -190,17 +225,21 @@ class ActionSerializerSuite extends QueryTest with SharedSparkSession with Delta readVersion = Some(23), isolationLevel = Some("SnapshotIsolation"), isBlindAppend = Some(true), + dataChange = None, operationMetrics = Some(Map("m1" -> "v1", "m2" -> "v2")), userMetadata = Some("123"), tags = None, - txnId = None).copy(engineInfo = None) + txnId = None, + lastManifestCommit = Some(LastManifestCommit(version = 43, contentRootVersion = 41)) + ).copy(engineInfo = None) // json of commit info actions without tag or engineInfo field val json1 = """{"commitInfo":{"inCommitTimestamp":123,"timestamp":123,"operation":"CONVERT",""" + """"operationParameters":{},"readVersion":23,""" + """"isolationLevel":"SnapshotIsolation","isBlindAppend":true,""" + - """"operationMetrics":{"m1":"v1","m2":"v2"},"userMetadata":"123"}}""".stripMargin + """"operationMetrics":{"m1":"v1","m2":"v2"},"userMetadata":"123",""" + + """"lastManifestCommit":{"version":43,"contentRootVersion":41}}}""".stripMargin assert(Action.fromJson(json1) === expectedCommitInfo) } @@ -213,17 +252,21 @@ class ActionSerializerSuite extends QueryTest with SharedSparkSession with Delta readVersion = Some(23), isolationLevel = Some("SnapshotIsolation"), isBlindAppend = Some(true), + dataChange = None, operationMetrics = Some(Map("m1" -> "v1", "m2" -> "v2")), userMetadata = Some("123"), tags = None, - txnId = None).copy(engineInfo = None) + txnId = None, + lastManifestCommit = Some(LastManifestCommit(version = 43, contentRootVersion = 41)) + ).copy(engineInfo = None) // json of commit info actions without tag or engineInfo field val json1 = """{"commitInfo":{"timestamp":123,"operation":"CONVERT",""" + """"operationParameters":{},"readVersion":23,""" + """"isolationLevel":"SnapshotIsolation","isBlindAppend":true,""" + - """"operationMetrics":{"m1":"v1","m2":"v2"},"userMetadata":"123"}}""".stripMargin + """"operationMetrics":{"m1":"v1","m2":"v2"},"userMetadata":"123",""" + + """"lastManifestCommit":{"version":43,"contentRootVersion":41}}}""".stripMargin assert(Action.fromJson(json1) === expectedCommitInfo) } @@ -488,7 +531,7 @@ class ActionSerializerSuite extends QueryTest with SharedSparkSession with Delta """"extendedFileMetadata":true,"partitionValues":{"x":"2"},"size":10}}""") private def deletionVectorWithRelativePath: DeletionVectorDescriptor = - DeletionVectorDescriptor.onDiskWithRelativePath( + DeletionVectorDescriptor.onDiskWithUuidRelativePath( id = UUID.randomUUID(), randomPrefix = "a1", sizeInBytes = 10, @@ -744,10 +787,12 @@ class ActionSerializerSuite extends QueryTest with SharedSparkSession with Delta readVersion = Some(23), isolationLevel = Some("SnapshotIsolation"), isBlindAppend = Some(true), + dataChange = None, operationMetrics = Some(Map("m1" -> "v1", "m2" -> "v2")), userMetadata = Some("123"), tags = Some(Map("k1" -> "v1")), - txnId = Some("123") + txnId = Some("123"), + lastManifestCommit = Some(LastManifestCommit(version = 43, contentRootVersion = 41)) ).copy(engineInfo = None) testActionSerDe( @@ -758,7 +803,8 @@ class ActionSerializerSuite extends QueryTest with SharedSparkSession with Delta """"operationParameters":{},"clusterId":"23","readVersion":23,""" + """"isolationLevel":"SnapshotIsolation","isBlindAppend":true,""" + """"operationMetrics":{"m1":"v1","m2":"v2"},"userMetadata":"123",""" + - """"tags":{"k1":"v1"},"txnId":"123"}}""".stripMargin) + """"tags":{"k1":"v1"},"txnId":"123",""" + + """"lastManifestCommit":{"version":43,"contentRootVersion":41}}}""".stripMargin) test("CommitInfo (with operationParameters) - json serialization/deserialization") { val operation = DeltaOperations.Convert( @@ -776,7 +822,8 @@ class ActionSerializerSuite extends QueryTest with SharedSparkSession with Delta """"sourceFormat":"parquet","collectStats":false},"clusterId":"23","readVersion"""" + """:23,"isolationLevel":"SnapshotIsolation","isBlindAppend":true,""" + """"operationMetrics":{"m1":"v1","m2":"v2"},""" + - """"userMetadata":"123","tags":{"k1":"v1"},"txnId":"123"}}""" + """"userMetadata":"123","tags":{"k1":"v1"},"txnId":"123",""" + + """"lastManifestCommit":{"version":43,"contentRootVersion":41}}}""" } else { """{"commitInfo":{"inCommitTimestamp":123,""" + """"timestamp":123,"operation":"CONVERT","operationParameters"""" + @@ -784,7 +831,8 @@ class ActionSerializerSuite extends QueryTest with SharedSparkSession with Delta """"sourceFormat":"parquet","collectStats":false},"clusterId":"23","readVersion""" + """":23,"isolationLevel":"SnapshotIsolation","isBlindAppend":true,""" + """"operationMetrics":{"m1":"v1","m2":"v2"},""" + - """"userMetadata":"123","tags":{"k1":"v1"},"txnId":"123"}}""" + """"userMetadata":"123","tags":{"k1":"v1"},"txnId":"123",""" + + """"lastManifestCommit":{"version":43,"contentRootVersion":41}}}""" } assert(commitInfo1.json == expectedCommitInfoJson1) val newCommitInfo1 = Action.fromJson(expectedCommitInfoJson1).asInstanceOf[CommitInfo] @@ -800,7 +848,8 @@ class ActionSerializerSuite extends QueryTest with SharedSparkSession with Delta """"isolationLevel":"SnapshotIsolation","isBlindAppend":true,""" + """"operationMetrics":{"m1":"v1","m2":"v2"},"userMetadata":"123",""" + """"tags":{"k1":"v1"},"engineInfo":"Apache-Spark/3.1.1 Delta-Lake/10.1.0",""" + - """"txnId":"123"}}""".stripMargin) + """"txnId":"123",""" + + """"lastManifestCommit":{"version":43,"contentRootVersion":41}}}""".stripMargin) } test("CommitInfo operationParameters deserialization with primitive types") { @@ -823,10 +872,12 @@ class ActionSerializerSuite extends QueryTest with SharedSparkSession with Delta readVersion = Some(5), isolationLevel = Some("WriteSerializable"), isBlindAppend = Some(false), + dataChange = None, operationMetrics = Some(Map("numFiles" -> "10")), userMetadata = Some("test metadata"), tags = Some(Map("source" -> "test")), - txnId = Some("txn-123") + txnId = Some("txn-123"), + lastManifestCommit = Some(LastManifestCommit(version = 43, contentRootVersion = 41)) ) // Serialize and deserialize to test the JsonMapDeserializer @@ -862,10 +913,12 @@ class ActionSerializerSuite extends QueryTest with SharedSparkSession with Delta readVersion = Some(42), isolationLevel = Some("WriteSerializable"), isBlindAppend = Some(true), + dataChange = None, operationMetrics = Some(Map("numFiles" -> "25", "numOutputRows" -> "1000")), userMetadata = operation.userMetadata, tags = Some(Map("environment" -> "production", "team" -> "data-eng")), - txnId = Some("txn-write-456") + txnId = Some("txn-write-456"), + lastManifestCommit = Some(LastManifestCommit(version = 43, contentRootVersion = 41)) ) // Test round-trip serialization/deserialization diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/AutoCompactSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/AutoCompactSuite.scala index ed9ca3ed997..65027afbd83 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/AutoCompactSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/AutoCompactSuite.scala @@ -162,6 +162,29 @@ class AutoCompactExecutionSuite extends import testImplicits._ + test("auto compact is skipped for a catalog-managed table") { + withCatalogManagedTable() { tableName => + val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) + val versionBefore = deltaLog.update().version + + val usageLogs = withSQLConf( + DeltaSQLConf.DELTA_AUTO_COMPACT_ENABLED.key -> "true", + DeltaSQLConf.DELTA_AUTO_COMPACT_MIN_NUM_FILES.key -> "0") { + captureOptimizeLogs(AutoCompact.OP_TYPE) { + spark.range(10) + .selectExpr("CAST(id AS INT) AS id") + .write + .mode("append") + .insertInto(tableName) + } + } + + assert(usageLogs.isEmpty) + assert(deltaLog.update().version === versionBefore + 1) + assert(spark.table(tableName).count() === 10) + } + } + test("auto compact event log: inline AC") { withTempDir { dir => withSQLConf( diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/CheckpointInstanceSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/CheckpointInstanceSuite.scala index 38a384e403c..62973aa9c5c 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/CheckpointInstanceSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/CheckpointInstanceSuite.scala @@ -25,10 +25,14 @@ class CheckpointInstanceSuite extends SparkFunSuite { test("checkpoint instance comparisons") { val ci1_single_1 = CheckpointInstance(1, Format.SINGLE, numParts = None) val ci1_withparts_2 = CheckpointInstance(1, Format.WITH_PARTS, numParts = Some(2)) + val ci1_amt_2 = CheckpointInstance( + 1, Format.AMT, numParts = Some(2), manifestCommitVersion = Some(1)) val ci1_sentinel = CheckpointInstance.sentinelValue(Some(1)) val ci2_single_1 = CheckpointInstance(2, Format.SINGLE, numParts = None) val ci2_withparts_4 = CheckpointInstance(2, Format.WITH_PARTS, numParts = Some(4)) + val ci2_amt_3 = CheckpointInstance( + 2, Format.AMT, numParts = Some(3), manifestCommitVersion = Some(2)) val ci2_sentinel = CheckpointInstance.sentinelValue(Some(2)) val ci3_single_1 = CheckpointInstance(3, Format.SINGLE, numParts = None) @@ -37,10 +41,20 @@ class CheckpointInstanceSuite extends SparkFunSuite { assert(ci1_single_1 < ci2_single_1) // version takes priority assert(ci1_single_1 < ci1_withparts_2) // parts takes priority when versions are same assert(ci2_withparts_4 < ci3_withparts_2) // version takes priority over parts + // An AMT checkpoint has the highest priority among checkpoints of the same version. + assert(ci1_amt_2 > ci1_withparts_2 && ci1_amt_2 > ci1_single_1) + assert(ci2_amt_3 > ci2_withparts_4 && ci2_amt_3 > ci2_single_1) + // AMT still yields to the Sentinel value, and to any checkpoint of a later version. + assert(ci1_amt_2 < ci1_sentinel) + assert(ci1_amt_2 < ci2_single_1) + // AMT reuses `numParts` for its leaf count, and carries no `fileName`. + assert(ci1_amt_2.numParts == Some(2) && ci1_amt_2.fileName.isEmpty) // all checkpoint instances for version 1/2 are less than sentinel value for version 2. - Seq(ci1_single_1, ci1_withparts_2, ci1_sentinel, ci2_single_1, ci2_withparts_4) - .foreach(ci => assert(ci < ci2_sentinel)) + Seq( + ci1_single_1, ci1_withparts_2, ci1_sentinel, ci2_single_1, ci2_withparts_4, + ci1_amt_2, ci2_amt_3 + ).foreach(ci => assert(ci < ci2_sentinel)) // all checkpoint instances for version 3 are greater than sentinel value for version 2. Seq(ci3_single_1, ci3_withparts_2).foreach(ci => assert(ci > ci2_sentinel)) @@ -49,7 +63,8 @@ class CheckpointInstanceSuite extends SparkFunSuite { Seq( ci1_single_1, ci1_withparts_2, ci1_sentinel, ci2_single_1, ci2_withparts_4, ci2_sentinel, - ci3_single_1, ci3_withparts_2 + ci3_single_1, ci3_withparts_2, + ci1_amt_2, ci2_amt_3 ).foreach(ci => assert(ci < CheckpointInstance.MaxValue)) } } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/CheckpointsSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/CheckpointsSuite.scala index 1f1ae7ea622..9199bb1c672 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/CheckpointsSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/CheckpointsSuite.scala @@ -26,7 +26,8 @@ import scala.concurrent.duration._ import com.databricks.spark.util.{Log4jUsageLogger, MetricDefinitions, UsageRecord} import org.apache.spark.sql.delta.DeltaTestUtils.createTestAddFile import org.apache.spark.sql.delta.actions._ -import org.apache.spark.sql.delta.coordinatedcommits.CatalogOwnedTestBaseSuite +import org.apache.spark.sql.delta.amt.AMTPassthrough +import org.apache.spark.sql.delta.coordinatedcommits.{CatalogManagedMaintenanceIncompatible, CatalogOwnedTestBaseSuite} import org.apache.spark.sql.delta.deletionvectors.DeletionVectorsSuite import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.storage.LocalLogStore @@ -723,7 +724,7 @@ class CheckpointsSuite } } - test("non-AMT tables do not persist backReference in checkpoint or commit JSON") { + test("non-AMT tables do not persist backReference or amtPassthrough in checkpoint or commit") { withClassicCheckpointPolicyForCatalogOwned { withTempDir { tempDir => val tablePath = tempDir.getAbsolutePath @@ -734,7 +735,7 @@ class CheckpointsSuite deltaLog.checkpoint() val version = deltaLog.snapshot.version - // 1. The classic checkpoint parquet must not carry backReference on add or remove. + // 1. The classic checkpoint parquet must carry neither backReference nor amtPassthrough. val checkpointFile = FileNames.checkpointFileSingular(deltaLog.logPath, version).toString val checkpointSchema = spark.read.format("parquet").load(checkpointFile).schema @@ -746,8 +747,10 @@ class CheckpointsSuite s"checkpoint add struct must not contain backReference, got: $addFields") assert(!removeFields.contains("backReference"), s"checkpoint remove struct must not contain backReference, got: $removeFields") + assert(!addFields.contains("amtPassthrough"), + s"checkpoint add struct must not contain amtPassthrough, got: $addFields") - // 2. No commit JSON should mention backReference (null fields are omitted from JSON). + // 2. No commit JSON should mention backReference or amtPassthrough. val commitProvider = DeltaCommitFileProvider(deltaLog.unsafeVolatileSnapshot) val hadoopConf = deltaLog.newDeltaHadoopConf() (0L to version).foreach { v => @@ -755,6 +758,9 @@ class CheckpointsSuite val content = deltaLog.store.read(deltaFile, hadoopConf) assert(!content.exists(_.contains("backReference")), s"commit JSON for version $v must not contain backReference: ${content.mkString("\n")}") + assert(!content.exists(_.contains("amtPassthrough")), + s"commit JSON for version $v must not contain amtPassthrough: " + + s"${content.mkString("\n")}") } } } @@ -1086,10 +1092,12 @@ class CheckpointsSuite val actionsToWrite = Checkpoints .buildCheckpoint(actionsDS, snapshot) - // buildCheckpoint drops backReference from the on-disk add/remove shape so we need to - // re-materialize it as a null column before .as[SingleAction]. + // buildCheckpoint drops backReference (add/remove) and amtPassthrough (add only) from the + // on-disk shape, so we re-materialize them as null columns before .as[SingleAction]. .withColumn("add", when(col("add.path").isNotNull, - col("add").withField("backReference", lit(null).cast(BackReference.STRUCT_TYPE)))) + col("add") + .withField("backReference", lit(null).cast(BackReference.STRUCT_TYPE)) + .withField("amtPassthrough", lit(null).cast(AMTPassthrough.STRUCT_TYPE)))) .withColumn("remove", when(col("remove.path").isNotNull, col("remove").withField("backReference", lit(null).cast(BackReference.STRUCT_TYPE)))) .as[SingleAction] @@ -1199,7 +1207,9 @@ class CheckpointsSuite } } - test("validate metadata cleanup is not called with createCheckpointAtVersion API") { + test( + "validate metadata cleanup is not called with createCheckpointAtVersion API", + CatalogManagedMaintenanceIncompatible) { withTempDir { dir => val usageRecords1 = Log4jUsageLogger.track { spark.range(10).write.format("delta").save(dir.getAbsolutePath) diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/ChecksumSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/ChecksumSuite.scala index a47c9d76586..cc4bc89adec 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/ChecksumSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/ChecksumSuite.scala @@ -394,6 +394,171 @@ class ChecksumSuite // A CRC written before this field existed deserializes to None. assert(JsonUtils.fromJson[VersionChecksum](jsonWithoutLmc).lastManifestCommit.isEmpty) } + + /** Loads a fresh snapshot of `tableName`, bypassing any cached DeltaLog/Snapshot. */ + private def freshSnapshot(tableName: String): Snapshot = { + DeltaLog.clearCache() + DeltaLog.forTable(spark, TableIdentifier(tableName)).update() + } + + private def withCrcTable(f: String => Unit): Unit = { + withSQLConf( + DeltaSQLConf.DELTA_WRITE_CHECKSUM_ENABLED.key -> "true", + DeltaSQLConf.INCREMENTAL_COMMIT_ENABLED.key -> "true", + DeltaSQLConf.DELTA_WRITE_SET_TRANSACTIONS_IN_CRC.key -> "true" + ) { + withTempTable(createTable = false)(f) + } + } + + test("Snapshot state fields are served from the CRC without state reconstruction") { + withCrcTable { tableName => + spark.range(10).withColumn("id2", col("id") % 2) + .write.format("delta").partitionBy("id").saveAsTable(tableName) + sql(s"INSERT INTO $tableName SELECT *, 1 FROM range(10, 20)") + + val s = freshSnapshot(tableName) + val crc = s.checksumOpt.getOrElse(fail("test setup should have produced a CRC file")) + + // Fields the CRC carries are served from it, so no state reconstruction is triggered. + assert(s.sizeInBytes == crc.tableSizeBytes) + assert(s.numOfFiles == crc.numFiles) + assert(s.numOfMetadata == crc.numMetadata) + assert(s.numOfProtocol == crc.numProtocol) + assert(s.setTransactions.toSet == crc.setTransactions.getOrElse(Nil).toSet) + assert(s.numOfSetTransactions == crc.setTransactions.getOrElse(Nil).length) + assert(s.domainMetadata.toSet == crc.domainMetadata.getOrElse(Nil).toSet) + assert(!s.stateReconstructionTriggered, + "fields available in the CRC must not trigger state reconstruction") + + // numOfRemoves is not tracked by the CRC and still needs the state reconstruction. + assert(s.numOfRemoves == 0L) + assert(s.stateReconstructionTriggered, + "numOfRemoves should fall back to state reconstruction") + } + } + + test("disabling fastQueryPath.enabled bypasses the CRC-backed snapshot state") { + withSQLConf(DeltaSQLConf.FAST_QUERY_PATH_ENABLED.key -> "false") { + withCrcTable { tableName => + spark.range(10).write.format("delta").saveAsTable(tableName) + val s = freshSnapshot(tableName) + assert(s.checksumOpt.isDefined, "test setup should have produced a CRC file") + // With the fast query path off, checksumOptForState is empty, so a CRC-backed field + // falls back to a full state reconstruction instead of being served from the checksum. + assert(s.sizeInBytes == s.checksumOpt.get.tableSizeBytes) + assert(s.stateReconstructionTriggered, + "disabling fastQueryPath.enabled must bypass the CRC and reconstruct state") + } + } + } + + test("Snapshot state falls back per field when the CRC lacks setTransactions") { + withSQLConf( + DeltaSQLConf.DELTA_WRITE_CHECKSUM_ENABLED.key -> "true", + DeltaSQLConf.INCREMENTAL_COMMIT_ENABLED.key -> "true", + // Produce a CRC that does not carry setTransactions. + DeltaSQLConf.DELTA_WRITE_SET_TRANSACTIONS_IN_CRC.key -> "false" + ) { + withTempTable(createTable = false) { tableName => + spark.range(5).write.format("delta").saveAsTable(tableName) + val s = freshSnapshot(tableName) + assert(s.checksumOpt.isDefined) + assert(s.checksumOpt.get.setTransactions.isEmpty, + "test setup should produce a CRC without setTransactions") + + // Fields the CRC does carry are still served from it. + assert(s.numOfFiles == s.checksumOpt.get.numFiles) + assert(s.sizeInBytes == s.checksumOpt.get.tableSizeBytes) + assert(!s.stateReconstructionTriggered, + "a CRC missing one field must not disable the fast path for the other fields") + + // Only the missing field falls back. + assert(s.setTransactions.isEmpty) + assert(s.stateReconstructionTriggered, + "setTransactions should fall back to state reconstruction") + } + } + } + + test("setTransactions are dropped from the CRC once a retention period is configured") { + withCrcTable { tableName => + spark.range(5).write.format("delta").saveAsTable(tableName) + sql(s"ALTER TABLE $tableName SET TBLPROPERTIES " + + s"('delta.setTransactionRetentionDuration' = '30 days')") + + val s = freshSnapshot(tableName) + assert(s.minSetTransactionRetentionTimestamp.isDefined) + + // The write path stops recording setTransactions in the CRC as soon as a retention + // period is configured, so a CRC can never hand out entries that state reconstruction + // would have filtered out and no extra guard is needed when reading. + assert(s.checksumOpt.get.setTransactions.isEmpty, + "the CRC must not carry setTransactions once a retention period is configured") + + // Other fields are still served from the CRC. + assert(s.numOfFiles == s.checksumOpt.get.numFiles) + assert(!s.stateReconstructionTriggered) + + // setTransactions therefore falls back to state reconstruction. + assert(s.setTransactions.isEmpty) + assert(s.stateReconstructionTriggered, + "setTransactions should fall back when the CRC does not carry them") + } + } + + test("DV metrics are served from the CRC when deletion vectors are readable") { + withCrcTable { tableName => + // Table with deletion vectors enabled, so the CRC carries the DV metrics. + sql(s"CREATE TABLE $tableName (id LONG) USING delta " + + s"TBLPROPERTIES ('delta.enableDeletionVectors' = 'true')") + sql(s"INSERT INTO $tableName SELECT * FROM range(10)") + sql(s"DELETE FROM $tableName WHERE id = 1") + + val s = freshSnapshot(tableName) + assert(s.checksumOpt.flatMap(_.numDeletedRecordsOpt).contains(1L), + "test setup should produce a CRC carrying the DV metrics") + assert(s.numDeletedRecordsOpt.contains(1L)) + assert(s.numDeletionVectorsOpt.contains(1L)) + assert(!s.stateReconstructionTriggered, + "DV metrics present in the CRC must not trigger state reconstruction") + } + } + + test("validateChecksum still compares the CRC against state reconstruction") { + withSQLConf( + DeltaSQLConf.DELTA_WRITE_CHECKSUM_ENABLED.key -> "true", + DeltaSQLConf.INCREMENTAL_COMMIT_ENABLED.key -> "true", + DeltaSQLConf.DELTA_CHECKSUM_MISMATCH_IS_FATAL.key -> "false" + ) { + withTempTable(createTable = false) { tableName => + spark.range(5).write.format("delta").saveAsTable(tableName) + val log = DeltaLog.forTable(spark, TableIdentifier(tableName)) + val version = log.update().version + val checksum = log.readChecksum(version).get + // Corrupt only the aggregate fields, leaving protocol/metadata intact. + log.store.write( + FileNames.checksumFile(log.logPath, version), + Iterator(JsonUtils.toJson(checksum.copy( + tableSizeBytes = checksum.tableSizeBytes + 1, + numFiles = checksum.numFiles + 1))), + overwrite = true) + + val s = freshSnapshot(tableName) + // Even though the accessors are now served from the (corrupted) CRC, validation must + // still compare it against an independently computed state and detect the mismatch. + assert(s.numOfFiles == checksum.numFiles + 1) + val usageLogs = Log4jUsageLogger.track { + assert(!s.validateChecksum(), "validation should have failed for a corrupted CRC") + } + val mismatches = filterUsageRecords(usageLogs, "delta.checksum.invalid") + assert(mismatches.size == 1) + val blob = JsonUtils.fromJson[Map[String, Any]](mismatches.head.blob) + val fields = blob("mismatchingFields").asInstanceOf[Seq[String]].toSet + assert(fields == Set("tableSizeBytes", "numFiles")) + } + } + } } class ChecksumWithCatalogOwnedBatch1Suite extends ChecksumSuite { diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/CloneTableTestMixin.scala b/spark/src/test/scala/org/apache/spark/sql/delta/CloneTableTestMixin.scala index 52179770dc3..a551cd959a4 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/CloneTableTestMixin.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/CloneTableTestMixin.scala @@ -103,11 +103,18 @@ trait CloneTableTestMixin extends DeltaColumnMappingTestUtils } // Extracted function so it can be overriden in subclasses. - protected def uniqueFileActionGroupBy(action: FileAction): String = { + protected def uniqueFileActionGroupBy( + action: FileAction, tableRoot: Path, useObjectIdentity: Boolean): String = { val filePath = action.pathAsUri.toString val dvId = action match { - case add: AddFile => Option(add.deletionVector).map(_.uniqueId).getOrElse("") - case remove: RemoveFile => Option(remove.deletionVector).map(_.uniqueId).getOrElse("") + case add: AddFile => + Option(add.deletionVector) + .map(_.uniqueId(tableRoot, useObjectIdentity)) + .getOrElse("") + case remove: RemoveFile => + Option(remove.deletionVector) + .map(_.uniqueId(tableRoot, useObjectIdentity)) + .getOrElse("") case _ => "" } filePath + dvId @@ -250,7 +257,13 @@ trait CloneTableTestMixin extends DeltaColumnMappingTestUtils case _ => None } } - assert(filePaths.groupBy(uniqueFileActionGroupBy(_)).forall(_._2.length === 1), + val useObjectIdentity = FileAction.useDeletionVectorObjectIdentity( + targetLog.unsafeVolatileSnapshot.metadata, + targetLog.unsafeVolatileSnapshot.protocol, + spark + ) + assert(filePaths.groupBy(uniqueFileActionGroupBy(_, targetLog.dataPath, useObjectIdentity)) + .forall(_._2.length === 1), "A file was added and removed in the same commit") // Check whether the resulting datasets are the same val targetDf = DataFrameUtils.ofRows( diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/CommitInfoDataChangeSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/CommitInfoDataChangeSuite.scala new file mode 100644 index 00000000000..1ccb6c1d83f --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/CommitInfoDataChangeSuite.scala @@ -0,0 +1,233 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta + +import org.apache.spark.sql.delta.actions.{Action, AddCDCFile, AddFile, CommitInfo, Metadata, Protocol, SetTransaction} +import org.apache.spark.sql.delta.sources.DeltaSQLConf +import org.apache.spark.sql.delta.test.DeltaSQLCommandTest + +import org.apache.spark.sql.QueryTest +import org.apache.spark.sql.test.SharedSparkSession + +/** + * Tests that the `dataChange` recorded in a commit's [[CommitInfo]] matches the file actions the + * commit actually contains, for the normal transaction path and commitLarge path. + */ +class CommitInfoDataChangeSuite + extends QueryTest + with SharedSparkSession + with DeltaSQLCommandTest + with DeltaTableProvider { + + private def addFile(path: String, dataChange: Boolean): AddFile = + AddFile(path, Map.empty, size = 1L, modificationTime = 1L, dataChange = dataChange) + + /** Reads the `dataChange` recorded in the [[CommitInfo]] of the given commit. */ + private def commitInfoDataChangeAt(deltaLog: DeltaLog, version: Long): Option[Boolean] = + deltaLog + .getChanges(version) + .collectFirst { case (`version`, actions) => actions } + .getOrElse(fail(s"no commit at version $version")) + .collectFirst { case ci: CommitInfo => ci } + .getOrElse(fail(s"no CommitInfo at version $version")) + .dataChange + + private def latestCommitInfoDataChange(path: String): Option[Boolean] = { + val deltaLog = DeltaLog.forTable(spark, path) + commitInfoDataChangeAt(deltaLog, deltaLog.update().version) + } + + test("dataChangeFromActions is false without file actions") { + assert(!CommitInfo.dataChangeFromActions(Nil)) + val nonFileActions: Seq[Action] = + Seq(Metadata(), Protocol(1, 2), SetTransaction("app", 1L, None)) + assert(!CommitInfo.dataChangeFromActions(nonFileActions)) + } + + test("dataChangeFromActions follows the file actions") { + assert(CommitInfo.dataChangeFromActions(Seq(addFile("a", dataChange = true)))) + assert(!CommitInfo.dataChangeFromActions(Seq(addFile("a", dataChange = false)))) + + val rearrange = addFile("a", dataChange = false) + val dataChanging = addFile("b", dataChange = true) + assert(CommitInfo.dataChangeFromActions(Seq(rearrange, dataChanging))) + assert(CommitInfo.dataChangeFromActions(Seq(dataChanging, rearrange))) + + assert(CommitInfo.dataChangeFromActions(Seq(dataChanging.remove))) + val rearrangeRemove = rearrange.removeWithTimestamp(dataChange = false) + assert(!CommitInfo.dataChangeFromActions(Seq(rearrangeRemove))) + } + + test("dataChangeFromActions ignores change data files") { + // AddCDCFile always carries dataChange = false, so a commit of only CDC files does not change + // the table's data. + val cdc = AddCDCFile("cdc", Map.empty, size = 1L) + assert(!cdc.dataChange) + assert(!CommitInfo.dataChangeFromActions(Seq(cdc))) + assert(CommitInfo.dataChangeFromActions(Seq(cdc, addFile("a", dataChange = true)))) + } + + test("commitChangedData reports the commit-level value only when it can be trusted") { + withSQLConf(DeltaSQLConf.DELTA_COMMIT_INFO_DATA_CHANGE_READ_ENABLED.key -> "true") { + assert(CommitInfo.commitChangedData(None).isEmpty) + // A commit predating the field, or written by a writer that does not populate it. + assert(CommitInfo.commitChangedData(Some(CommitInfo.empty())).isEmpty) + assert( + CommitInfo.commitChangedData(Some(CommitInfo.empty().copy(dataChange = Some(true)))) + .contains(true)) + assert( + CommitInfo.commitChangedData(Some(CommitInfo.empty().copy(dataChange = Some(false)))) + .contains(false)) + } + } + + test("commitChangedData reports nothing while the read path is disabled") { + withSQLConf(DeltaSQLConf.DELTA_COMMIT_INFO_DATA_CHANGE_READ_ENABLED.key -> "false") { + Seq(Some(true), Some(false), None).foreach { dataChange => + val commitInfo = CommitInfo.empty().copy(dataChange = dataChange) + assert(CommitInfo.commitChangedData(Some(commitInfo)).isEmpty, + s"the read path is off, so dataChange=$dataChange must not be reported") + } + } + } + + test("append records dataChange = true") { + withTempDir { dir => + val path = dir.getCanonicalPath + spark.range(5).write.format(writeFormat).save(path) + assert(latestCommitInfoDataChange(path).contains(true)) + + spark.range(5, 10).write.format(writeFormat).mode("append").save(path) + assert(latestCommitInfoDataChange(path).contains(true)) + } + } + + test("DELETE records dataChange = true") { + withTempDir { dir => + val path = dir.getCanonicalPath + spark.range(10).write.format(writeFormat).save(path) + sql(s"DELETE FROM $tableProvider.`$path` WHERE id < 5") + assert(latestCommitInfoDataChange(path).contains(true)) + } + } + + test("metadata-only commit records dataChange = false") { + withTempDir { dir => + val path = dir.getCanonicalPath + spark.range(5).write.format(writeFormat).save(path) + sql(s"ALTER TABLE $tableProvider.`$path` SET TBLPROPERTIES ('someKey' = 'someValue')") + assert(latestCommitInfoDataChange(path).contains(false)) + } + } + + test("OPTIMIZE records dataChange = false") { + withTempDir { dir => + val path = dir.getCanonicalPath + spark.range(5).repartition(1).write.format(writeFormat).save(path) + spark.range(5, 10).repartition(1).write.format(writeFormat).mode("append").save(path) + val deltaLog = DeltaLog.forTable(spark, path) + val versionBeforeOptimize = deltaLog.update().version + + sql(s"OPTIMIZE $tableProvider.`$path`") + + val optimizeVersion = deltaLog.update().version + assert(optimizeVersion > versionBeforeOptimize, "OPTIMIZE did not commit") + assert(commitInfoDataChangeAt(deltaLog, optimizeVersion).contains(false)) + } + } + + test("a rearrange-only commit records dataChange = false") { + withTempDir { dir => + val path = dir.getCanonicalPath + spark.range(5).write.format(writeFormat).save(path) + val deltaLog = DeltaLog.forTable(spark, path) + val file = deltaLog.update().allFiles.head() + + deltaLog.startTransaction().commit( + Seq(file.removeWithTimestamp(dataChange = false)), + DeltaOperations.ManualUpdate) + + assert(commitInfoDataChangeAt(deltaLog, deltaLog.update().version).contains(false)) + } + } + + test("the dataChange write option records dataChange = false") { + withTempDir { dir => + val path = dir.getCanonicalPath + spark.range(10).repartition(2).write.format(writeFormat).save(path) + val deltaLog = DeltaLog.forTable(spark, path) + val versionBeforeCompaction = deltaLog.update().version + + // The option is read as `DeltaOptions.rearrangeOnly`, which makes `WriteIntoDelta` stamp + // `dataChange = !rearrangeOnly` on every file action it hands to `txn.commit`. The commit + // then derives the CommitInfo summary from those actions. + spark.read.format(writeFormat).load(path) + .repartition(1) + .write.format(writeFormat).mode("overwrite") + .option("dataChange", "false") + .save(path) + + val compactionVersion = deltaLog.update().version + assert(compactionVersion > versionBeforeCompaction, "the compaction did not commit") + assert(commitInfoDataChangeAt(deltaLog, compactionVersion).contains(false)) + } + } + + test("an overwrite without the dataChange write option records dataChange = true") { + withTempDir { dir => + val path = dir.getCanonicalPath + spark.range(10).write.format(writeFormat).save(path) + val deltaLog = DeltaLog.forTable(spark, path) + val versionBeforeOverwrite = deltaLog.update().version + + spark.range(10, 20).write.format(writeFormat).mode("overwrite").save(path) + + val overwriteVersion = deltaLog.update().version + assert(overwriteVersion > versionBeforeOverwrite, "the overwrite did not commit") + assert(commitInfoDataChangeAt(deltaLog, overwriteVersion).contains(true)) + } + } + + /////////////////////////////////////////////////////////////////////////// + // commitLarge + /////////////////////////////////////////////////////////////////////////// + + test("CONVERT TO DELTA records dataChange = true") { + withTempPath { dir => + val path = dir.getCanonicalPath + spark.range(5).write.format("parquet").save(path) + sql(s"CONVERT TO DELTA parquet.`$path`") + assert(latestCommitInfoDataChange(path).contains(true)) + } + } + + + test("SHALLOW CLONE records dataChange = true") { + withTempDir { sourceDir => + withTempDir { targetDir => + val sourcePath = sourceDir.getCanonicalPath + val targetPath = targetDir.getCanonicalPath + spark.range(5).write.format(writeFormat).save(sourcePath) + + sql(s"CREATE OR REPLACE TABLE $tableProvider.`$targetPath` " + + s"SHALLOW CLONE $tableProvider.`$sourcePath`") + + assert(latestCommitInfoDataChange(targetPath).contains(true)) + } + } + } +} diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/CommitInfoSerializerSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/CommitInfoSerializerSuite.scala index 657f016d78f..09f85c0c287 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/CommitInfoSerializerSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/CommitInfoSerializerSuite.scala @@ -44,10 +44,12 @@ class CommitInfoSerializerSuite extends QueryTest with SharedSparkSession { readVersion = Some(23), isolationLevel = Some("SnapshotIsolation"), isBlindAppend = Some(true), + dataChange = Some(true), operationMetrics = Some(Map("m1" -> "v1", "m2" -> "v2")), userMetadata = Some("123"), tags = Some(Map("k1" -> "v1")), - txnId = Some("123") + txnId = Some("123"), + lastManifestCommit = Some(LastManifestCommit(version = 43, contentRootVersion = 41)) ).copy(engineInfo = None) val inMemoryCommitInfo = commitInfo.copy(operationParameters = operation.jsonEncodedValues) @@ -220,6 +222,63 @@ class CommitInfoSerializerSuite extends QueryTest with SharedSparkSession { ) } + private def createEmptyCommitInfo(): CommitInfo = { + CommitInfo( + version = None, + time = 0L, + operation = null, + inCommitTimestamp = None, + operationParameters = Map.empty, + commandContext = Map.empty, + readVersion = None, + isolationLevel = None, + isBlindAppend = None, + dataChange = None, + operationMetrics = None, + userMetadata = None, + tags = None, + txnId = None, + lastManifestCommit = None) + } + + test("CommitInfo round-trips lastManifestCommit and omits it when None") { + val base = createEmptyCommitInfo() + val lmf = LastManifestCommit(version = 43, contentRootVersion = 41) + + assert(!base.json.contains("lastManifestCommit")) + val roundTrippedCommitInfo = Action.fromJson(base.json).asInstanceOf[CommitInfo] + assert(roundTrippedCommitInfo.lastManifestCommit.isEmpty) + + val baseWithLmf = base.copy(lastManifestCommit = Some(lmf)) + val roundTrippedCommitInfoWithLmf = Action.fromJson(baseWithLmf.json).asInstanceOf[CommitInfo] + assert(roundTrippedCommitInfoWithLmf.lastManifestCommit.contains(lmf)) + } + + test("CommitInfo round-trips dataChange and omits it when None") { + val base = createEmptyCommitInfo() + assert(base.dataChange.isEmpty) + assert(!base.json.contains("dataChange")) + assert(Action.fromJson(base.json).asInstanceOf[CommitInfo].dataChange.isEmpty) + + Seq(true, false).foreach { dataChange => + val withDataChange = base.copy(dataChange = Some(dataChange)) + assert(withDataChange.json.contains(s""""dataChange":$dataChange""")) + val roundTripped = Action.fromJson(withDataChange.json).asInstanceOf[CommitInfo] + assert(roundTripped.dataChange.contains(dataChange)) + assert(roundTripped === withDataChange) + } + } + + test("CommitInfo written before dataChange existed deserializes to None") { + // A commit file produced by a writer that predates the field: it must keep deserializing, and + // readers must see `None` so that they fall back to inspecting the file actions. + val legacyJson = + """{"commitInfo":{"timestamp":123,"operation":"WRITE","operationParameters":{},""" + + """"readVersion":23,"isolationLevel":"WriteSerializable","isBlindAppend":true}}""" + val commitInfo = Action.fromJson(legacyJson).asInstanceOf[CommitInfo] + assert(commitInfo.dataChange.isEmpty) + assert(commitInfo.isBlindAppend.contains(true)) + } } /** diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/CommitSanityCheckSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/CommitSanityCheckSuite.scala index af50acb04a6..8711fa39ea5 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/CommitSanityCheckSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/CommitSanityCheckSuite.scala @@ -75,7 +75,8 @@ class CommitSanityCheckSuite extends QueryTest newProtocolOpt = None, op = DeltaOperations.ManualUpdate, context = Map.empty, - metrics = Map.empty + metrics = Map.empty, + dataChange = Some(true) ) } else { txn.commit(addFiles, DeltaOperations.ManualUpdate) diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/ConvertToDeltaListingRebalanceSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/ConvertToDeltaListingRebalanceSuite.scala new file mode 100644 index 00000000000..cab50750a0a --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/ConvertToDeltaListingRebalanceSuite.scala @@ -0,0 +1,158 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta + +import java.io.File + +import org.apache.spark.sql.delta.commands.convert.ParquetTable +import org.apache.spark.sql.delta.sources.DeltaSQLConf + +import org.apache.spark.SparkConf +import org.apache.spark.sql.functions._ +import org.apache.spark.sql.types.{LongType, StructType} + +/** + * Validates the CONVERT TO DELTA file-listing rebalance + * (`spark.databricks.delta.convert.rebalanceFileListing`): listed files are spread across tasks by + * file count before the footer reads, so one large partition directory is no longer a single + * skewed task. Drives `ManualListingFileManifest` directly over a skewed layout and checks the + * per-partition file distribution of `allFiles`, conf OFF vs ON. + */ +class ConvertToDeltaListingRebalanceSuite extends ConvertToDeltaSuiteBaseCommons { + + // Path order of the heterogeneous fixture (01-ab, 02-bc, 03-ac) fixes the merged column order and + // casing (A is seen before a), so the inferred schema is exactly A, b, c. + private val expectedPathOrderedSchema = new StructType() + .add("A", LongType, nullable = true) + .add("b", LongType, nullable = true) + .add("c", LongType, nullable = true) + + // recursiveListDirs uses defaultParallelism to partition; pin it so the OFF case (directory + // distribution) is deterministic and the ON case fans out to a known width. + override protected def sparkConf: SparkConf = + super.sparkConf.set("spark.default.parallelism", "8") + + override protected def convertToDelta( + identifier: String, + partitionSchema: Option[String] = None, + collectStats: Boolean = true): Unit = { + val stats = if (collectStats) "" else " NO STATISTICS" + partitionSchema match { + case Some(ps) => spark.sql(s"CONVERT TO DELTA $identifier PARTITIONED BY ($ps)$stats") + case None => spark.sql(s"CONVERT TO DELTA $identifier$stats") + } + } + + // 60 one-row files: country=US has 45 (the dominant dir), country=CA has 15. + private def writeSkewedParquet(dir: String): Unit = { + withSQLConf("spark.sql.files.maxRecordsPerFile" -> "1") { + spark.range(60) + .withColumn("country", when(col("id") < 45, lit("US")).otherwise(lit("CA"))) + .withColumn("yyyy", lit("2024")) + .withColumn("mm", lit("01")) + .repartition(4) + .write.mode("overwrite") + .partitionBy("country", "yyyy", "mm") + .parquet(dir) + } + } + + private def perPartitionFileCounts(dir: String, rebalance: Boolean): Seq[Int] = { + withSQLConf(DeltaSQLConf.DELTA_CONVERT_REBALANCE_FILE_LISTING.key -> rebalance.toString) { + // Go through ParquetTable so the manifest gets the Hadoop conf that + // ParquetToSparkSchemaConverter needs (the parquet SQLConf booleans); it returns a + // ManualListingFileManifest for an unpartitioned-catalog, non-metadata-log parquet path. + val manifest = new ParquetTable( + spark, dir, catalogTable = None, userPartitionSchema = None).fileManifest + try { + manifest.allFiles.rdd + .mapPartitions(it => Iterator(it.size)) + .collect().toSeq.filter(_ > 0) + } finally { + manifest.close() + } + } + } + + test("rebalance spreads footer-read work across tasks (no single skewed task)") { + withTempDir { tmp => + val dir = new File(tmp, "src").getCanonicalPath + writeSkewedParquet(dir) + + val off = perPartitionFileCounts(dir, rebalance = false) + val on = perPartitionFileCounts(dir, rebalance = true) + + assert(off.sum == 60 && on.sum == 60, s"file counts must match: off=${off.sum} on=${on.sum}") + // Without rebalance, the US directory (45 files) is a single skewed partition. + assert(off.max >= 40, s"expected directory skew without rebalance, max=${off.max}") + // With rebalance, no partition holds a large fraction of the files. + assert(on.max <= 20, s"expected balanced partitions with rebalance, max=${on.max}") + assert(on.max < off.max, + s"rebalance must reduce the max partition: on=${on.max} off=${off.max}") + } + } + + // Three differing schemas in path-sorted subdirs so the merged column order is deterministic: + // 01-ab=(A,b), 02-bc=(b,c), 03-ac=(a,c). Path order fixes first-appearance to A, b, c, and A + // (seen before a) fixes the casing. + private def writeHeterogeneousParquet(dir: String): Unit = { + withSQLConf("spark.sql.files.maxRecordsPerFile" -> "1") { + spark.range(8).select(col("id").as("A"), (col("id") * 2).as("b")) + .write.mode("append").parquet(new File(dir, "01-ab").getCanonicalPath) + spark.range(8).select((col("id") * 3).as("b"), (col("id") * 4).as("c")) + .write.mode("append").parquet(new File(dir, "02-bc").getCanonicalPath) + spark.range(8).select(col("id").as("a"), (col("id") * 5).as("c")) + .write.mode("append").parquet(new File(dir, "03-ac").getCanonicalPath) + } + } + + private def inferredSchema(dir: String, rebalance: Boolean): StructType = { + withSQLConf(DeltaSQLConf.DELTA_CONVERT_REBALANCE_FILE_LISTING.key -> rebalance.toString) { + val manifest = new ParquetTable( + spark, dir, catalogTable = None, userPartitionSchema = None).fileManifest + try manifest.parquetSchema.get finally manifest.close() + } + } + + test("inferred schema is the exact path-ordered schema when rebalancing is enabled") { + withTempDir { tmp => + val dir = new File(tmp, "src").getCanonicalPath + writeHeterogeneousParquet(dir) + + // Rebalancing processes files in path order, so the merged schema is exactly A, b, c. + val s1 = inferredSchema(dir, rebalance = true) + val s2 = inferredSchema(dir, rebalance = true) + + assert(s1 == expectedPathOrderedSchema, s"unexpected schema: $s1") + assert(s2 == expectedPathOrderedSchema, s"unexpected schema: $s2") + } + } + + test("CONVERT TO DELTA commits the exact path-ordered schema") { + withTempDir { tmp => + val dir = new File(tmp, "src").getCanonicalPath + writeHeterogeneousParquet(dir) + + withSQLConf(DeltaSQLConf.DELTA_CONVERT_REBALANCE_FILE_LISTING.key -> "true") { + convertToDelta(s"parquet.`$dir`", collectStats = false) + } + + val deltaLog = DeltaLog.forTable(spark, dir) + assert(deltaLog.update().metadata.schema === expectedPathOrderedSchema) + } + } +} diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeleteMetricsSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeleteMetricsSuite.scala index 5dcf37b9898..3a0eacb79ab 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeleteMetricsSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeleteMetricsSuite.scala @@ -450,4 +450,14 @@ class DeleteMetricsSuite extends QueryTest } } + test("DELETE output metrics schema is not nullable") { + withTable("target") { + spark.range(10).write.format("delta").saveAsTable("target") + + val schema = spark.sql("DELETE FROM target WHERE id < 3").schema + + assert(schema.forall(!_.nullable)) + } + } + } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeleteSuiteBase.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeleteSuiteBase.scala index 3e5b94f884d..abf18b521a1 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeleteSuiteBase.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeleteSuiteBase.scala @@ -53,7 +53,7 @@ trait DeleteBaseMixin expectedErrorClassForDataSetTempView: String = null): Unit = { testWithTempView(s"test delete on temp view - $name") { isSQLTempView => withTable("tab") { - Seq((0, 3), (1, 2)).toDF("key", "value").write.format("delta").saveAsTable("tab") + Seq((0, 3), (1, 2)).toDF("key", "value").write.format(writeFormat).saveAsTable("tab") if (isSQLTempView) { sql(s"CREATE TEMP VIEW v AS $text") } else { @@ -91,13 +91,13 @@ trait DeleteBaseMixin expectResult: Seq[Row]): Unit = { testWithTempView(s"test delete on temp view - $name") { isSQLTempView => withTable("tab") { - Seq((0, 3), (1, 2)).toDF("key", "value").write.format("delta").saveAsTable("tab") + Seq((0, 3), (1, 2)).toDF("key", "value").write.format(writeFormat).saveAsTable("tab") createTempViewFromSelect(text, isSQLTempView) executeDelete( "v", "key >= 1 and value < 3" ) - checkAnswer(spark.read.format("delta").table("v"), expectResult) + checkAnswer(spark.read.format(writeFormat).table("v"), expectResult) } } } @@ -307,7 +307,7 @@ trait DeleteBaseTests extends DeleteBaseMixin { withTempPath { tempDir => val tempPath = tempDir.getCanonicalPath val df = Seq((2, 2), (3, 2)).toDF("key", "value") - df.write.format("delta").partitionBy("key").save(tempPath) + df.write.format(writeFormat).partitionBy("key").save(tempPath) val e = intercept[AnalysisException] { executeDelete(target = s"delta.`$tempPath/key=2`", where = "value = 2") @@ -319,7 +319,7 @@ trait DeleteBaseTests extends DeleteBaseMixin { test("delete cached table by name") { withTable("cached_delta_table") { Seq((2, 2), (1, 4)).toDF("key", "value") - .write.format("delta").saveAsTable("cached_delta_table") + .write.format(writeFormat).saveAsTable("cached_delta_table") spark.table("cached_delta_table").cache() spark.table("cached_delta_table").collect() @@ -552,7 +552,7 @@ trait DeleteBaseTests extends DeleteBaseMixin { customErrorRegex: Option[String] = None) { test(s"$functionType functions in delete - expect exception: $expectException") { withTable("deltaTable") { - data.write.format("delta").saveAsTable("deltaTable") + data.write.format(writeFormat).saveAsTable("deltaTable") val expectedErrorRegex = "(?s).*(?i)unsupported.*(?i).*Invalid expressions.*" @@ -575,7 +575,7 @@ trait DeleteBaseTests extends DeleteBaseMixin { if (catchException) { - val dataBeforeException = spark.read.format("delta").table("deltaTable").collect() + val dataBeforeException = spark.read.format(writeFormat).table("deltaTable").collect() val e = intercept[Exception] { executeDelete(target = "deltaTable", where = where) } @@ -584,7 +584,7 @@ trait DeleteBaseTests extends DeleteBaseMixin { } else e.getMessage assert(message.matches(errorRegex), s"unexpected error in $functionType case: ${e.getClass.getName}: $message") - checkAnswer(spark.read.format("delta").table("deltaTable"), dataBeforeException) + checkAnswer(spark.read.format(writeFormat).table("deltaTable"), dataBeforeException) } else { executeDelete(target = "deltaTable", where = where) } @@ -678,7 +678,7 @@ trait DeleteSubqueryBaseMixin extends DeleteBaseMixin { .toDF("c", "t1") .coalesce(1) .write - .format("delta") + .format(writeFormat) .mode("overwrite") .saveAsTable("target") if (source.nonEmpty) { @@ -707,7 +707,7 @@ trait DeleteSubqueryBaseMixin extends DeleteBaseMixin { val partSpec = if (isPartitioned) "partitioned by (c)" else "" test(name + s" isPartitioned=$isPartitioned") { withTable("target") { - sql(s"CREATE TABLE target(c int, t1 string) USING delta $partSpec") + sql(s"CREATE TABLE target(c int, t1 string) USING $tableProvider $partSpec") body } } @@ -833,7 +833,7 @@ trait DeleteSubqueryExistsTests extends DeleteSubqueryBaseMixin { .toDF("c", "t1") .coalesce(1) .write - .format("delta") + .format(writeFormat) .mode("overwrite") .saveAsTable("target") // Create an empty source view with the correct schema @@ -859,7 +859,7 @@ trait DeleteSubqueryExistsTests extends DeleteSubqueryBaseMixin { .toDF("c", "t1") .repartition(4) .write - .format("delta") + .format(writeFormat) .mode("overwrite") .saveAsTable("target") Seq((3, "x"), (7, "y"), (15, "z")) diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeletionVectorsTestUtils.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeletionVectorsTestUtils.scala index a3d52c9c605..1cd78a75d04 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeletionVectorsTestUtils.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeletionVectorsTestUtils.scala @@ -382,7 +382,7 @@ trait DeletionVectorsTestUtils extends QueryTest with SharedSparkSession with De withDVWriter(log, dvFileId) { writer => filesWithDVs.flatMap { case (currentFile, dv) => val range = writer.write(serializeRoaringBitmapArrayWithDefaultFormat(dv)) - val dvData = DeletionVectorDescriptor.onDiskWithRelativePath( + val dvData = DeletionVectorDescriptor.onDiskWithUuidRelativePath( id = dvFileId, sizeInBytes = range.length, cardinality = dv.cardinality, @@ -457,7 +457,7 @@ trait DeletionVectorsTestUtils extends QueryTest with SharedSparkSession with De val dvFileId = UUID.randomUUID() withDVWriter(log, dvFileId) { writer => val range = writer.write(serializeRoaringBitmapArrayWithDefaultFormat(bitmapArray)) - DeletionVectorDescriptor.onDiskWithRelativePath( + DeletionVectorDescriptor.onDiskWithUuidRelativePath( id = dvFileId, sizeInBytes = range.length, cardinality = bitmapArray.cardinality, diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaAlterTableReplaceTests.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaAlterTableReplaceTests.scala index 6d9c082283e..4fce787d5dd 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaAlterTableReplaceTests.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaAlterTableReplaceTests.scala @@ -16,6 +16,8 @@ package org.apache.spark.sql.delta +import java.util.regex.Pattern + import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.functions.{array, col, map, struct} import org.apache.spark.sql.internal.SQLConf @@ -25,6 +27,24 @@ trait DeltaAlterTableReplaceTests extends DeltaAlterTableTestBase { import testImplicits._ + protected def assertUnsupportedAlterTableReplaceColOp( + command: String, + subClass: String, + parameters: Map[String, String]): Unit = { + val e = intercept[DeltaAnalysisException] { + sql(command) + } + // The oldSchema and newSchema parameters are large and incidental to what the tests verify, + // so we match them with a wildcard and focus on the other parameters. Because matchPVals + // treats every expected value as a regex, the checked parameters are quoted to match literally. + checkError( + e, + s"DELTA_UNSUPPORTED_ALTER_TABLE_REPLACE_COL_OP.$subClass", + parameters = parameters.map { case (k, v) => k -> Pattern.quote(v) } ++ + Map("oldSchema" -> "(?s).*", "newSchema" -> "(?s).*"), + matchPVals = true) + } + ddlTest("REPLACE COLUMNS - add a comment") { val df = Seq((1, "a"), (2, "b")).toDF("v1", "v2") .withColumn("s", struct("v1", "v2")) @@ -212,17 +232,18 @@ trait DeltaAlterTableReplaceTests extends DeltaAlterTableTestBase { ddlTest("REPLACE COLUMNS - drop column") { // Column Mapping allows columns to be dropped def checkReplace( - text: String, + command: String, tableName: String, columnDropped: Seq[String], - messages: String*): Unit = { + columnNames: String): Unit = { if (columnMappingEnabled) { - spark.sql(text) + spark.sql(command) val (deltaLog, snapshot) = getDeltaLogWithSnapshot(tableName) val field = snapshot.schema.findNestedField(columnDropped, includeCollections = true) assert(field.isEmpty, "Column was not deleted") } else { - assertNotSupported(text, messages: _*) + assertUnsupportedAlterTableReplaceColOp( + command, "DROP_COLUMNS", Map("columnNames" -> columnNames)) } } @@ -235,17 +256,17 @@ trait DeltaAlterTableReplaceTests extends DeltaAlterTableTestBase { // trying to drop v1 of each struct, but it should fail because dropping column is // not supported unless column mapping is enabled checkReplace( - s""" + command = s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v2 string, | s STRUCT, | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin, - tableName, Seq("v1"), "dropping column(s)", "v1") + tableName = tableName, columnDropped = Seq("v1"), columnNames = "v1") // s.v1 checkReplace( - s""" + command = s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v1 int, | v2 string, @@ -253,10 +274,10 @@ trait DeltaAlterTableReplaceTests extends DeltaAlterTableTestBase { | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin, - tableName, Seq("s", "v1"), "dropping column(s)", "v1", "from s") + tableName = tableName, columnDropped = Seq("s", "v1"), columnNames = "s.v1") // a.v1 checkReplace( - s""" + command = s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v1 int, | v2 string, @@ -264,10 +285,11 @@ trait DeltaAlterTableReplaceTests extends DeltaAlterTableTestBase { | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin, - tableName, Seq("a", "element", "v1"), "dropping column(s)", "v1", "from a") + tableName = tableName, columnDropped = Seq("a", "element", "v1"), + columnNames = "a.element.v1") // m.key.v1 checkReplace( - s""" + command = s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v1 int, | v2 string, @@ -275,10 +297,10 @@ trait DeltaAlterTableReplaceTests extends DeltaAlterTableTestBase { | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin, - tableName, Seq("m", "key", "v1"), "dropping column(s)", "v1", "from m.key") + tableName = tableName, columnDropped = Seq("m", "key", "v1"), columnNames = "m.key.v1") // m.value.v1 checkReplace( - s""" + command = s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v1 int, | v2 string, @@ -286,7 +308,7 @@ trait DeltaAlterTableReplaceTests extends DeltaAlterTableTestBase { | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin, - tableName, Seq("m", "value", "v1"), "dropping column(s)", "v1", "from m.value") + tableName = tableName, columnDropped = Seq("m", "value", "v1"), columnNames = "m.value.v1") } } @@ -299,7 +321,7 @@ trait DeltaAlterTableReplaceTests extends DeltaAlterTableTestBase { // trying to change the data type of v1 of each struct to long, but it should fail because // changing data type is not supported. - assertNotSupported(s""" + assertUnsupportedAlterTableReplaceColOp(s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v1 long, | v2 string, @@ -307,9 +329,10 @@ trait DeltaAlterTableReplaceTests extends DeltaAlterTableTestBase { | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin, - "changing data type", "v1", "from IntegerType to LongType") + "CHANGE_DATA_TYPE", + Map("columnName" -> "v1", "fromType" -> "INT", "toType" -> "BIGINT")) // s.v1 - assertNotSupported(s""" + assertUnsupportedAlterTableReplaceColOp(s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v1 int, | v2 string, @@ -317,9 +340,10 @@ trait DeltaAlterTableReplaceTests extends DeltaAlterTableTestBase { | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin, - "changing data type", "s.v1", "from IntegerType to LongType") + "CHANGE_DATA_TYPE", + Map("columnName" -> "s.v1", "fromType" -> "INT", "toType" -> "BIGINT")) // a.element.v1 - assertNotSupported(s""" + assertUnsupportedAlterTableReplaceColOp(s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v1 int, | v2 string, @@ -327,9 +351,10 @@ trait DeltaAlterTableReplaceTests extends DeltaAlterTableTestBase { | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin, - "changing data type", "a.element.v1", "from IntegerType to LongType") + "CHANGE_DATA_TYPE", + Map("columnName" -> "a.element.v1", "fromType" -> "INT", "toType" -> "BIGINT")) // m.key.v1 - assertNotSupported(s""" + assertUnsupportedAlterTableReplaceColOp(s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v1 int, | v2 string, @@ -337,9 +362,10 @@ trait DeltaAlterTableReplaceTests extends DeltaAlterTableTestBase { | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin, - "changing data type", "m.key.v1", "from IntegerType to LongType") + "CHANGE_DATA_TYPE", + Map("columnName" -> "m.key.v1", "fromType" -> "INT", "toType" -> "BIGINT")) // m.value.v1 - assertNotSupported(s""" + assertUnsupportedAlterTableReplaceColOp(s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v1 int, | v2 string, @@ -347,7 +373,8 @@ trait DeltaAlterTableReplaceTests extends DeltaAlterTableTestBase { | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin, - "changing data type", "m.value.v1", "from IntegerType to LongType") + "CHANGE_DATA_TYPE", + Map("columnName" -> "m.value.v1", "fromType" -> "INT", "toType" -> "BIGINT")) } } @@ -639,7 +666,7 @@ trait DeltaAlterTableReplaceTests extends DeltaAlterTableTestBase { |)""".stripMargin, "NOT NULL is not supported in Hive-style REPLACE COLUMNS") // s.v3 - assertNotSupported(s""" + assertUnsupportedAlterTableReplaceColOp(s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v1 int, | v2 string, @@ -647,9 +674,9 @@ trait DeltaAlterTableReplaceTests extends DeltaAlterTableTestBase { | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin, - "adding non-nullable column", "s.v3") + "ADD_NON_NULLABLE_COLUMN", Map("columnName" -> "s.v3")) // a.element.v3 - assertNotSupported(s""" + assertUnsupportedAlterTableReplaceColOp(s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v1 int, | v2 string, @@ -657,9 +684,9 @@ trait DeltaAlterTableReplaceTests extends DeltaAlterTableTestBase { | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin, - "adding non-nullable column", "a.element.v3") + "ADD_NON_NULLABLE_COLUMN", Map("columnName" -> "a.element.v3")) // m.key.v3 - assertNotSupported(s""" + assertUnsupportedAlterTableReplaceColOp(s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v1 int, | v2 string, @@ -667,9 +694,9 @@ trait DeltaAlterTableReplaceTests extends DeltaAlterTableTestBase { | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin, - "adding non-nullable column", "m.key.v3") + "ADD_NON_NULLABLE_COLUMN", Map("columnName" -> "m.key.v3")) // m.value.v3 - assertNotSupported(s""" + assertUnsupportedAlterTableReplaceColOp(s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v1 int, | v2 string, @@ -677,7 +704,7 @@ trait DeltaAlterTableReplaceTests extends DeltaAlterTableTestBase { | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin, - "adding non-nullable column", "m.value.v3") + "ADD_NON_NULLABLE_COLUMN", Map("columnName" -> "m.value.v3")) } } @@ -700,7 +727,7 @@ trait DeltaAlterTableReplaceTests extends DeltaAlterTableTestBase { |)""".stripMargin, "NOT NULL is not supported in Hive-style REPLACE COLUMNS") // s.v1 - assertNotSupported(s""" + assertUnsupportedAlterTableReplaceColOp(s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v1 int, | v2 string, @@ -708,9 +735,9 @@ trait DeltaAlterTableReplaceTests extends DeltaAlterTableTestBase { | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin, - "tightening nullability", "s.v1") + "TIGHTEN_NULLABILITY", Map("columnName" -> "s.v1")) // a.element.v1 - assertNotSupported(s""" + assertUnsupportedAlterTableReplaceColOp(s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v1 int, | v2 string, @@ -718,9 +745,9 @@ trait DeltaAlterTableReplaceTests extends DeltaAlterTableTestBase { | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin, - "tightening nullability", "a.element.v1") + "TIGHTEN_NULLABILITY", Map("columnName" -> "a.element.v1")) // m.key.v1 - assertNotSupported(s""" + assertUnsupportedAlterTableReplaceColOp(s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v1 int, | v2 string, @@ -728,9 +755,9 @@ trait DeltaAlterTableReplaceTests extends DeltaAlterTableTestBase { | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin, - "tightening nullability", "m.key.v1") + "TIGHTEN_NULLABILITY", Map("columnName" -> "m.key.v1")) // m.value.v1 - assertNotSupported(s""" + assertUnsupportedAlterTableReplaceColOp(s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v1 int, | v2 string, @@ -738,7 +765,7 @@ trait DeltaAlterTableReplaceTests extends DeltaAlterTableTestBase { | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin, - "tightening nullability", "m.value.v1") + "TIGHTEN_NULLABILITY", Map("columnName" -> "m.value.v1")) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaAlterTableTests.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaAlterTableTests.scala index a757847cddd..3375a73ce34 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaAlterTableTests.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaAlterTableTests.scala @@ -146,8 +146,12 @@ trait DeltaAlterTableTests extends DeltaAlterTableTestBase { | 'key' = 'value' |)""".stripMargin) } - assert(e.getMessage.contains("expects a table. Please use ALTER VIEW instead.") || - e.getMessage.contains("EXPECT_TABLE_NOT_VIEW.USE_ALTER_VIEW")) + checkError( + e, + "EXPECT_TABLE_NOT_VIEW.USE_ALTER_VIEW", + parameters = Map( + "operation" -> "ALTER TABLE ... SET TBLPROPERTIES", "viewName" -> "`v`"), + queryContext = Array(ExpectedContext(fragment = "v", start = 13, stop = 13))) } } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaCDCSQLSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaCDCSQLSuite.scala index 58a2947258c..faa05a7f5d5 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaCDCSQLSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaCDCSQLSuite.scala @@ -475,3 +475,6 @@ class DeltaCDCSQLWithCatalogOwnedBatch2Suite extends DeltaCDCSQLSuite { class DeltaCDCSQLWithCatalogOwnedBatch100Suite extends DeltaCDCSQLSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(100) } + +// Routes cdcRead through the V2 changelog SQL surface. See [[ChangelogV2CdcReadMixin]]. +class DeltaCDCSQLChangelogV2Suite extends DeltaCDCSQLSuite with ChangelogV2CdcReadMixin diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaCDCStreamSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaCDCStreamSuite.scala index 99b8185aa7b..c46757650f6 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaCDCStreamSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaCDCStreamSuite.scala @@ -34,13 +34,17 @@ import org.apache.hadoop.fs.Path import org.apache.spark.SparkConf import org.apache.spark.sql.DataFrame +import org.apache.spark.sql.QueryTest import org.apache.spark.sql.functions._ import org.apache.spark.sql.streaming.{StreamingQuery, StreamingQueryException, StreamTest, Trigger} import org.apache.spark.sql.types.StructType -trait DeltaCDCStreamSuiteBase extends StreamTest with DeltaSQLCommandTest +trait DeltaCDCStreamSuiteBase + extends QueryTest + with DeltaSQLCommandTest + with DeltaColumnMappingTestUtils with DeltaSourceSuiteBase - with DeltaColumnMappingTestUtils { + with StreamTest { import testImplicits._ import io.delta.implicits._ @@ -822,6 +826,55 @@ trait DeltaCDCStreamSuiteBase extends StreamTest with DeltaSQLCommandTest } } + test("CDF stream rolls the offset past a change-data commit (keeps END marker)") { + // Regression guard: a processed change-data (AddCDCFile) commit must roll the + // offset to (version + 1, BASE_INDEX), not park on its last real index. Plain (non-shared) + // Delta on purpose: the sharing mock can't reproduce it (offset comes from latestOffset). + withTempDir { inputDir => + withTempDir { checkpointDir => + withTempDir { outputDir => + // v0: two AddFiles (one per partition). + spark.range(2) + .withColumn("part", 'id % 2) + .withColumn("col3", lit(0)) + .repartition(1) + .write + .format("delta") + .partitionBy("part") + .option("enableChangeDataFeed", "true") + .save(inputDir.getAbsolutePath) + val deltaTable = io.delta.tables.DeltaTable.forPath(inputDir.getAbsolutePath) + // v1, v2: UPDATEs -> AddCDCFile commits; the last version processed is change data. + deltaTable.update(expr("col3 < 2"), Map("col3" -> lit("1"))) + deltaTable.update(expr("col3 < 2"), Map("col3" -> lit("2"))) + + val query = loadStreamWithOptions(inputDir.getAbsolutePath, Map( + DeltaOptions.CDC_READ_OPTION -> "true", + "startingVersion" -> "0")) + .select("id") + .writeStream + .format("delta") + .outputMode("append") + .option("checkpointLocation", checkpointDir.toString) + .start(outputDir.getAbsolutePath) + try { + query.processAllAvailable() + } finally { + query.stop() + query.awaitTermination() + } + + val endOffset = + JsonUtils.fromJson[DeltaSourceOffset](query.lastProgress.sources.head.endOffset) + assert(endOffset.index === DeltaSourceOffset.BASE_INDEX, + s"CDF offset parked on a real index instead of rolling past the change-data " + + s"commit: $endOffset") + assert(endOffset.reservoirVersion === 3L, s"endOffset = $endOffset") + } + } + } + } + test("maxFilesPerTrigger - batch reject stops iteration to prevent data loss") { withTempDir { inputDir => // v0: 2 AddFiles (one per partition) @@ -907,7 +960,7 @@ trait DeltaCDCStreamSuiteBase extends StreamTest with DeltaSQLCommandTest testStream(df)( StartStream(Trigger.AvailableNow), Execute { query => - assert(query.awaitTermination(10000)) + assert(query.awaitTermination(streamingTimeout.toMillis)) }, CheckProgress(rowsPerBatch), CheckAnswer( diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaCDCSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaCDCSuite.scala index 4ab7f848321..299e7451777 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaCDCSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaCDCSuite.scala @@ -31,6 +31,7 @@ import org.apache.spark.sql.delta.coordinatedcommits.CatalogOwnedTestBaseSuite import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.{DeltaSQLCommandTest, DeltaSQLTestUtils} import org.apache.spark.sql.delta.test.DeltaTestImplicits._ +import org.apache.spark.sql.delta.test.shims.ChangelogSyntaxSupportedShim import org.apache.spark.sql.delta.util.{DeltaCommitFileProvider, FileNames} import org.apache.hadoop.fs.Path @@ -1177,3 +1178,132 @@ class DeltaCDCScalaWithCatalogOwnedBatch2Suite extends DeltaCDCScalaSuite { class DeltaCDCScalaWithCatalogOwnedBatch100Suite extends DeltaCDCScalaSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(100) } + +/** + * Routes [[DeltaCDCSuiteBase.cdcRead]] through the V2 changelog SQL surface + * (` CHANGES FROM ...` -> TableCatalog.loadChangelog -> DeltaV2ChangelogScan / + * DeltaV2ChangelogBatch), so the existing DeltaCDCScalaSuite / DeltaCDCSQLSuite test bodies + * validate the V2 changelog read path. Sister of [[ChangelogV2CDCUtilMixin]], which does the same + * for the computeCDC() entry point used by the generated Merge/Delete/Update CDC suites. + * + * STRICT mode is scoped to the read only. Path-based reads register the path as a named + * (unmanaged) table so resolution goes through the configured V2 spark_catalog (returns a + * SparkTable); a bare `delta.`path`` would resolve to a V1 DeltaTableV2 and be rejected by + * ChangelogSupport. + */ +trait ChangelogV2CdcReadMixin extends DeltaCDCSuiteBase with ChangelogSyntaxSupportedShim { + + override protected def sparkConf: SparkConf = super.sparkConf + .set("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") + .set(DeltaSQLConf.DELTA_CHANGELOG_V2_ENABLED.key, "true") + // The V2 changelog read path requires row tracking (DELTA_CHANGELOG_REQUIRES_ROW_TRACKING). + .set(DeltaConfigs.ROW_TRACKING_ENABLED.defaultTablePropertyKey, "true") + + // Tests that do not apply to the V2 changelog read path: + // - V1-specific error semantics (they intercept V1 error codes that the V2 path does not raise) + // - mixed timestamp/version boundaries (the V2 CHANGES grammar only allows VERSION..VERSION or + // TIMESTAMP..TIMESTAMP) + // - timestamp -> version resolution, unsupported on the JVM kernel backend today: + // getActiveCommitAtTimestamp requires a JvmSnapshot and the drop-in SnapshotImpl is not + // unwrapped. TODO: re-enable the timestamp families once the kernel history SPI supports + // the drop-in snapshot. + // - open-ended reads: V1 resolves the missing end bound lazily at execution time, the V2 path + // resolves it eagerly at analysis time, so versions written after the DataFrame is built are + // not observed. + // - protocol assertions: the V2 path requires row tracking (see sparkConf above), which adds the + // rowTracking and domainMetadata table features the V1 test does not expect. + protected def excludedV2Exact: Set[String] = Set( + "An error should be thrown when CDC is not enabled", + "aggregating non-numeric cdc data columns", + "changes - end timestamp exceeding latest commit timestamp", + "changes - only start is a timestamp", + "changes - only start is a timestamp - inclusive behavior", + "changes - start and end are timestamps", + "changes - start timestamp exceeding latest commit timestamp", + "end timestamp < start timestamp", + "ending version not specified resolves to latest at execution time", + "non-monotonic timestamps", + "protocol version", + "resolve expression for timestamp function", + "table schema changed after dataframe with ending not specified", + "table schema changed after dataframe with ending specified" + ) + + // Whole test families excluded by name prefix (every member is V1-specific or timestamp-based). + protected def excludedV2Prefixes: Seq[String] = Seq( + "range boundary cannot be null", + "CDF timestamp format", + "version from timestamp", + "CDC read respects timezone and DST", + "CDC read's commit timestamps are correct" + ) + + // Filter by name in a test() override and ignore() the matches; everything else runs. The V2 + // changelog read path uses the `SELECT ... CHANGES` clause, which only the Spark 4.2 parser + // supports; supportsChangelogSyntax is a compile-time shim (true on spark-4.2, false on + // spark-4.0-4.1) so tests cancel on older versions. + override protected def test(testName: String, testTags: org.scalatest.Tag*)(testFun: => Any)( + implicit pos: org.scalactic.source.Position): Unit = { + if (excludedV2Exact.contains(testName) || excludedV2Prefixes.exists(testName.startsWith)) { + ignore(testName + " (excluded on the V2 changelog read path)", testTags: _*)(testFun) + } else { + super.test(testName, testTags: _*) { + assume(supportsChangelogSyntax, "The SELECT ... CHANGES clause requires Spark 4.2 or newer") + testFun + } + } + } + + override def cdcRead( + tblId: TblId, + start: Boundary, + end: Boundary, + schemaMode: Option[DeltaBatchCDFSchemaMode] = Some(BatchCDFSchemaLegacy), + readerOptions: Map[String, String] = Map.empty): DataFrame = { + + // Set the batch CDF schema mode using SQL conf if we specified it, mirroring the V1 overrides. + if (schemaMode.isDefined) { + var result: DataFrame = null + withSQLConf(DeltaSQLConf.DELTA_CDF_DEFAULT_SCHEMA_MODE_FOR_COLUMN_MAPPING_TABLE.key -> + schemaMode.get.name) { + result = cdcRead(tblId, start, end, None, readerOptions) + } + return result + } + + val startClause: String = start match { + case sv: StartingVersion => s"FROM VERSION ${sv.value}" + case st: StartingTimestamp => s"FROM TIMESTAMP '${st.value}'" + case Unbounded => + // changesClause requires a starting boundary; unbounded start is streaming-only. + throw new UnsupportedOperationException( + "V2 changelog batch read requires a starting boundary") + } + val endClause: String = end match { + case ev: EndingVersion => s"TO VERSION ${ev.value}" + case et: EndingTimestamp => s"TO TIMESTAMP '${et.value}'" + case Unbounded => "" + } + + withSQLConf(DeltaSQLConf.V2_ENABLE_MODE.key -> "STRICT") { + tblId match { + case _: TableName => + spark.sql(s"SELECT * FROM ${tblId.id} CHANGES $startClause $endClause") + .drop("_metadata") + case _: TablePath => + val tempName = s"v2cdc_temp_${System.nanoTime()}" + spark.sql(s"CREATE TABLE $tempName USING delta LOCATION '${tblId.id}'") + try { + spark.sql(s"SELECT * FROM $tempName CHANGES $startClause $endClause") + .drop("_metadata") + } finally { + spark.sql(s"DROP TABLE IF EXISTS $tempName") + } + case _ => + throw new IllegalArgumentException("No table name or path provided") + } + } + } +} + +class DeltaCDCScalaChangelogV2Suite extends DeltaCDCScalaSuite with ChangelogV2CdcReadMixin diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaColumnMappingSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaColumnMappingSuite.scala index 907d55e76aa..05062496f27 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaColumnMappingSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaColumnMappingSuite.scala @@ -2282,6 +2282,33 @@ class DeltaColumnMappingSuite extends QueryTest } } + test("column-mapping removal is blocked for a catalog-managed table") { + withSQLConf(DeltaSQLConf.ALLOW_COLUMN_MAPPING_REMOVAL.key -> "true") { + withCatalogManagedTable() { tableName => + spark.sql( + s"ALTER TABLE $tableName SET TBLPROPERTIES " + + s"('${DeltaConfigs.COLUMN_MAPPING_MODE.key}' = 'name')") + spark.sql(s"INSERT INTO $tableName VALUES (1)") + val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) + val snapshotBefore = deltaLog.update() + + checkError( + intercept[DeltaUnsupportedOperationException] { + spark.sql( + s"ALTER TABLE $tableName SET TBLPROPERTIES " + + s"('${DeltaConfigs.COLUMN_MAPPING_MODE.key}' = 'none')") + }, + "DELTA_UNSUPPORTED_CATALOG_MANAGED_TABLE_OPERATION", + parameters = Map("operation" -> "DATA_REORGANIZATION")) + + val snapshotAfter = deltaLog.update() + assert(snapshotAfter.version === snapshotBefore.version) + assert(snapshotAfter.metadata === snapshotBefore.metadata) + assert(snapshotAfter.allFiles.collect().toSet === snapshotBefore.allFiles.collect().toSet) + } + } + } + test("enabling column mapping disallowed if column mapping metadata already exists") { withSQLConf( // enabling this fixes the issue of committing invalid metadata in the first place diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaColumnRenameSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaColumnRenameSuite.scala index bd6019b1ff6..ee6730f5e8f 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaColumnRenameSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaColumnRenameSuite.scala @@ -126,8 +126,11 @@ class DeltaColumnRenameSuite extends QueryTest val e = intercept[AnalysisException] { spark.sql(s"Alter table t1 RENAME COLUMN map to map1") } - assert(e.getMessage.contains("enable Column Mapping") && - e.getMessage.contains("mapping mode 'name'")) + checkError( + e, + "DELTA_UNSUPPORTED_RENAME_COLUMN.ENABLE_COLUMN_MAPPING", + sqlState = Some("0AKDC"), + parameters = Map("readerVersion" -> "2", "writerVersion" -> "5")) alterTableWithProps("t1", Map( DeltaConfigs.COLUMN_MAPPING_MODE.key -> "name", @@ -179,8 +182,11 @@ class DeltaColumnRenameSuite extends QueryTest val e = intercept[AnalysisException] { spark.sql(s"Alter table t1 RENAME COLUMN map to map1") } - assert(e.getMessage.contains("enable Column Mapping") && - e.getMessage.contains("mapping mode 'name'")) + checkError( + e, + "DELTA_UNSUPPORTED_RENAME_COLUMN.ENABLE_COLUMN_MAPPING", + sqlState = Some("0AKDC"), + parameters = Map("readerVersion" -> "2", "writerVersion" -> "5")) // Upgrading this schema shouldn't cause any errors even if there are leaf column name // duplications such as a.c, b.c. diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaDataFrameWriterV2Suite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaDataFrameWriterV2Suite.scala index 1662e990563..c6d76284e7e 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaDataFrameWriterV2Suite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaDataFrameWriterV2Suite.scala @@ -41,7 +41,8 @@ import org.apache.spark.util.Utils trait OpenSourceDataFrameWriterV2Tests extends QueryTest with SharedSparkSession - with BeforeAndAfter { + with BeforeAndAfter + with DeltaTableProvider { import testImplicits._ @@ -74,7 +75,7 @@ trait OpenSourceDataFrameWriterV2Tests } test("Append: basic append") { - spark.sql("CREATE TABLE table_name (id bigint, data string) USING delta") + spark.sql(createTableSQL("table_name", "id bigint, data string")) checkAnswer(spark.table("table_name"), Seq.empty) @@ -92,7 +93,7 @@ trait OpenSourceDataFrameWriterV2Tests } test("Append: by name not position") { - spark.sql("CREATE TABLE table_name (id bigint, data string) USING delta") + spark.sql(createTableSQL("table_name", "id bigint, data string")) checkAnswer(spark.table("table_name"), Seq.empty) @@ -117,7 +118,7 @@ trait OpenSourceDataFrameWriterV2Tests test("Overwrite: overwrite by expression: true") { spark.sql( - "CREATE TABLE table_name (id bigint, data string) USING delta PARTITIONED BY (id)") + createTableSQL("table_name", "id bigint, data string", partitionBy = "PARTITIONED BY (id)")) checkAnswer(spark.table("table_name"), Seq.empty) @@ -136,7 +137,7 @@ trait OpenSourceDataFrameWriterV2Tests test("Overwrite: overwrite by expression: id = 3") { spark.sql( - "CREATE TABLE table_name (id bigint, data string) USING delta PARTITIONED BY (id)") + createTableSQL("table_name", "id bigint, data string", partitionBy = "PARTITIONED BY (id)")) checkAnswer(spark.table("table_name"), Seq.empty) @@ -149,10 +150,14 @@ trait OpenSourceDataFrameWriterV2Tests val e = intercept[AnalysisException] { spark.table("source2").writeTo("table_name").overwrite($"id" === 3) } - assert(e.getErrorClass == "DELTA_REPLACE_WHERE_MISMATCH") - assert(e.getMessage.startsWith( - "[DELTA_REPLACE_WHERE_MISMATCH] Written data does not conform to partial table overwrite " + - "condition or constraint")) + checkError( + exception = e, + condition = "DELTA_REPLACE_WHERE_MISMATCH.INVARIANT_VIOLATION", + sqlState = Some("44000"), + parameters = Map( + "replaceWhere" -> "\\(id = 3L\\)", + "invariantViolationMessage" -> "(?s).*"), + matchPVals = true) checkAnswer( spark.table("table_name"), @@ -160,7 +165,7 @@ trait OpenSourceDataFrameWriterV2Tests } test("Overwrite: by name not position") { - spark.sql("CREATE TABLE table_name (id bigint, data string) USING delta") + spark.sql(createTableSQL("table_name", "id bigint, data string")) checkAnswer(spark.table("table_name"), Seq.empty) @@ -186,7 +191,7 @@ trait OpenSourceDataFrameWriterV2Tests test("OverwritePartitions: overwrite conflicting partitions") { spark.sql( - "CREATE TABLE table_name (id bigint, data string) USING delta PARTITIONED BY (id)") + createTableSQL("table_name", "id bigint, data string", partitionBy = "PARTITIONED BY (id)")) checkAnswer(spark.table("table_name"), Seq.empty) @@ -205,7 +210,7 @@ trait OpenSourceDataFrameWriterV2Tests } test("OverwritePartitions: overwrite all rows if not partitioned") { - spark.sql("CREATE TABLE table_name (id bigint, data string) USING delta") + spark.sql(createTableSQL("table_name", "id bigint, data string")) checkAnswer(spark.table("table_name"), Seq.empty) @@ -223,7 +228,7 @@ trait OpenSourceDataFrameWriterV2Tests } test("OverwritePartitions: by name not position") { - spark.sql("CREATE TABLE table_name (id bigint, data string) USING delta") + spark.sql(createTableSQL("table_name", "id bigint, data string")) checkAnswer(spark.table("table_name"), Seq.empty) @@ -248,7 +253,7 @@ trait OpenSourceDataFrameWriterV2Tests } test("Create: basic behavior") { - spark.table("source").writeTo("table_name").using("delta").create() + spark.table("source").writeTo("table_name").using(tableProvider).create() checkAnswer( spark.table("table_name"), @@ -263,7 +268,7 @@ trait OpenSourceDataFrameWriterV2Tests } test("Create: with using") { - spark.table("source").writeTo("table_name").using("delta").create() + spark.table("source").writeTo("table_name").using(tableProvider).create() checkAnswer( spark.table("table_name"), @@ -279,7 +284,7 @@ trait OpenSourceDataFrameWriterV2Tests test("Create: with property") { spark.table("source").writeTo("table_name") - .tableProperty("prop", "value").using("delta").create() + .tableProperty("prop", "value").using(tableProvider).create() checkAnswer( spark.table("table_name"), @@ -294,7 +299,7 @@ trait OpenSourceDataFrameWriterV2Tests } test("Create: identity partitioned table") { - spark.table("source").writeTo("table_name").using("delta").partitionedBy($"id").create() + spark.table("source").writeTo("table_name").using(tableProvider).partitionedBy($"id").create() checkAnswer( spark.table("table_name"), @@ -310,10 +315,10 @@ trait OpenSourceDataFrameWriterV2Tests test("Create: fail if table already exists") { spark.sql( - "CREATE TABLE table_name (id bigint, data string) USING delta PARTITIONED BY (id)") + createTableSQL("table_name", "id bigint, data string", partitionBy = "PARTITIONED BY (id)")) val exc = intercept[TableAlreadyExistsException] { - spark.table("source").writeTo("table_name").using("delta").create() + spark.table("source").writeTo("table_name").using(tableProvider).create() } assert(exc.getMessage.contains("table_name")) @@ -329,7 +334,7 @@ trait OpenSourceDataFrameWriterV2Tests test("Replace: basic behavior") { spark.sql( - "CREATE TABLE table_name (id bigint, data string) USING delta PARTITIONED BY (id)") + createTableSQL("table_name", "id bigint, data string", partitionBy = "PARTITIONED BY (id)")) spark.sql("INSERT INTO TABLE table_name SELECT * FROM source") checkAnswer( @@ -346,7 +351,7 @@ trait OpenSourceDataFrameWriterV2Tests spark.table("source2") .withColumn("even_or_odd", when(($"id" % 2) === 0, "even").otherwise("odd")) - .writeTo("table_name").using("delta") + .writeTo("table_name").using(tableProvider) .tableProperty("deLta.aPpeNdonly", "true").replace() checkAnswer( @@ -366,7 +371,7 @@ trait OpenSourceDataFrameWriterV2Tests } test("Replace: partitioned table") { - spark.sql("CREATE TABLE table_name (id bigint, data string) USING delta") + spark.sql(createTableSQL("table_name", "id bigint, data string")) spark.sql("INSERT INTO TABLE table_name SELECT * FROM source") checkAnswer( @@ -383,7 +388,7 @@ trait OpenSourceDataFrameWriterV2Tests spark.table("source2") .withColumn("even_or_odd", when(($"id" % 2) === 0, "even").otherwise("odd")) - .writeTo("table_name").using("delta") + .writeTo("table_name").using(tableProvider) .partitionedBy($"id") .replace() @@ -405,7 +410,7 @@ trait OpenSourceDataFrameWriterV2Tests test("Replace: fail if table does not exist") { val exc = intercept[AnalysisException] { - spark.table("source").writeTo("table_name").using("delta").replace() + spark.table("source").writeTo("table_name").using(tableProvider).replace() } checkError(exc, "TABLE_OR_VIEW_NOT_FOUND", Some("42P01"), @@ -413,7 +418,7 @@ trait OpenSourceDataFrameWriterV2Tests } test("CreateOrReplace: table does not exist") { - spark.table("source2").writeTo("table_name").using("delta").createOrReplace() + spark.table("source2").writeTo("table_name").using(tableProvider).createOrReplace() checkAnswer( spark.table("table_name"), @@ -430,7 +435,7 @@ trait OpenSourceDataFrameWriterV2Tests test("CreateOrReplace: table exists") { spark.sql( - "CREATE TABLE table_name (id bigint, data string) USING delta PARTITIONED BY (id)") + createTableSQL("table_name", "id bigint, data string", partitionBy = "PARTITIONED BY (id)")) spark.sql("INSERT INTO TABLE table_name SELECT * FROM source") checkAnswer( @@ -447,7 +452,7 @@ trait OpenSourceDataFrameWriterV2Tests spark.table("source2") .withColumn("even_or_odd", when(($"id" % 2) === 0, "even").otherwise("odd")) - .writeTo("table_name").using("delta").createOrReplace() + .writeTo("table_name").using(tableProvider).createOrReplace() checkAnswer( spark.table("table_name"), @@ -471,7 +476,7 @@ trait OpenSourceDataFrameWriterV2Tests .withColumn("ts", lit("2019-06-01 10:00:00.000000").cast("timestamp")) .writeTo("table_name") .partitionedBy(years($"ts")) - .using("delta") + .using(tableProvider) .create() } assert(e.getMessage.contains("Partitioning by expressions")) @@ -483,7 +488,7 @@ trait OpenSourceDataFrameWriterV2Tests .withColumn("ts", lit("2019-06-01 10:00:00.000000").cast("timestamp")) .writeTo("table_name") .partitionedBy(months($"ts")) - .using("delta") + .using(tableProvider) .create() } assert(e.getMessage.contains("Partitioning by expressions")) @@ -495,7 +500,7 @@ trait OpenSourceDataFrameWriterV2Tests .withColumn("ts", lit("2019-06-01 10:00:00.000000").cast("timestamp")) .writeTo("table_name") .partitionedBy(days($"ts")) - .using("delta") + .using(tableProvider) .create() } assert(e.getMessage.contains("Partitioning by expressions")) @@ -507,7 +512,7 @@ trait OpenSourceDataFrameWriterV2Tests .withColumn("ts", lit("2019-06-01 10:00:00.000000").cast("timestamp")) .writeTo("table_name") .partitionedBy(hours($"ts")) - .using("delta") + .using(tableProvider) .create() } assert(e.getMessage.contains("Partitioning by expressions")) @@ -518,7 +523,7 @@ trait OpenSourceDataFrameWriterV2Tests spark.table("source") .writeTo("table_name") .partitionedBy(bucket(4, $"id")) - .using("delta") + .using(tableProvider) .create() } assert(e.getMessage.contains("is not supported for Delta tables")) @@ -531,8 +536,215 @@ class DeltaDataFrameWriterV2Suite import testImplicits._ + import org.apache.spark.sql.delta.sources.DeltaSQLConf.{ + REPLACEWHERE_CONSTRAINT_CHECK_ENABLED, + REPLACEWHERE_DATACOLUMNS_ENABLED, + REPLACEWHERE_LITERAL_STRING_PREDICATES_ENABLED + } + + private object OverwriteApi extends Enumeration { + val SQL, DATAFRAME_WRITER_V2, REPLACE_WHERE_OPTION = Value + } + + private def overwriteTable( + api: OverwriteApi.Value, + predicate: String, + replacement: Dataset[Row]): Unit = { + api match { + case OverwriteApi.SQL => + withTempView("replacement") { + replacement.createOrReplaceTempView("replacement") + spark.sql(s"INSERT INTO table_name REPLACE WHERE $predicate SELECT * FROM replacement") + } + case OverwriteApi.DATAFRAME_WRITER_V2 => + replacement.writeTo("table_name").overwrite(expr(predicate)) + case OverwriteApi.REPLACE_WHERE_OPTION => + replacement.write.format("delta").mode("overwrite") + .option("replaceWhere", predicate).saveAsTable("table_name") + } + } + + private def withLiteralStringPredicateConf(enabled: Boolean)(f: => Unit): Unit = { + withSQLConf( + REPLACEWHERE_LITERAL_STRING_PREDICATES_ENABLED.key -> enabled.toString, + "spark.databricks.delta.v2.enableMode" -> "NONE", + REPLACEWHERE_DATACOLUMNS_ENABLED.key -> "true", + REPLACEWHERE_CONSTRAINT_CHECK_ENABLED.key -> "true", + SQLConf.ESCAPED_STRING_LITERALS.key -> "false") { + f + } + } + + for { + api <- OverwriteApi.values + enabled <- Seq(true, false) + } { + test(s"Overwrite: literal string predicates: startsWith percent ($api, enabled=$enabled)") { + withLiteralStringPredicateConf(enabled) { + withTable("table_name") { + spark.sql("CREATE TABLE table_name (id int, value string) USING delta") + Seq((1, "%first"), (2, "%second"), (3, "plain"), (4, null)) + .toDF("id", "value").writeTo("table_name").append() + val replacement = Seq((10, "%first")).toDF("id", "value") + + overwriteTable(api, "startswith(value, '%')", replacement) + + if (enabled || api == OverwriteApi.REPLACE_WHERE_OPTION) { + checkAnswer( + spark.table("table_name"), + Seq(Row(3, "plain"), Row(4, null), Row(10, "%first"))) + } else { + checkAnswer(spark.table("table_name"), Seq(Row(4, null), Row(10, "%first"))) + } + } + } + } + + test(s"Overwrite: literal string predicates: endsWith underscore ($api, enabled=$enabled)") { + withLiteralStringPredicateConf(enabled) { + withTable("table_name") { + spark.sql("CREATE TABLE table_name (id int, value string) USING delta") + Seq((1, "first_"), (2, "second_"), (3, "plain"), (4, null)) + .toDF("id", "value").writeTo("table_name").append() + val replacement = Seq((10, "first_")).toDF("id", "value") + + overwriteTable(api, "endswith(value, '_')", replacement) + + if (enabled || api == OverwriteApi.REPLACE_WHERE_OPTION) { + checkAnswer( + spark.table("table_name"), + Seq(Row(3, "plain"), Row(4, null), Row(10, "first_"))) + } else { + checkAnswer(spark.table("table_name"), Seq(Row(4, null), Row(10, "first_"))) + } + } + } + } + + test(s"Overwrite: literal string predicates: contains backslash ($api, enabled=$enabled)") { + withLiteralStringPredicateConf(enabled) { + withTable("table_name") { + spark.sql("CREATE TABLE table_name (id int, value string) USING delta") + Seq((1, "a\\b"), (2, "c\\d"), (3, "plain"), (4, null)) + .toDF("id", "value").writeTo("table_name").append() + val replacement = Seq((10, "a\\b")).toDF("id", "value") + val predicate = """contains(value, r'\')""" + + if (enabled || api == OverwriteApi.REPLACE_WHERE_OPTION) { + overwriteTable(api, predicate, replacement) + checkAnswer( + spark.table("table_name"), + Seq(Row(3, "plain"), Row(4, null), Row(10, "a\\b"))) + } else { + val error = intercept[AnalysisException] { + overwriteTable(api, predicate, replacement) + } + assert( + error.getErrorClass.stripSuffix(".INVARIANT_VIOLATION") == + "DELTA_REPLACE_WHERE_MISMATCH") + checkAnswer( + spark.table("table_name"), + Seq(Row(1, "a\\b"), Row(2, "c\\d"), Row(3, "plain"), Row(4, null))) + } + } + } + } + + test(s"Overwrite: literal string predicates: contains percent ($api, enabled=$enabled)") { + withLiteralStringPredicateConf(enabled) { + withTable("table_name") { + spark.sql("CREATE TABLE table_name (id int, value string) USING delta") + Seq((1, "%first"), (2, "second%"), (3, "plain"), (4, ""), (5, null)) + .toDF("id", "value").writeTo("table_name").append() + val replacement = Seq((10, "%new")).toDF("id", "value") + + overwriteTable(api, "contains(value, '%')", replacement) + + if (enabled || api == OverwriteApi.REPLACE_WHERE_OPTION) { + checkAnswer( + spark.table("table_name"), + Seq(Row(3, "plain"), Row(4, ""), Row(5, null), Row(10, "%new"))) + } else { + checkAnswer(spark.table("table_name"), Seq(Row(5, null), Row(10, "%new"))) + } + } + } + } + + test(s"Overwrite: literal string predicates: mixed incoming rows ($api, enabled=$enabled)") { + withLiteralStringPredicateConf(enabled) { + withTable("table_name") { + spark.sql("CREATE TABLE table_name (id int, value string) USING delta") + Seq((1, "%old"), (2, "plain"), (3, null)) + .toDF("id", "value").writeTo("table_name").append() + val replacement = Seq((10, "%new"), (11, "plain")).toDF("id", "value") + + if (enabled || api == OverwriteApi.REPLACE_WHERE_OPTION) { + val path = catalog.loadTable(Identifier.of(Array("default"), "table_name")) + .asInstanceOf[DeltaTableV2].path + val log = DeltaLog.forTable(spark, path) + val version = log.update().version + val error = intercept[AnalysisException] { + overwriteTable(api, "contains(value, '%')", replacement) + } + assert( + error.getErrorClass.stripSuffix(".INVARIANT_VIOLATION") == + "DELTA_REPLACE_WHERE_MISMATCH") + assert(log.update().version == version) + checkAnswer( + spark.table("table_name"), Seq(Row(1, "%old"), Row(2, "plain"), Row(3, null))) + } else { + overwriteTable(api, "contains(value, '%')", replacement) + checkAnswer( + spark.table("table_name"), Seq(Row(3, null), Row(10, "%new"), Row(11, "plain"))) + } + } + } + } + + test(s"Overwrite: literal string predicates: brackets and braces ($api, enabled=$enabled)") { + withLiteralStringPredicateConf(enabled) { + withTable("table_name") { + spark.sql("CREATE TABLE table_name (id int, value string) USING delta") + Seq((1, "[a]"), (2, "a{2}"), (3, "[a-c]"), (4, "[^a]"), (5, "^"), (6, "-"), + (20, "a"), (21, "aa"), (22, "b"), (23, "c"), (24, "plain")) + .toDF("id", "value").writeTo("table_name").append() + val replacement = Seq((10, "[a]"), (11, "a{2}"), (12, "[a-c]"), (13, "[^a]"), + (14, "^"), (15, "-")).toDF("id", "value") + val predicate = "contains(value, '[a]') OR contains(value, 'a{2}') OR " + + "contains(value, '[a-c]') OR contains(value, '[^a]') OR " + + "contains(value, '^') OR contains(value, '-')" + + overwriteTable(api, predicate, replacement) + + checkAnswer( + spark.table("table_name"), + Seq(Row(10, "[a]"), Row(11, "a{2}"), Row(12, "[a-c]"), Row(13, "[^a]"), + Row(14, "^"), Row(15, "-"), Row(20, "a"), Row(21, "aa"), Row(22, "b"), + Row(23, "c"), Row(24, "plain"))) + } + } + } + + test(s"Overwrite: literal string predicates: LIKE prefix wildcard ($api, enabled=$enabled)") { + withLiteralStringPredicateConf(enabled) { + withTable("table_name") { + spark.sql("CREATE TABLE table_name (id int, value string) USING delta") + Seq((1, "alice"), (2, "a%b"), (3, "a_b"), (4, "a"), (5, "plain"), (6, null)) + .toDF("id", "value").writeTo("table_name").append() + val replacement = Seq((10, "anna")).toDF("id", "value") + + overwriteTable(api, "value LIKE 'a%'", replacement) + + checkAnswer( + spark.table("table_name"), Seq(Row(5, "plain"), Row(6, null), Row(10, "anna"))) + } + } + } + } + test("Append: basic append by path") { - spark.sql("CREATE TABLE table_name (id bigint, data string) USING delta") + spark.sql(createTableSQL("table_name", "id bigint, data string")) checkAnswer(spark.table("table_name"), Seq.empty) val location = catalog.loadTable(Identifier.of(Array("default"), "table_name")) @@ -554,10 +766,10 @@ class DeltaDataFrameWriterV2Suite test("Create: basic behavior by path") { withTempDir { tempDir => val dir = tempDir.getCanonicalPath - spark.table("source").writeTo(s"delta.`$dir`").using("delta").create() + spark.table("source").writeTo(s"delta.`$dir`").using(tableProvider).create() checkAnswer( - spark.read.format("delta").load(dir), + spark.read.format(writeFormat).load(dir), Seq(Row(1L, "a"), Row(2L, "b"), Row(3L, "c"))) val table = catalog.loadTable(Identifier.of(Array("delta"), dir)) @@ -571,7 +783,7 @@ class DeltaDataFrameWriterV2Suite test("Create: using empty dataframe") { spark.table("source").where("false") - .writeTo("table_name").using("delta") + .writeTo("table_name").using(tableProvider) .tableProperty("delta.appendOnly", "true") .partitionedBy($"id").create() @@ -587,7 +799,7 @@ class DeltaDataFrameWriterV2Suite test("Replace: basic behavior using empty df") { spark.sql( - "CREATE TABLE table_name (id bigint, data string) USING delta PARTITIONED BY (id)") + createTableSQL("table_name", "id bigint, data string", partitionBy = "PARTITIONED BY (id)")) spark.sql("INSERT INTO TABLE table_name SELECT * FROM source") checkAnswer( @@ -604,7 +816,7 @@ class DeltaDataFrameWriterV2Suite spark.table("source2").where("false") .withColumn("even_or_odd", when(($"id" % 2) === 0, "even").otherwise("odd")) - .writeTo("table_name").using("delta") + .writeTo("table_name").using(tableProvider) .tableProperty("deLta.aPpeNdonly", "true").replace() checkAnswer( @@ -625,7 +837,7 @@ class DeltaDataFrameWriterV2Suite test("throw error with createOrReplace and Replace if overwriteSchema=false") { spark.sql( - "CREATE TABLE table_name (id bigint, data string) USING delta PARTITIONED BY (id)") + createTableSQL("table_name", "id bigint, data string", partitionBy = "PARTITIONED BY (id)")) spark.sql("INSERT INTO TABLE table_name SELECT * FROM source") checkAnswer( @@ -638,7 +850,7 @@ class DeltaDataFrameWriterV2Suite f: CreateTableWriter[_] => CreateTableWriter[_]): Unit = { val e = intercept[IllegalArgumentException] { val dfwV2 = df.writeTo("table_name") - .using("delta") + .using(tableProvider) .option("overwriteSchema", "false") f(dfwV2).replace() } @@ -646,7 +858,7 @@ class DeltaDataFrameWriterV2Suite val e2 = intercept[IllegalArgumentException] { val dfwV2 = df.writeTo("table_name") - .using("delta") + .using(tableProvider) .option("overwriteSchema", "false") f(dfwV2).createOrReplace() } @@ -675,9 +887,9 @@ class DeltaDataFrameWriterV2Suite test("saveAsTable overwrite mode should not do implicit casting") { val table = "not_implicit_casting" withTable(table) { - spark.sql(s"CREATE TABLE $table(id bigint, p int) USING delta PARTITIONED BY (p)") + spark.sql(s"CREATE TABLE $table(id bigint, p int) USING $tableProvider PARTITIONED BY (p)") val e = intercept[DeltaAnalysisException] { - Seq(1 -> 1).toDF("id", "p").write.mode("overwrite").format("delta").saveAsTable(table) + Seq(1 -> 1).toDF("id", "p").write.mode("overwrite").format(writeFormat).saveAsTable(table) } checkError( e.getCause.asInstanceOf[DeltaAnalysisException], @@ -689,9 +901,9 @@ class DeltaDataFrameWriterV2Suite test("implicit casting supported for saveAsTable append and writeTo operations") { val table = "implicit_casting" withTable(table) { - spark.sql(s"CREATE TABLE $table(id bigint, p int) USING delta PARTITIONED BY (p)") + spark.sql(s"CREATE TABLE $table(id bigint, p int) USING $tableProvider PARTITIONED BY (p)") // All of these should succeed with implicit casting (int -> bigint) - Seq(1 -> 1).toDF("id", "p").write.mode("append").format("delta").saveAsTable(table) + Seq(1 -> 1).toDF("id", "p").write.mode("append").format(writeFormat).saveAsTable(table) Seq(1 -> 1).toDF("id", "p").writeTo(table).append() Seq(1 -> 1).toDF("id", "p").writeTo(table).overwrite($"p" === 1) Seq(1 -> 1).toDF("id", "p").writeTo(table).overwritePartitions() @@ -702,7 +914,8 @@ class DeltaDataFrameWriterV2Suite val table = "allow_missing_columns" withTable(table) { spark.sql( - s"CREATE TABLE $table(col1 int, col2 int, col3 int) USING delta PARTITIONED BY (col3)") + s"CREATE TABLE $table(col1 int, col2 int, col3 int) USING $tableProvider " + + "PARTITIONED BY (col3)") // append Seq((0, 10)).toDF("col1", "col3").writeTo(table).append() @@ -731,7 +944,7 @@ class DeltaDataFrameWriterV2Suite test("Append: writeTo with replaceUsing option just appends") { withSQLConf( DeltaSQLConf.REPLACE_USING_OPTION_IN_DATAFRAME_WRITER_ENABLED.key -> "true") { - spark.sql("CREATE TABLE table_name (id bigint, data string) USING delta") + spark.sql(createTableSQL("table_name", "id bigint, data string")) spark.table("source").writeTo("table_name").append() @@ -758,7 +971,7 @@ class DeltaDataFrameWriterV2Suite test("Append: writeTo with replaceOn and targetAlias option just appends") { withSQLConf( DeltaSQLConf.REPLACE_ON_OPTION_IN_DATAFRAME_WRITER_ENABLED.key -> "true") { - spark.sql("CREATE TABLE table_name (id bigint, data string) USING delta") + spark.sql(createTableSQL("table_name", "id bigint, data string")) spark.table("source").writeTo("table_name").append() diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaErrorsSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaErrorsSuite.scala index 83ab4ce0ccf..3fc4a1302ea 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaErrorsSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaErrorsSuite.scala @@ -27,7 +27,7 @@ import scala.sys.process.Process // scalastyle:off import.ordering.noEmptyLine // scalastyle:off line.size.limit import org.apache.spark.sql.delta.DeltaErrors.generateDocsLink -import org.apache.spark.sql.delta.actions.{Action, Metadata, Protocol} +import org.apache.spark.sql.delta.actions.{Action, CommitInfo, Metadata, Protocol} import org.apache.spark.sql.delta.actions.TableFeatureProtocolUtils.{TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION} import org.apache.spark.sql.delta.catalog.DeltaCatalog import org.apache.spark.sql.delta.constraints.CharVarcharConstraint @@ -38,20 +38,23 @@ import org.apache.spark.sql.delta.schema.{DeltaInvariantViolationException, Inva import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaSQLTestUtils +import org.apache.spark.sql.delta.util.JsonUtils import io.delta.sql.DeltaSparkSessionExtension import org.apache.hadoop.fs.Path import org.json4s.JString +import org.mockito.Mockito.{mock, when} import org.scalatest.GivenWhenThen -import org.apache.spark.{SparkContext, SparkThrowable} +import org.apache.spark.{ErrorClassesJsonReader, SparkContext, SparkThrowable} import org.apache.spark.sql.{AnalysisException, QueryTest, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute -import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable} +import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable, CatalogTableType} import org.apache.spark.sql.catalyst.dsl.expressions._ import org.apache.spark.sql.catalyst.expressions.{AttributeReference, ExprId, Length, LessThanOrEqual, Literal, SparkVersion} import org.apache.spark.sql.catalyst.expressions.Uuid import org.apache.spark.sql.catalyst.parser.CatalystSqlParser +import org.apache.spark.sql.catalyst.util.sideBySide import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._ import org.apache.spark.sql.connector.catalog.Identifier import org.apache.spark.sql.errors.QueryErrorsBase @@ -90,8 +93,6 @@ trait DeltaErrorsSuiteBase DeltaErrors.incorrectLogStoreImplementationException(sparkConf, new Throwable()), "sourceNotDeterministicInMergeException" -> DeltaErrors.sourceNotDeterministicInMergeException(spark), - "columnMappingAdviceMessage" -> - DeltaErrors.columnRenameNotSupported, "icebergClassMissing" -> DeltaErrors.icebergClassMissing(sparkConf, new Throwable()), "tableFeatureReadRequiresWriteException" -> DeltaErrors.tableFeatureReadRequiresWriteException(requiredWriterVersion = 7), @@ -580,16 +581,16 @@ trait DeltaErrorsSuiteBase throw DeltaErrors.replaceWhereMismatchException("replaceWhereArgValue", new InvariantViolationException("Invariant violated.")) } - checkError(e, "DELTA_REPLACE_WHERE_MISMATCH", "44000", - Map("replaceWhere" -> "replaceWhereArgValue", "message" -> "Invariant violated.")) + checkError(e, "DELTA_REPLACE_WHERE_MISMATCH.INVARIANT_VIOLATION", "44000", + Map("replaceWhere" -> "replaceWhereArgValue", + "invariantViolationMessage" -> "Invariant violated.")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.replaceWhereMismatchException("replaceWhere", "badPartitions") } - checkError(e, "DELTA_REPLACE_WHERE_MISMATCH", "44000", - Map("replaceWhere" -> "replaceWhere", - "message" -> "Invalid data would be written to partitions badPartitions.")) + checkError(e, "DELTA_REPLACE_WHERE_MISMATCH.INVALID_PARTITIONS", "44000", + Map("replaceWhere" -> "replaceWhere", "badPartitions" -> "badPartitions")) } { val e = intercept[DeltaIllegalStateException] { @@ -752,9 +753,36 @@ trait DeltaErrorsSuiteBase } { val e = intercept[DeltaAnalysisException] { - throw DeltaErrors.cannotChangeDataType("example message") + throw DeltaErrors.cannotChangeDataType( + DataTypeChangeViolation.ChangeDataType( + column = "col", fromType = IntegerType, toType = LongType)) } - checkError(e, "DELTA_CANNOT_CHANGE_DATA_TYPE", "429BQ", Map("dataType" -> "example message")) + checkError(e, "DELTA_CANNOT_CHANGE_DATA_TYPE.CHANGE_DATA_TYPE", "429BQ", + Map("columnName" -> "col", "fromType" -> "INT", "toType" -> "BIGINT")) + } + { + val e = intercept[DeltaAnalysisException] { + throw DeltaErrors.cannotChangeDataType( + DataTypeChangeViolation.AddNonNullableColumn("col")) + } + checkError(e, "DELTA_CANNOT_CHANGE_DATA_TYPE.ADD_NON_NULLABLE_COLUMN", "429BQ", + Map("columnName" -> "col")) + } + { + val e = intercept[DeltaAnalysisException] { + throw DeltaErrors.cannotChangeDataType( + DataTypeChangeViolation.DropColumns(Seq("col1", "col2"))) + } + checkError(e, "DELTA_CANNOT_CHANGE_DATA_TYPE.DROP_COLUMNS", "429BQ", + Map("columnNames" -> "col1, col2")) + } + { + val e = intercept[DeltaAnalysisException] { + throw DeltaErrors.cannotChangeDataType( + DataTypeChangeViolation.TightenNullability("col")) + } + checkError(e, "DELTA_CANNOT_CHANGE_DATA_TYPE.TIGHTEN_NULLABILITY", "429BQ", + Map("columnName" -> "col")) } { val table = CatalogTable(TableIdentifier("my table"), null, null, null) @@ -912,12 +940,16 @@ trait DeltaErrorsSuiteBase val s1 = StructType(Seq(StructField("c0", IntegerType))) val s2 = StructType(Seq(StructField("c0", StringType))) val e = intercept[DeltaAnalysisException] { - throw DeltaErrors.alterTableReplaceColumnsException(s1, s2, "incompatible") + throw DeltaErrors.alterTableReplaceColumnsException( + s1, s2, DataTypeChangeViolation.ChangeDataType( + column = "c0", fromType = IntegerType, toType = StringType)) } - checkError(e, "DELTA_UNSUPPORTED_ALTER_TABLE_REPLACE_COL_OP", "0AKDC", Map( - "details" -> "incompatible", + checkError(e, "DELTA_UNSUPPORTED_ALTER_TABLE_REPLACE_COL_OP.CHANGE_DATA_TYPE", "0AKDC", Map( "oldSchema" -> s1.treeString, - "newSchema" -> s2.treeString)) + "newSchema" -> s2.treeString, + "columnName" -> "c0", + "fromType" -> "INT", + "toType" -> "STRING")) } { checkError( @@ -1021,7 +1053,7 @@ trait DeltaErrorsSuiteBase val e = intercept[DeltaIllegalStateException] { throw DeltaErrors.failRelativizePath("somePath") } - checkError(e, "DELTA_FAIL_RELATIVIZE_PATH", "XXKDS", Map( + checkError(e, "DELTA_FAIL_RELATIVIZE_PATH", "22KD1", Map( "path" -> "somePath", "config" -> DeltaSQLConf.DELTA_VACUUM_RELATIVIZE_IGNORE_ERROR.key )) @@ -1033,13 +1065,111 @@ trait DeltaErrorsSuiteBase checkError(e, "DELTA_ILLEGAL_FILE_FOUND", "XXKDS", Map("file" -> "someFile")) } { - val name = "name" - val input = "input" - val explain = "explain" - val e = intercept[DeltaIllegalArgumentException] { - throw DeltaErrors.illegalDeltaOptionException(name, input, explain) + val err = intercept[DeltaIllegalArgumentException] { + throw DeltaErrors.illegalDeltaOptionMustBeInteger("name", "input") + } + checkError(err, "DELTA_ILLEGAL_OPTION.MUST_BE_INTEGER", "42616", + Map("input" -> "input", "name" -> "name")) + } + { + val err = intercept[DeltaIllegalArgumentException] { + throw DeltaErrors.illegalDeltaOptionMustBeNonNegativeNumber("name", "input") + } + checkError(err, "DELTA_ILLEGAL_OPTION.MUST_BE_NON_NEGATIVE_NUMBER", "42616", + Map("input" -> "input", "name" -> "name")) + } + { + val err = intercept[DeltaIllegalArgumentException] { + throw DeltaErrors.illegalDeltaOptionMustBePositiveNumber("name", "input") + } + checkError(err, "DELTA_ILLEGAL_OPTION.MUST_BE_POSITIVE_NUMBER", "42616", + Map("input" -> "input", "name" -> "name")) + } + { + val err = intercept[DeltaIllegalArgumentException] { + throw DeltaErrors.illegalDeltaOptionNoEmptyColumnNames("name", "input") + } + checkError(err, "DELTA_ILLEGAL_OPTION.NO_EMPTY_COLUMN_NAMES", "42616", + Map("input" -> "input", "name" -> "name")) + } + { + val err = intercept[DeltaIllegalArgumentException] { + throw DeltaErrors.illegalDeltaOptionMustBeSizeConfiguration("name", "input") + } + checkError(err, "DELTA_ILLEGAL_OPTION.MUST_BE_SIZE_CONFIGURATION", "42616", + Map("input" -> "input", "name" -> "name")) + } + { + val err = intercept[DeltaIllegalArgumentException] { + throw DeltaErrors.illegalDeltaOptionDynamicPartitionOverwriteOnly("name", "input") + } + checkError(err, "DELTA_ILLEGAL_OPTION.DYNAMIC_PARTITION_OVERWRITE_ONLY", "42616", + Map("input" -> "input", "name" -> "name")) + } + { + val err = intercept[DeltaIllegalArgumentException] { + throw DeltaErrors.illegalDeltaOptionSchemaModeWithTimeTravel("name", "input") + } + checkError(err, "DELTA_ILLEGAL_OPTION.SCHEMA_MODE_WITH_TIME_TRAVEL", "42616", + Map("input" -> "input", "name" -> "name")) + } + { + val err = intercept[DeltaIllegalArgumentException] { + throw DeltaErrors.illegalDeltaOptionInvalidReorgParquetFormatVersion( + "name", "input", new IllegalArgumentException("bad version")) + } + checkError(err, "DELTA_ILLEGAL_OPTION.INVALID_REORG_PARQUET_FORMAT_VERSION", "42616", + Map("input" -> "input", "name" -> "name")) + } + { + val err = intercept[DeltaIllegalArgumentException] { + throw DeltaErrors.illegalDeltaOptionMustBeOneOf("name", "input", Seq("a", "b")) + } + checkError(err, "DELTA_ILLEGAL_OPTION.MUST_BE_ONE_OF", "42616", + Map("input" -> "input", "name" -> "name", "validValues" -> "a, b")) + } + { + val err = intercept[DeltaIllegalArgumentException] { + throw DeltaErrors.illegalDeltaOptionMustBeBoolean("name", "input") + } + checkError(err, "DELTA_ILLEGAL_OPTION.MUST_BE_ONE_OF", "42616", + Map("input" -> "input", "name" -> "name", "validValues" -> "'true', 'false'")) + } + { + val err = intercept[DeltaIllegalArgumentException] { + throw DeltaErrors.illegalDeltaOptionInvalidParquetFormatVersion( + "name", "input", new IllegalArgumentException("bad version")) + } + checkError(err, "DELTA_ILLEGAL_OPTION.INVALID_PARQUET_FORMAT_VERSION", "42616", + Map("input" -> "input", "name" -> "name", "causeExceptionMessage" -> "bad version")) + } + { + val err = intercept[DeltaIllegalArgumentException] { + throw DeltaErrors.illegalDeltaOptionUnrecognizedNamedArgument( + "name", "input", "read_files", "a, b") + } + checkError(err, "DELTA_ILLEGAL_OPTION.UNRECOGNIZED_NAMED_ARGUMENT", "42616", + Map("input" -> "input", "name" -> "name", "functionName" -> "read_files", + "validArguments" -> "a, b")) + } + { + val err = intercept[DeltaIllegalArgumentException] { + throw DeltaErrors.illegalDeltaOptionParquetCompressionCodecConflict( + "name", "input", "delta.parquet.compression.codec", "gzip") + } + checkError(err, "DELTA_ILLEGAL_OPTION.PARQUET_COMPRESSION_CODEC_CONFLICT", "42616", + Map("input" -> "input", "name" -> "name", + "property" -> "delta.parquet.compression.codec", "propertyValue" -> "gzip")) + } + { + val err = intercept[DeltaIllegalArgumentException] { + throw DeltaErrors.illegalDeltaOptionParquetWriterVersionConflict( + "name", "input", "delta.parquet.format.version", "v3", "V2") } - checkError(e, "DELTA_ILLEGAL_OPTION", "42616", Map("name" -> name, "input" -> input, "explain" -> explain)) + checkError(err, "DELTA_ILLEGAL_OPTION.PARQUET_WRITER_VERSION_CONFLICT", "42616", + Map("input" -> "input", "name" -> "name", + "property" -> "delta.parquet.format.version", "propertyValue" -> "v3", + "resolvedVersion" -> "V2")) } { val version = "version" @@ -1339,14 +1469,15 @@ trait DeltaErrorsSuiteBase val e = intercept[DeltaAnalysisException] { throw DeltaErrors.nonPartitionColumnAbsentException(false) } - checkError(e, "DELTA_NON_PARTITION_COLUMN_ABSENT", "KD005", Map("details" -> "")) + checkError(e, "DELTA_NON_PARTITION_COLUMN_ABSENT.ALL_PARTITION_COLUMNS", "KD005", + Map.empty[String, String]) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.nonPartitionColumnAbsentException(true) } - checkError(e, "DELTA_NON_PARTITION_COLUMN_ABSENT", "KD005", - Map("details" -> " Columns which are of NullType have been dropped.")) + checkError(e, "DELTA_NON_PARTITION_COLUMN_ABSENT.NULL_TYPE_COLUMNS_DROPPED", "KD005", + Map.empty[String, String]) } { val e = intercept[DeltaAnalysisException] { @@ -1620,10 +1751,19 @@ trait DeltaErrorsSuiteBase } { val e = intercept[DeltaIllegalArgumentException] { - throw DeltaErrors.invalidIdempotentWritesOptionsException("someReason") + throw DeltaErrors.invalidIdempotentWritesMissingWriteOptionsException() } - checkError(e, "DELTA_INVALID_IDEMPOTENT_WRITES_OPTIONS", "42616", - Map("reason" -> "someReason")) + checkError(e, + "DELTA_INVALID_IDEMPOTENT_WRITES_OPTIONS.MISSING_DATAFRAME_WRITE_OPTIONS", "42616", + Map.empty[String, String]) + } + { + val e = intercept[DeltaIllegalArgumentException] { + throw DeltaErrors.invalidIdempotentWritesMissingSessionConfsException() + } + checkError(e, + "DELTA_INVALID_IDEMPOTENT_WRITES_OPTIONS.MISSING_SESSION_CONFS", "42616", + Map.empty[String, String]) } { val e = intercept[DeltaAnalysisException] { @@ -2891,6 +3031,14 @@ trait DeltaErrorsSuiteBase "the same checkpoint to write into this table. Did you run multiple instances of the " + "same streaming query at the same time?")) } + { + val e = intercept[ConflictingMetadataDomainException] { + throw org.apache.spark.sql.delta.DeltaErrors + .conflictingMetadataDomainException("delta.liquid") + } + checkError(e, "DELTA_CONFLICTING_METADATA_DOMAIN", "2D521", + Map("domain" -> "delta.liquid")) + } { val e = intercept[io.delta.exceptions.ConcurrentWriteException] { throw org.apache.spark.sql.delta.DeltaErrors.concurrentWriteException(None) @@ -2900,12 +3048,39 @@ trait DeltaErrorsSuiteBase "current transaction read the table.")) } { + // No conflicting commit: The plain DELTA_PROTOCOL_CHANGED superclass should be used. val e = intercept[io.delta.exceptions.ProtocolChangedException] { throw org.apache.spark.sql.delta.DeltaErrors.protocolChangedException(None) } - checkError(e, "DELTA_PROTOCOL_CHANGED", "2D521", Map.empty[String, String]) - assert(e.getMessage.contains("The protocol version of the Delta table has been changed " + - "by a concurrent update.")) + checkError(e, "DELTA_PROTOCOL_CHANGED", "2D521", + Map("docLink" -> generateDocsLink("/concurrency-control.html"))) + } + { + // A conflicting commit at version 0 indicates that a concurrent writer created the table while this + // transaction wrote to an empty directory. + val conflictingCommit = CommitInfo.empty(version = Some(0)) + .copy(timestamp = new Timestamp(0), operationParameters = Map.empty) + val e = intercept[io.delta.exceptions.ProtocolChangedException] { + throw org.apache.spark.sql.delta.DeltaErrors + .protocolChangedException(Some(conflictingCommit)) + } + checkError(e, "DELTA_PROTOCOL_CHANGED.WRITE_TO_EMPTY_DIRECTORY", "2D521", + Map( + "docLink" -> generateDocsLink("/concurrency-control.html"), + "conflictingCommit" -> JsonUtils.toJson(conflictingCommit))) + } + { + // A conflicting commit at a >0 version should be reported via the CONFLICTING_COMMIT subclass. + val conflictingCommit = CommitInfo.empty(version = Some(1)) + .copy(timestamp = new Timestamp(0), operationParameters = Map.empty) + val e = intercept[io.delta.exceptions.ProtocolChangedException] { + throw org.apache.spark.sql.delta.DeltaErrors + .protocolChangedException(Some(conflictingCommit)) + } + checkError(e, "DELTA_PROTOCOL_CHANGED.CONFLICTING_COMMIT", "2D521", + Map( + "docLink" -> generateDocsLink("/concurrency-control.html"), + "conflictingCommit" -> JsonUtils.toJson(conflictingCommit))) } { val e = intercept[io.delta.exceptions.MetadataChangedException] { @@ -2988,6 +3163,33 @@ trait DeltaErrorsSuiteBase } } + test("raising error class without subclass when subclasses exist") { + // DELTA_METADATA_MISMATCH defines subclasses, so a fully-qualified subclass extends the + // main-class template. This guards that the test exercises the has-subclasses scenario. + val mainTemplate = DeltaThrowableHelper.getMainMessageTemplate("DELTA_METADATA_MISMATCH") + val subTemplate = + DeltaThrowableHelper.getSubMessageTemplate("DELTA_METADATA_MISMATCH.SCHEMA_MISMATCH") + assert(subTemplate.nonEmpty) + assert(DeltaThrowableHelper.getMessageTemplate("DELTA_METADATA_MISMATCH") == mainTemplate) + assert(DeltaThrowableHelper.getMessageTemplate("DELTA_METADATA_MISMATCH.SCHEMA_MISMATCH") == + mainTemplate + " " + subTemplate) + + val e = intercept[DeltaIllegalArgumentException] { + throw new DeltaIllegalArgumentException(errorClass = "DELTA_METADATA_MISMATCH") + } + assert(e.getErrorClass == "DELTA_METADATA_MISMATCH") + // Assert getMessage directly rather than via checkError: the point is to verify that getMessage + // itself renders a bare class that has subclasses, which checkError does not exercise. + assert(e.getMessage == "[DELTA_METADATA_MISMATCH] " + mainTemplate) + + // getParameterNames must also resolve for a bare main class that has subclasses. Use + // DELTA_CONCURRENT_APPEND, whose main template and subclass both carry parameters. + assert(DeltaThrowableHelper.getParameterNames("DELTA_CONCURRENT_APPEND", errorSubClass = null) + .toSeq == Seq("operation", "tableName", "version")) + assert(DeltaThrowableHelper.getParameterNames("DELTA_CONCURRENT_APPEND", "WITH_PARTITION_HINT") + .toSeq == Seq("operation", "tableName", "version", "partitionValues", "docLink")) + } + test("throwChangelogReadFailed preserves SparkThrowable cause and wraps others") { // A cause that already carries a Spark error class is rethrown unchanged. val sparkThrowableCause = new DeltaAnalysisException( @@ -3006,6 +3208,97 @@ trait DeltaErrorsSuiteBase Map.empty[String, String]) } + // Message templates keyed by the fully-qualified error class (including any sub-class). + protected lazy val deltaErrorClassToInfoMap = + new ErrorClassesJsonReader(Seq(DeltaThrowableHelper.deltaErrorClassSource)).errorInfoMap + + /** + * Asserts that the message of sub-class `subClass` is byte-for-byte identical across every error + * class in `errorClasses`. Duplicated messages that are meant to stay in sync drift apart over + * time; this check catches accidental divergence so a single edit cannot silently desync them. + */ + protected def assertIdenticalSubClassMessage( + errorClasses: Seq[String], subClass: String): Unit = { + // Group the error classes by their message for `subClass`. They must all share one message, + // i.e. the grouping must collapse to a single entry. + val classesByMessage = errorClasses.groupBy { errorClass => + deltaErrorClassToInfoMap(errorClass).subClass.getOrElse(Map.empty)(subClass).messageTemplate + } + if (classesByMessage.size > 1) { + val groups = + classesByMessage.values.map(_.sorted.mkString("{", ", ", "}")).mkString(", ") + val differingMessages = classesByMessage.keys.take(2).toSeq + val diff = sideBySide(differingMessages(0), differingMessages(1)).mkString("\n") + fail( + s"The '$subClass' sub-class message must be identical across " + + s"${errorClasses.mkString(", ")}, but these groups use different messages: " + + s"$groups\nExample difference:\n$diff") + } + } + + test("ENABLE_COLUMN_MAPPING advice is identical across the DROP/RENAME COLUMN errors") { + assertIdenticalSubClassMessage( + Seq("DELTA_UNSUPPORTED_DROP_COLUMN", "DELTA_UNSUPPORTED_RENAME_COLUMN"), + "ENABLE_COLUMN_MAPPING") + } + + test("DROP/RENAME COLUMN advise enabling column mapping") { + val minProtocol = ColumnMappingTableFeature.minProtocolVersion + val versionParams = Map( + "readerVersion" -> minProtocol.minReaderVersion.toString, + "writerVersion" -> minProtocol.minWriterVersion.toString) + + // Without a column-mapping suggestion, only the base message (no sub-class) is used. + checkError( + intercept[DeltaAnalysisException] { + throw DeltaErrors.dropColumnNotSupported(suggestUpgrade = false) + }, + "DELTA_UNSUPPORTED_DROP_COLUMN", "0AKDC", Map.empty[String, String]) + + checkError( + intercept[DeltaAnalysisException] { + throw DeltaErrors.dropColumnNotSupported(suggestUpgrade = true) + }, + "DELTA_UNSUPPORTED_DROP_COLUMN.ENABLE_COLUMN_MAPPING", "0AKDC", versionParams) + + checkError( + intercept[DeltaAnalysisException] { + throw DeltaErrors.columnRenameNotSupported + }, + "DELTA_UNSUPPORTED_RENAME_COLUMN.ENABLE_COLUMN_MAPPING", "0AKDC", versionParams) + } + + test("catalog-managed maintenance operation uses the catalog allowlist") { + val managedSnapshot = mock(classOf[SnapshotDescriptor]) + when(managedSnapshot.isCatalogOwned).thenReturn(true) + val property = CatalogManagedTableMaintenanceOperation.ALLOWED_OPERATIONS_PROPERTY + + def tableWithOperations(value: String): CatalogTable = CatalogTable( + identifier = TableIdentifier("table"), + tableType = CatalogTableType.MANAGED, + storage = CatalogStorageFormat.empty.copy( + properties = Map(property -> value)), + schema = new StructType()) + + DeltaErrors.checkCatalogManagedTableOperationAllowed( + CatalogManagedTableMaintenanceOperation.DATA_CLEANUP, + managedSnapshot, + Some(tableWithOperations("DATA_CLEANUP,METADATA_CLEANUP"))) + + intercept[DeltaUnsupportedOperationException] { + DeltaErrors.checkCatalogManagedTableOperationAllowed( + CatalogManagedTableMaintenanceOperation.DATA_REORGANIZATION, + managedSnapshot, + Some(tableWithOperations("DATA_CLEANUP,METADATA_CLEANUP"))) + } + intercept[DeltaUnsupportedOperationException] { + DeltaErrors.checkCatalogManagedTableOperationAllowed( + CatalogManagedTableMaintenanceOperation.DATA_CLEANUP, + managedSnapshot, + None) + } + } + private def setCustomContext(session: SparkSession, context: SparkContext): Unit = { val scField = session.getClass.getDeclaredField("sparkContext") scField.setAccessible(true) diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaFastDropFeatureSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaFastDropFeatureSuite.scala index aec12180acb..4e0261cdcae 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaFastDropFeatureSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaFastDropFeatureSuite.scala @@ -635,7 +635,7 @@ class DeltaFastDropFeatureSuite val snapshot = log.update() val dvPath = DeletionVectorDescriptor .urlEncodedRelativePathIfExists(col("deletionVector"), log.dataPath) - val isDVTombstone = DeletionVectorDescriptor.isDeletionVectorPath(col("path")) + val isDVTombstone = DeletionVectorDescriptor.isSparkImplDeletionVectorPath(col("path")) val isInlineDeletionVector = DeletionVectorDescriptor.isInline(col("deletionVector")) val uniqueDvsFromParquetRemoveFiles = snapshot diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaGenerateSymlinkManifestSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaGenerateSymlinkManifestSuite.scala index 5f8e0196f69..aad9e72cf28 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaGenerateSymlinkManifestSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaGenerateSymlinkManifestSuite.scala @@ -24,7 +24,7 @@ import org.apache.spark.sql.delta.DeltaOperations.Delete import org.apache.spark.sql.delta.commands.DeltaGenerateCommand import org.apache.spark.sql.delta.hooks.GenerateSymlinkManifest import org.apache.spark.sql.delta.sources.DeltaSQLConf -import org.apache.spark.sql.delta.test.DeltaSQLCommandTest +import org.apache.spark.sql.delta.test.{DeltaSQLCommandTest, DeltaSQLTestUtils} import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.util.DeltaFileOperations import org.apache.hadoop.conf.Configuration @@ -45,6 +45,7 @@ class DeltaGenerateSymlinkManifestSuite trait DeltaGenerateSymlinkManifestSuiteBase extends DeltaGenerateSymlinkManifestTestHelper + with DeltaSQLTestUtils with DeletionVectorsTestUtils with DeltaTestUtilsForTempViews { @@ -78,6 +79,35 @@ trait DeltaGenerateSymlinkManifestSuiteBase } } + test("symlink manifest generation is blocked for a catalog-managed table") { + withCatalogManagedTable(createTable = false) { tableName => + spark.sql(s"CREATE TABLE $tableName (id INT) USING delta TBLPROPERTIES " + + s"('delta.feature.${CatalogOwnedTableFeature.name}' = 'supported', " + + "'delta.enableDeletionVectors' = 'false')") + assertManifest(tableName, expectSameFiles = false, expectedNumFiles = 0) + + checkError( + intercept[DeltaUnsupportedOperationException] { + spark.sql(s"GENERATE symlink_format_manifest FOR TABLE $tableName") + }, + "DELTA_UNSUPPORTED_CATALOG_MANAGED_TABLE_OPERATION", + parameters = Map("operation" -> "GENERATE_SYMLINK_FORMAT_MANIFEST")) + assertManifest(tableName, expectSameFiles = false, expectedNumFiles = 0) + + spark.sql( + s"ALTER TABLE $tableName SET TBLPROPERTIES " + + s"('${DeltaConfigs.SYMLINK_FORMAT_MANIFEST_ENABLED.key}' = 'true')") + val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) + val versionBefore = deltaLog.update().version + + spark.sql(s"INSERT INTO $tableName VALUES (1)") + + assert(deltaLog.update().version === versionBefore + 1) + assertManifest(tableName, expectSameFiles = false, expectedNumFiles = 0) + checkAnswer(spark.table(tableName), Row(1)) + } + } + test("basic case: SQL command - throw error on bad tables") { var e: Exception = intercept[AnalysisException] { spark.sql("GENERATE symlink_format_manifest FOR TABLE nonExistentTable") @@ -874,4 +904,3 @@ class SymlinkManifestFailureTestFileSystem extends RawLocalFileSystem { object SymlinkManifestFailureTestFileSystem { val SCHEME = "testScheme" } - diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaHistoryManagerSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaHistoryManagerSuite.scala index 3071ec8af98..e6d08534d0a 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaHistoryManagerSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaHistoryManagerSuite.scala @@ -662,6 +662,68 @@ abstract class DeltaHistoryManagerBase extends DeltaTimeTravelTests { } } + test("getAllStagedAndCommittedFileActions returns add and remove actions across the commit range") { + val tblName = "delta_get_all_file_actions" + withTable(tblName) { + sql(s"CREATE TABLE $tblName (id INT) USING delta") + sql(s"INSERT INTO $tblName VALUES (1), (2), (3)") // adds + sql(s"INSERT INTO $tblName VALUES (4), (5)") // adds + sql(s"INSERT OVERWRITE $tblName VALUES (6)") // removes the earlier files, adds new + val deltaLog = DeltaTableV2(spark, TableIdentifier(tblName)).deltaLog + val latest = deltaLog.update().version + val historyManager = deltaLog.history + + { + val allActions = historyManager.getAllStagedAndCommittedFileActions(0, latest).collect() + assert(allActions.exists(_.add != null), "expected AddFile actions in the range") + assert(allActions.exists(_.remove != null), "expected RemoveFile actions in the range") + + // Up to the version before the overwrite there are only adds (no removes yet), so the + // adds getAllStagedAndCommittedFileActions returns must equal the files of the snapshot as of that version. + val beforeOverwriteVersion = latest - 1 + val beforeOverwrite = historyManager.getAllStagedAndCommittedFileActions(0, beforeOverwriteVersion).collect() + assert(!beforeOverwrite.exists(_.remove != null), + "no file was removed before the overwrite commit") + val addPathsBeforeOverwrite = beforeOverwrite.filter(_.add != null).map(_.add.path).toSet + val snapshotPaths = + deltaLog.getSnapshotAt(beforeOverwriteVersion).allFiles.collect().map(_.path).toSet + assert(addPathsBeforeOverwrite === snapshotPaths, + "adds up to the pre-overwrite version should equal that snapshot's files") + + // An inverted range (startVersion > endVersion) is rejected. + intercept[IllegalArgumentException] { + historyManager.getAllStagedAndCommittedFileActions(latest + 1, latest) + } + } + } + } + + test("getRemoveFileActions returns only the removes within the commit range") { + val tblName = "delta_get_remove_file_actions" + withTable(tblName) { + sql(s"CREATE TABLE $tblName (id INT) USING delta") + sql(s"INSERT INTO $tblName VALUES (1), (2), (3)") // adds only + sql(s"INSERT OVERWRITE $tblName VALUES (4)") // removes the earlier files + val deltaLog = DeltaTableV2(spark, TableIdentifier(tblName)).deltaLog + val latest = deltaLog.update().version + val historyManager = deltaLog.history + + { + val removes = historyManager.getRemoveFileActions(0, latest).collect() + assert(removes.nonEmpty, "expected removed files in the range") + assert(removes.forall(_.path != null)) + + // A range that excludes the overwrite commit has no removes. + assert(historyManager.getRemoveFileActions(0, latest - 1).count() === 0) + + // An inverted range (startVersion > endVersion) is rejected. + intercept[IllegalArgumentException] { + historyManager.getRemoveFileActions(latest + 1, latest) + } + } + } + } + test("getCommitFromNonICTRange should handle empty history by throwing proper error") { val tblName = "delta_table" withTable(tblName) { @@ -676,8 +738,10 @@ abstract class DeltaHistoryManagerBase extends DeltaTimeTravelTests { deltaLog.history.getCommitFromNonICTRange(0, 1, start) } - assert(e.getMessage.contains("DELTA_NO_COMMITS_FOUND")) - assert(e.getMessage.contains(deltaLog.logPath.toString)) + checkError( + e, + "DELTA_NO_COMMITS_FOUND", + parameters = Map("logPath" -> deltaLog.logPath.toString)) } } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaIncrementalSetTransactionsSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaIncrementalSetTransactionsSuite.scala index eb953130b28..92920398a5b 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaIncrementalSetTransactionsSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaIncrementalSetTransactionsSuite.scala @@ -145,21 +145,13 @@ class DeltaIncrementalSetTransactionsSuite withSQLConf(DeltaSQLConf.DELTA_WRITE_SET_TRANSACTIONS_IN_CRC.key -> "true") { DeltaLog.clearCache() commitSetTxn(deltaLog, "app-1", version = 2, lastUpdated = 2) // 2nd commit - if (catalogOwnedDefaultCreationEnabledInTests) { - // For CatalogOwned tables with Row Tracking enabled (QoL feature), the commit DOES - // trigger state reconstruction via RowId.assignFreshRowIds -> extractHighWatermark -> - // domainMetadata access. This sets _computedStateTriggered=true, causing - // setTransactions to be included in the CRC immediately (not lazily as in the - // non-CatalogOwned case). - assert(deltaLog.update().checksumOpt.get.setTransactions.nonEmpty) // crc has set-txn - assertSetTransactions(deltaLog, expectedTxns = Map("app-1" -> 2), viaCRC = true) - } else { - // By default, commit doesn't trigger stateReconstruction and so the - // incremental CRC won't have setTransactions present until `setTransactions` API is - // explicitly invoked before the commit. - assert(deltaLog.update().checksumOpt.get.setTransactions.isEmpty) // crc has no set-txn - assertSetTransactions(deltaLog, expectedTxns = Map("app-1" -> 2), viaCRC = false) - } + // Commit doesn't trigger stateReconstruction and so the incremental CRC won't have + // setTransactions present until `setTransactions` API is explicitly invoked before the + // commit. This also holds for CatalogOwned tables with Row Tracking enabled: the + // `domainMetadata` read done by RowId.assignFreshRowIds -> extractHighWatermark is + // served from the CRC, so it no longer forces a state reconstruction. + assert(deltaLog.update().checksumOpt.get.setTransactions.isEmpty) // crc has no set-txn + assertSetTransactions(deltaLog, expectedTxns = Map("app-1" -> 2), viaCRC = false) DeltaLog.clearCache() // Do commit after forcing computeState. Now SetTransaction tracking will start. @@ -223,18 +215,11 @@ class DeltaIncrementalSetTransactionsSuite DeltaLog.clearCache() withSQLConf(DeltaSQLConf.DELTA_WRITE_SET_TRANSACTIONS_IN_CRC.key -> "true") { commitSetTxn(log, "app-1", version = 1, lastUpdated = 1) - if (catalogOwnedDefaultCreationEnabledInTests) { - // For CatalogOwned tables with Row Tracking enabled (QoL feature), the commit DOES - // trigger state reconstruction via RowId.assignFreshRowIds, which sets - // _computedStateTriggered=true, causing setTransactions to be included in CRC. - assert(log.update().checksumOpt.flatMap(_.setTransactions).nonEmpty) - } else { - // During 1st commit, the feature is enabled. But still the new commit crc shouldn't - // contain the [[SetTransaction]] actions as we don't have an estimate of how many - // [[SetTransaction]] actions might be already part of this table till now. - // So incremental computation of [[SetTransaction]] won't trigger. - assert(log.update().checksumOpt.flatMap(_.setTransactions).isEmpty) - } + // During 1st commit, the feature is enabled. But still the new commit crc shouldn't + // contain the [[SetTransaction]] actions as we don't have an estimate of how many + // [[SetTransaction]] actions might be already part of this table till now. + // So incremental computation of [[SetTransaction]] won't trigger. + assert(log.update().checksumOpt.flatMap(_.setTransactions).isEmpty) if (computeStatePreloaded) { // Calling `validateChecksum` will pre-load the computeState @@ -242,22 +227,19 @@ class DeltaIncrementalSetTransactionsSuite } commitSetTxn(log, "app-1", version = 100, lastUpdated = 1) - if (catalogOwnedDefaultCreationEnabledInTests) { - // For CatalogOwned tables with Row Tracking enabled, state reconstruction is - // triggered during every commit (via RowId.assignFreshRowIds), so setTransactions are - // always included in CRC regardless of whether computeState was preloaded. - assert(log.update().checksumOpt.flatMap(_.setTransactions).nonEmpty) - } else { - // During 2nd commit, we have following 2 cases: - // 1. If `computeStatePreloaded` is set, then the Snapshot has already calculated - // computeState, and so we have estimate of number of SetTransactions till this - // point. So next commit will trigger incremental computation of [[SetTransaction]] - // 2. If `computeStatePreloaded` is not set, then Snapshot doesn't have computeState - // pre-computed. So next commit will not trigger incremental computation of - // [[SetTransaction]]. - assert(log.update().checksumOpt.flatMap(_.setTransactions).nonEmpty === - computeStatePreloaded) - } + // During 2nd commit, we have following 2 cases: + // 1. If `computeStatePreloaded` is set, then the Snapshot has already calculated + // computeState, and so we have estimate of number of SetTransactions till this + // point. So next commit will trigger incremental computation of [[SetTransaction]] + // 2. If `computeStatePreloaded` is not set, then Snapshot doesn't have computeState + // pre-computed. So next commit will not trigger incremental computation of + // [[SetTransaction]]. + // Note that commits themselves never pre-compute the state, not even for CatalogOwned + // tables with Row Tracking enabled: the `domainMetadata` read performed by + // RowId.assignFreshRowIds is served from the CRC instead of forcing a state + // reconstruction. + assert(log.update().checksumOpt.flatMap(_.setTransactions).nonEmpty === + computeStatePreloaded) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaInsertIntoColumnOrderSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaInsertIntoColumnOrderSuite.scala index dd52f649c27..9b8e6af4b5f 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaInsertIntoColumnOrderSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaInsertIntoColumnOrderSuite.scala @@ -27,7 +27,7 @@ import org.apache.spark.sql.internal.SQLConf */ class DeltaInsertIntoColumnOrderSuite extends DeltaInsertIntoTest { - override def beforeAll(): Unit = { + override protected def beforeAll(): Unit = { super.beforeAll() spark.conf.set(SQLConf.ANSI_ENABLED.key, "true") } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaInsertIntoImplicitCastSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaInsertIntoImplicitCastSuite.scala index 903f42ef1da..dc3c421c30d 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaInsertIntoImplicitCastSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaInsertIntoImplicitCastSuite.scala @@ -33,11 +33,8 @@ import org.apache.spark.sql.types._ * [[DeltaInsertIntoTest]] for a list of these INSERT operations covered. */ trait DeltaInsertIntoImplicitCastBase extends DeltaInsertIntoTest { - override def beforeAll(): Unit = { + override protected def beforeAll(): Unit = { super.beforeAll() - // Enable the null expansion fix by preserving NULL source structs in INSERT operations. - // Without this fix, NULL source structs are incorrectly expanded to structs with NULL fields. - spark.conf.set(DeltaSQLConf.DELTA_MERGE_PRESERVE_NULL_SOURCE_STRUCTS.key, "true") spark.conf.set(SQLConf.ANSI_ENABLED.key, "true") } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaInsertIntoMissingColumnSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaInsertIntoMissingColumnSuite.scala index d340c151229..5e98fc56d5c 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaInsertIntoMissingColumnSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaInsertIntoMissingColumnSuite.scala @@ -29,7 +29,7 @@ import org.apache.spark.sql.types._ */ class DeltaInsertIntoMissingColumnSuite extends DeltaInsertIntoTest { - override def beforeAll(): Unit = { + override protected def beforeAll(): Unit = { super.beforeAll() spark.conf.set(SQLConf.ANSI_ENABLED.key, "true") } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaInsertIntoSchemaEvolutionSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaInsertIntoSchemaEvolutionSuite.scala index e56c7dc5e15..e540a7d4f7c 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaInsertIntoSchemaEvolutionSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaInsertIntoSchemaEvolutionSuite.scala @@ -25,9 +25,9 @@ import org.apache.spark.sql.SaveMode import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ -trait DeltaInsertIntoEvolutionSuiteBase extends DeltaInsertIntoTest { +trait DeltaInsertIntoEvolutionSuiteBase extends DeltaInsertIntoTest with DeltaTableProvider { - override def beforeAll(): Unit = { + override protected def beforeAll(): Unit = { super.beforeAll() spark.conf.set(SQLConf.ANSI_ENABLED.key, "true") } @@ -295,8 +295,8 @@ class DeltaInsertIntoSchemaEvolutionSuite extends DeltaInsertIntoEvolutionSuiteB test("insert by name with extra top-level column and implicit cast fails " + "when byNameSchemaEvolution is disabled") { withTable("target") { - sql("CREATE TABLE target (a INT, b INT) USING DELTA") - withSQLConf( + sql(createTableSQL("target", "a INT, b INT")) + withConf( DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key -> "true", DeltaSQLConf.DELTA_INSERT_BY_NAME_SCHEMA_EVOLUTION_ENABLED.key -> "false") { val ex = intercept[SparkThrowable] { diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaInsertIntoTest.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaInsertIntoTest.scala index e8338f08461..dc3f0bf68b6 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaInsertIntoTest.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaInsertIntoTest.scala @@ -42,7 +42,8 @@ import org.apache.spark.sql.types.StructType trait DeltaInsertIntoTest extends QueryTest with DeltaDMLTestUtilsPathBased - with DeltaSQLCommandTest { + with DeltaSQLCommandTest + with DeltaTableProvider { val catalogName = "spark_catalog" @@ -227,7 +228,7 @@ trait DeltaInsertIntoTest withSchemaEvolution: Boolean): Unit = spark.read.table("source").write.mode(mode) .option("mergeSchema", withSchemaEvolution.toString) - .format("delta") + .format(writeFormat) .insertInto("target") } @@ -243,7 +244,7 @@ trait DeltaInsertIntoTest withSchemaEvolution: Boolean): Unit = { spark.read.table("source").write.mode(mode) .option("mergeSchema", withSchemaEvolution.toString) - .format("delta") + .format(writeFormat) .saveAsTable("target") } } @@ -261,7 +262,7 @@ trait DeltaInsertIntoTest val deltaLog = DeltaLog.forTable(spark, TableIdentifier("target")) spark.read.table("source").write.mode(mode) .option("mergeSchema", withSchemaEvolution.toString) - .format("delta") + .format(writeFormat) .save(deltaLog.dataPath.toString) } } @@ -277,15 +278,12 @@ trait DeltaInsertIntoTest whereCol: String, whereValue: Int, withSchemaEvolution: Boolean): Unit = { - withSQLConf( - DeltaSQLConf.REPLACE_ON_OPTION_IN_DATAFRAME_WRITER_ENABLED.key -> "true") { - spark.read.table("source").write.mode(mode) - .option("replaceOn", s"t.$whereCol = $whereValue") - .option("targetAlias", "t") - .option("mergeSchema", withSchemaEvolution.toString) - .format("delta") - .insertInto("target") - } + spark.read.table("source").write.mode(mode) + .option("replaceOn", s"t.$whereCol = $whereValue") + .option("targetAlias", "t") + .option("mergeSchema", withSchemaEvolution.toString) + .format(writeFormat) + .insertInto("target") } } @@ -300,16 +298,13 @@ trait DeltaInsertIntoTest whereCol: String, whereValue: Int, withSchemaEvolution: Boolean): Unit = { - withSQLConf( - DeltaSQLConf.REPLACE_ON_OPTION_IN_DATAFRAME_WRITER_ENABLED.key -> "true") { - val deltaLog = DeltaLog.forTable(spark, TableIdentifier("target")) - spark.read.table("source").write.mode(mode) - .option("replaceOn", s"t.$whereCol = $whereValue") - .option("targetAlias", "t") - .option("mergeSchema", withSchemaEvolution.toString) - .format("delta") - .save(deltaLog.dataPath.toString) - } + val deltaLog = DeltaLog.forTable(spark, TableIdentifier("target")) + spark.read.table("source").write.mode(mode) + .option("replaceOn", s"t.$whereCol = $whereValue") + .option("targetAlias", "t") + .option("mergeSchema", withSchemaEvolution.toString) + .format(writeFormat) + .save(deltaLog.dataPath.toString) } } @@ -328,7 +323,7 @@ trait DeltaInsertIntoTest .mode(mode) .option("partitionOverwriteMode", "dynamic") .option("mergeSchema", withSchemaEvolution.toString) - .format("delta") + .format(writeFormat) .insertInto("target") } @@ -404,7 +399,7 @@ trait DeltaInsertIntoTest .writeStream .option("checkpointLocation", checkpointLocation.toString) .option("mergeSchema", withSchemaEvolution.toString) - .format("delta") + .format(writeFormat) .trigger(Trigger.AvailableNow()) .toTable("target") try { @@ -494,6 +489,30 @@ trait DeltaInsertIntoTest def toDF: DataFrame = readFromJSON(data, schema) } + protected def checkExpectedRows( + actual: => DataFrame, + insert: Insert, + initialData: TestData, + insertData: TestData): Unit = { + checkAnswer(actual, insert.expectedResult(initialData.toDF, insertData.toDF)) + } + + protected def checkExpectedRows(actual: => DataFrame, expectedData: TestData): Unit = { + checkAnswer(actual, expectedData.toDF) + } + + protected def createTableFromTestData( + tableName: String, + data: TestData, + partitionBy: Seq[String] = Seq.empty): Unit = { + val writer = data.toDF.write.format(writeFormat) + if (partitionBy.nonEmpty) { + writer.partitionBy(partitionBy: _*) + } + writer.saveAsTable(tableName) + } + + /** * Test runner to cover INSERT operations defined above. * @param name Test name @@ -529,14 +548,10 @@ trait DeltaInsertIntoTest for (insert <- inserts) { test(s"${insert.name} - $name") { withTable("source", "target") { - val writer = initialData.toDF.write.format("delta") - if (partitionBy.nonEmpty) { - writer.partitionBy(partitionBy: _*) - } - writer.saveAsTable("target") + createTableFromTestData("target", initialData, partitionBy) // Write the data to insert to a table so that we can use it in both SQL and dataframe // writer inserts. - insertData.toDF.write.format("delta").saveAsTable("source") + createTableFromTestData("source", insertData) def runInsert(): Unit = insert.runInsert( @@ -546,18 +561,18 @@ trait DeltaInsertIntoTest withSchemaEvolution = withSchemaEvolution ) - withSQLConf(confs: _*) { + withConf(confs: _*) { expectedResult match { case ExpectedResult.Success(expectedSchema: StructType) => runInsert() val target = spark.read.table("target") assert(target.schema === expectedSchema) - checkAnswer(target, insert.expectedResult(initialData.toDF, insertData.toDF)) + checkExpectedRows(target, insert, initialData, insertData) case ExpectedResult.Success(expectedData: TestData) => runInsert() val target = spark.read.table("target") assert(target.schema === expectedData.schema) - checkAnswer(spark.read.table("target"), expectedData.toDF) + checkExpectedRows(target, expectedData) case ExpectedResult.Failure(checkError) => val ex = if (insert == StreamingInsert) { intercept[StreamingQueryException] { diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaInsertReplaceUsingDFWriterTests.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaInsertReplaceUsingDFWriterTests.scala index 83493e1ed14..5ab6f4eecd0 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaInsertReplaceUsingDFWriterTests.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaInsertReplaceUsingDFWriterTests.scala @@ -1308,12 +1308,11 @@ trait DeltaInsertReplaceUsingDFWriterTests target = path, replaceUsingCols = "") }, - condition = "DELTA_ILLEGAL_OPTION", + condition = "DELTA_ILLEGAL_OPTION.NO_EMPTY_COLUMN_NAMES", sqlState = Some("42616"), parameters = Map( "name" -> "replaceUsing", - "input" -> "", - "explain" -> "must not contain empty column names") + "input" -> "") ) } } @@ -1334,12 +1333,11 @@ trait DeltaInsertReplaceUsingDFWriterTests target = path, replaceUsingCols = optionValue) }, - condition = "DELTA_ILLEGAL_OPTION", + condition = "DELTA_ILLEGAL_OPTION.NO_EMPTY_COLUMN_NAMES", sqlState = Some("42616"), parameters = Map( "name" -> "replaceUsing", - "input" -> optionValue, - "explain" -> "must not contain empty column names") + "input" -> optionValue) ) } } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaInsertReplaceWhereTargetAliasSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaInsertReplaceWhereTargetAliasSuite.scala index 8db29dcb605..084868f08a9 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaInsertReplaceWhereTargetAliasSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaInsertReplaceWhereTargetAliasSuite.scala @@ -526,10 +526,10 @@ class DeltaInsertReplaceWhereTargetAliasSuite .option("replaceWhere", "t.id = 1") .saveAsTable(testTableName) }, - condition = "DELTA_REPLACE_WHERE_MISMATCH", + condition = "DELTA_REPLACE_WHERE_MISMATCH.INVARIANT_VIOLATION", parameters = Map( "replaceWhere" -> "t.id = 1", - "message" -> "(?s).*id : 99.*"), + "invariantViolationMessage" -> "(?s).*id : 99.*"), matchPVals = true) } } @@ -827,10 +827,10 @@ class DeltaInsertReplaceWhereTargetAliasSuite .option("replaceWhere", "t.data IS NULL") .saveAsTable(testTableName) }, - condition = "DELTA_REPLACE_WHERE_MISMATCH", + condition = "DELTA_REPLACE_WHERE_MISMATCH.INVARIANT_VIOLATION", parameters = Map( "replaceWhere" -> "t.data IS NULL", - "message" -> "(?s).*data : non-null.*"), + "invariantViolationMessage" -> "(?s).*data : non-null.*"), matchPVals = true) } } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaLogMinorCompactionSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaLogMinorCompactionSuite.scala index 7e5f3663e3b..1f47fc6a2e8 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaLogMinorCompactionSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaLogMinorCompactionSuite.scala @@ -35,32 +35,10 @@ import org.apache.spark.sql.test.SharedSparkSession class DeltaLogMinorCompactionSuite extends QueryTest with SharedSparkSession with DeltaSQLCommandTest + with DeltaMinorCompactionTestUtils with DeltaSQLTestUtils with CatalogOwnedTestBaseSuite { - /** Helper method to do minor compaction of [[DeltaLog]] from [startVersion, endVersion] */ - private def minorCompactDeltaLog( - tablePath: String, - startVersion: Long, - endVersion: Long): Unit = { - val deltaLog = DeltaLog.forTable(spark, tablePath) - val logReplay = new InMemoryLogReplay( - minFileRetentionTimestamp = None, - minSetTransactionRetentionTimestamp = None) - val hadoopConf = deltaLog.newDeltaHadoopConf() - - (startVersion to endVersion).foreach { versionToRead => - val file = FileNames.unsafeDeltaFile(deltaLog.logPath, versionToRead) - val actionsIterator = deltaLog.store.readAsIterator(file, hadoopConf).map(Action.fromJson) - logReplay.append(versionToRead, actionsIterator) - } - deltaLog.store.write( - path = FileNames.compactedDeltaFile(deltaLog.logPath, startVersion, endVersion), - actions = logReplay.checkpoint.map(_.json).toIterator, - overwrite = true, - hadoopConf = hadoopConf) - } - // Helper method to validate a commit. protected def validateCommit( log: DeltaLog, diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaLogSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaLogSuite.scala index 8ef9936f58e..a62a0054718 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaLogSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaLogSuite.scala @@ -808,7 +808,7 @@ class DeltaLogSuite extends QueryTest val expected = commits.map(c => c.version -> c.getActionsIterator().processAndClose(_.toList)) val parallelActions = - DeltaFileProviderUtils.parallelReadAndParseDeltaFilesAsIterator(spark, log, commits) + DeltaFileProviderUtils.parallelReadAndParseDeltaFilesAsIterator(spark, commits) .map(_.processAndClose(_.toList)) val actual = commits.map(_.version).zip(parallelActions) assert(actual === expected) @@ -1010,105 +1010,6 @@ class DeltaLogSuite extends QueryTest } } - private def createDeltaTableWithCommits(tableDir: File, numVersions: Int): DeltaLog = { - val log = DeltaLog.forTable(spark, tableDir.getCanonicalPath) - spark.sql(s"CREATE TABLE delta.`${tableDir.getCanonicalPath}` (id INT) USING delta") - (1 to numVersions).foreach { i => - spark.sql(s"INSERT INTO delta.`${tableDir.getCanonicalPath}` VALUES ($i)") - } - log - } - - test("getChangesIterator yields one SingleCommit per commit with the right version") { - withTempDir { tableDir => - val log = createDeltaTableWithCommits(tableDir, numVersions = 3) - // CREATE is version 0, the three INSERTs are versions 1..3. - val commits = log.getChangesIterator(startVersion = 0).toList - assert(commits.map(_.version) === (0L to 3L).toList) - - // The handle exposes the commit file's modification time (not the in-commit timestamp), so it - // matches what getChangeLogFiles reports for the same commit. - val fileModTimes = - log.getChangeLogFiles(startVersion = 0).map(_._2.getModificationTime).toList - assert(commits.map(_.fileModificationTimestamp) === fileModTimes) - } - } - - test("getChangesIterator endVersion override bounds the range (both inclusive)") { - withTempDir { tableDir => - val log = createDeltaTableWithCommits(tableDir, numVersions = 5) - val commits = log.getChangesIterator( - startVersion = 1, endVersion = 3, catalogTableOpt = None, failOnDataLoss = true).toList - assert(commits.map(_.version) === List(1L, 2L, 3L)) - } - } - - test("SingleCommit.getActionsIterator returns the commit's actions and matches getChanges") { - withTempDir { tableDir => - val log = createDeltaTableWithCommits(tableDir, numVersions = 3) - val viaHandle = log.getChangesIterator(startVersion = 0).map { commit => - (commit.version, commit.getActionsIterator().processAndClose(_.toList)) - }.toList - val viaGetChanges = log.getChanges(startVersion = 0).map { - case (version, actions) => (version, actions.toList) - }.toList - assert(viaHandle === viaGetChanges) - } - } - - test("SingleCommit.getActionsIterator is rewindable and replays the same actions") { - withTempDir { tableDir => - val log = createDeltaTableWithCommits(tableDir, numVersions = 1) - // Version 1 is the single INSERT; open its actions and read twice via rewind(). - val commit = log.getChangesIterator(startVersion = 1).next() - val iter = commit.getActionsIterator() - try { - val firstPass = iter.toList - assert(firstPass.nonEmpty) - iter.rewind() - val secondPass = iter.toList - assert(secondPass === firstPass) - } finally { - iter.close() - } - } - } - - test("SingleCommit.getActionsIterator returns all actions of a multi-action commit") { - withTempDir { dir => - val log = DeltaLog.forTable(spark, new Path(dir.getCanonicalPath)) - // Commit 0 creates the table and adds "old". Commit 1 packs several data actions -- three new - // AddFiles and a RemoveFile of "old" -- into one commit so the commit has more than one - // action. - log.startTransaction().commitManually( - Metadata(configuration = Map(DeltaConfigs.CHECKPOINT_INTERVAL.key -> "10")), - createTestAddFile(encodedPath = "old")) - val dataActions: Seq[Action] = Seq( - createTestAddFile(encodedPath = "a"), - createTestAddFile(encodedPath = "b"), - createTestAddFile(encodedPath = "c"), - RemoveFile("old", Some(System.currentTimeMillis()), dataChange = true)) - log.startTransaction().commitManually(dataActions: _*) - - // Read commit 1 (the multi-action one) back through the new API. - val commit = log.getChangesIterator(startVersion = 1).next() - assert(commit.version === 1L) - val actions = commit.getActionsIterator().processAndClose(_.toList) - - // It contains more than one action, and every data action we committed is present (order and - // exact identity of synthesized CommitInfo/Protocol are not asserted -- commit() adds those). - assert(actions.size > 1, s"expected a multi-action commit, got: $actions") - val addedPaths = actions.collect { case a: AddFile => a.path }.toSet - assert(addedPaths === Set("a", "b", "c"), s"missing AddFiles, got: $addedPaths") - val removedPaths = actions.collect { case r: RemoveFile => r.path }.toSet - assert(removedPaths === Set("old"), s"missing RemoveFile, got: $removedPaths") - - // The authoritative invariant: getActionsIterator returns exactly what getChanges returns for - // this same multi-action commit. - val viaGetChanges = log.getChanges(startVersion = 1).next()._2.toList - assert(actions === viaGetChanges) - } - } } class DeltaLogWithCatalogOwnedBatch1Suite extends DeltaLogSuite { diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaNotSupportedDDLSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaNotSupportedDDLSuite.scala index 08ee847e4f9..d2912861560 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaNotSupportedDDLSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaNotSupportedDDLSuite.scala @@ -35,9 +35,10 @@ class DeltaNotSupportedDDLSuite abstract class DeltaNotSupportedDDLBase extends QueryTest - with DeltaSQLTestUtils { + with DeltaSQLTestUtils + with DeltaTableProvider { - val format = "delta" + val format = tableProvider val nonPartitionedTableName = "deltaTbl" diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaOptionSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaOptionSuite.scala index 0e629582a99..d8fafc1378a 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaOptionSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaOptionSuite.scala @@ -254,7 +254,7 @@ class DeltaOptionSuite extends QueryTest s"partitionOverwriteMode is set to invalid value in options invalidMode=$invalidMode") { withSQLConf(DeltaSQLConf.DYNAMIC_PARTITION_OVERWRITE_ENABLED.key -> "true") { withTempDir { tempDir => - val e = intercept[IllegalArgumentException] { + val e = intercept[DeltaIllegalArgumentException] { Seq(1, 2, 3).toDF .withColumn("part", $"value" % 2) .write @@ -263,11 +263,14 @@ class DeltaOptionSuite extends QueryTest .option("partitionOverwriteMode", invalidMode) .save(tempDir.getAbsolutePath) } - assert(e.getMessage === - DeltaErrors.illegalDeltaOptionException( - PARTITION_OVERWRITE_MODE_OPTION, invalidMode, "must be 'STATIC' or 'DYNAMIC'" - ).getMessage - ) + checkError( + e, + "DELTA_ILLEGAL_OPTION.MUST_BE_ONE_OF", + parameters = Map( + "input" -> invalidMode, + "name" -> PARTITION_OVERWRITE_MODE_OPTION, + "validValues" -> + DeltaOptions.PARTITION_OVERWRITE_MODE_VALUES.map("'" + _ + "'").mkString(", "))) } } } @@ -326,12 +329,11 @@ class DeltaOptionSuite extends QueryTest .option(USE_NULL_INTOLERANT_EQUALITY_WITH_DPO, "true") .saveAsTable("temp") }, - condition = "DELTA_ILLEGAL_OPTION", + condition = "DELTA_ILLEGAL_OPTION.DYNAMIC_PARTITION_OVERWRITE_ONLY", sqlState = Some("42616"), parameters = Map( "name" -> USE_NULL_INTOLERANT_EQUALITY_WITH_DPO, - "input" -> "true", - "explain" -> "This option should be specified only in Dynamic Partition Overwrite mode.") + "input" -> "true") ) } } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaRetentionSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaRetentionSuite.scala index 9d1b70307c1..d17579eddc8 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaRetentionSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaRetentionSuite.scala @@ -21,8 +21,10 @@ import java.io.File import scala.concurrent.duration._ import scala.language.postfixOps +import com.databricks.spark.util.Log4jUsageLogger import org.apache.spark.sql.delta.DeltaTestUtils.createTestAddFile import org.apache.spark.sql.delta.actions.{Action, AddFile, RemoveFile, SetTransaction} +import org.apache.spark.sql.delta.coordinatedcommits.CatalogManagedMaintenanceIncompatible import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaSQLTestUtils @@ -33,6 +35,7 @@ import org.apache.hadoop.fs.{FileStatus, FileSystem, Path, RawLocalFileSystem} import org.apache.spark.SparkConf import org.apache.spark.sql.QueryTest +import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.util.ManualClock // scalastyle:off: removeFile @@ -56,7 +59,45 @@ class DeltaRetentionSuite extends QueryTest } } - test("delete expired logs") { + test("direct metadata cleanup is blocked for catalog-managed tables") { + withCatalogManagedTable() { tableName => + val log = DeltaLog.forTable(spark, TableIdentifier(tableName)) + val logPath = new File(log.logPath.toUri) + val filesBeforeCleanup = getLogFiles(logPath).map(_.getCanonicalPath).toSet + + checkError( + intercept[DeltaUnsupportedOperationException] { + log.cleanUpExpiredLogs(log.update()) + }, + "DELTA_UNSUPPORTED_CATALOG_MANAGED_TABLE_OPERATION", + parameters = Map("operation" -> "METADATA_CLEANUP")) + assert(getLogFiles(logPath).map(_.getCanonicalPath).toSet === filesBeforeCleanup) + } + } + + test("checkpoint does not clean up catalog-managed table metadata") { + withCatalogManagedTable() { tableName => + spark.sql(s"ALTER TABLE $tableName SET TBLPROPERTIES " + + "('delta.enableExpiredLogCleanup' = 'true')") + spark.sql(s"INSERT INTO $tableName VALUES (1)") + val log = DeltaLog.forTable(spark, TableIdentifier(tableName)) + val snapshot = log.update() + val firstCommit = FileNames.unsafeDeltaFile(log.logPath, 0) + val fs = firstCommit.getFileSystem(log.newDeltaHadoopConf()) + assert(fs.exists(firstCommit)) + fs.setTimes(firstCommit, 0L, -1L) + + val usageRecords = Log4jUsageLogger.track { + log.checkpoint(snapshot) + } + + assert(log.readLastCheckpointFile().exists(_.version == snapshot.version)) + assert(DeltaTestUtils.filterUsageRecords(usageRecords, "delta.log.cleanup").isEmpty) + assert(fs.exists(firstCommit)) + } + } + + test("delete expired logs", CatalogManagedMaintenanceIncompatible) { withTempDir { tempDir => val startTime = getStartTimeForRetentionTest val clock = new ManualClock(startTime) @@ -107,7 +148,9 @@ class DeltaRetentionSuite extends QueryTest } } - test("log files being already deleted shouldn't fail log deletion job") { + test( + "log files being already deleted shouldn't fail log deletion job", + CatalogManagedMaintenanceIncompatible) { withTempDir { tempDir => val startTime = getStartTimeForRetentionTest val clock = new ManualClock(startTime) @@ -283,7 +326,9 @@ class DeltaRetentionSuite extends QueryTest } } - test("the checkpoint and checksum for version 0 should be cleaned") { + test( + "the checkpoint and checksum for version 0 should be cleaned", + CatalogManagedMaintenanceIncompatible) { withTempDir { tempDir => val clock = new ManualClock(getStartTimeForRetentionTest) val log = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath), clock) @@ -373,7 +418,9 @@ class DeltaRetentionSuite extends QueryTest } for (v2CheckpointFormat <- V2Checkpoint.Format.ALL_AS_STRINGS) - test(s"sidecar file cleanup [v2CheckpointFormat: $v2CheckpointFormat]") { + test( + s"sidecar file cleanup [v2CheckpointFormat: $v2CheckpointFormat]", + CatalogManagedMaintenanceIncompatible) { val checkpointPolicy = CheckpointPolicy.V2.name withSQLConf((DeltaSQLConf.CHECKPOINT_V2_TOP_LEVEL_FILE_FORMAT.key -> v2CheckpointFormat)) { withTempDir { tempDir => @@ -484,7 +531,8 @@ class DeltaRetentionSuite extends QueryTest for (v2CheckpointFormat <- V2Checkpoint.Format.ALL_AS_STRINGS) test( s"compat file created with metadata cleanup when checkpoints are deleted" + - s" [v2CheckpointFormat: $v2CheckpointFormat]") { + s" [v2CheckpointFormat: $v2CheckpointFormat]", + CatalogManagedMaintenanceIncompatible) { val checkpointPolicy = CheckpointPolicy.V2.name withSQLConf((DeltaSQLConf.CHECKPOINT_V2_TOP_LEVEL_FILE_FORMAT.key -> v2CheckpointFormat)) { withTempDir { tempDir => @@ -556,7 +604,7 @@ class DeltaRetentionSuite extends QueryTest } }.flatten).foreach { case (chkConfigName, chkConfig) => test(s"cleanup does not delete the checkpoint if it is required by non-expired versions. " + - s"Config: $chkConfigName.") { + s"Config: $chkConfigName.", CatalogManagedMaintenanceIncompatible) { withSQLConf(chkConfig: _*) { // Disable the following check as the test relies on time travel beyond // deletedFileRetentionDuration @@ -693,7 +741,9 @@ class DeltaRetentionSuite extends QueryTest } } - test(s"cleanup does not delete the JSON logs if the multi-part checkpoint is incomplete.") { + test( + s"cleanup does not delete the JSON logs if the multi-part checkpoint is incomplete.", + CatalogManagedMaintenanceIncompatible) { withSQLConf(DeltaSQLConf.DELTA_CHECKPOINT_PART_SIZE.key -> "1") { withTempDir { tempDir => val startTime = getStartTimeForRetentionTest @@ -776,7 +826,9 @@ class DeltaRetentionSuite extends QueryTest } } - test("Metadata cleanup respects requireCheckpointProtectionBeforeVersion") { + test( + "Metadata cleanup respects requireCheckpointProtectionBeforeVersion", + CatalogManagedMaintenanceIncompatible) { withSQLConf( DeltaSQLConf.ALLOW_METADATA_CLEANUP_WHEN_ALL_PROTOCOLS_SUPPORTED.key -> "false", DeltaSQLConf.ALLOW_METADATA_CLEANUP_CHECKPOINT_EXISTENCE_CHECK_DISABLED.key -> "true") { @@ -916,7 +968,9 @@ class DeltaRetentionSuite extends QueryTest } } - test("Cleanup is allowed if a checkpoint already exists at the boundary") { + test( + "Cleanup is allowed if a checkpoint already exists at the boundary", + CatalogManagedMaintenanceIncompatible) { withSQLConf(DeltaSQLConf.ALLOW_METADATA_CLEANUP_WHEN_ALL_PROTOCOLS_SUPPORTED.key -> "false") { testRequireCheckpointProtectionBeforeVersion( createNumCommitsOutsideRetentionPeriod = 8, @@ -930,7 +984,9 @@ class DeltaRetentionSuite extends QueryTest } } - test("Metadata cleanup protocol validation positive tests.") { + test( + "Metadata cleanup protocol validation positive tests.", + CatalogManagedMaintenanceIncompatible) { withSQLConf( DeltaSQLConf.ALLOW_METADATA_CLEANUP_CHECKPOINT_EXISTENCE_CHECK_DISABLED.key -> "true") { // In all tests below, we cannot satisfy the version requirement and thus fallback @@ -996,7 +1052,9 @@ class DeltaRetentionSuite extends QueryTest } } - test("Metadata cleanup protocol validation negative tests.") { + test( + "Metadata cleanup protocol validation negative tests.", + CatalogManagedMaintenanceIncompatible) { withSQLConf( DeltaSQLConf.ALLOW_METADATA_CLEANUP_CHECKPOINT_EXISTENCE_CHECK_DISABLED.key -> "true") { // In all tests below, we cannot satisfy the version requirement and thus fallback @@ -1071,7 +1129,9 @@ class DeltaRetentionSuite extends QueryTest } } - test("Metadata cleanup protocol validation with incomplete CRCs.") { + test( + "Metadata cleanup protocol validation with incomplete CRCs.", + CatalogManagedMaintenanceIncompatible) { withSQLConf( DeltaSQLConf.ALLOW_METADATA_CLEANUP_CHECKPOINT_EXISTENCE_CHECK_DISABLED.key -> "true") { // We fall back to protocol validations which cannot be completed due to missing @@ -1134,7 +1194,9 @@ class DeltaRetentionWithCatalogOwnedBatch2Suite * * Note: This test is too slow for batchSize = 100 and wouldn't necessarily work for batchSize = 1 */ - test("unbackfilled expired commits are always retained") { + test( + "unbackfilled expired commits are always retained", + CatalogManagedMaintenanceIncompatible) { withTempDir { tempDir => val startTime = getStartTimeForRetentionTest val clock = new ManualClock(startTime) @@ -1184,4 +1246,3 @@ class DeltaRetentionWithCatalogOwnedBatch2Suite } } } - diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSinkSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSinkSuite.scala index cc3565a256a..a9312fe53f0 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSinkSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSinkSuite.scala @@ -27,8 +27,10 @@ import org.apache.spark.sql.delta.test.{DeltaColumnMappingSelectedTestMixin, Del import org.apache.spark.sql.delta.test.DeltaSQLTestUtils import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.test.shims.StreamingTestShims.{MemoryStream, MicroBatchExecution, StreamingQueryWrapper} +import org.apache.spark.sql.delta.test.shims.VariantShreddingTestShims import io.delta.tables.{DeltaTable => IODeltaTable} import org.apache.commons.io.FileUtils +import org.apache.parquet.hadoop.ParquetFileReader import org.scalatest.time.SpanSugar._ import org.apache.spark.SparkConf @@ -45,8 +47,9 @@ import org.apache.spark.sql.types._ import org.apache.spark.util.Utils abstract class DeltaSinkTest - extends StreamTest - with DeltaSQLCommandTest { + extends QueryTest + with DeltaSQLCommandTest + with StreamTest { override val streamingTimeout = 60.seconds import testImplicits._ @@ -429,6 +432,146 @@ class DeltaSinkSuite } } + /** Asserts each of `partitionValues` is a physical `col-=` dir, not logical. */ + private def assertPhysicalColumnMappedPartitionDirs( + tableRoot: File, partitionValues: Seq[String]): Unit = { + val dirNames = Option(tableRoot.listFiles()).getOrElse(Array.empty[File]) + .filter(_.isDirectory).map(_.getName) + partitionValues.foreach { value => + assert( + dirNames.count(n => n.startsWith("col-") && n.endsWith("=" + value)) == 1, + s"expected one physical partition dir col-*=$value under $tableRoot, " + + s"got: ${dirNames.mkString(", ")}") + assert( + !dirNames.exists(n => !n.startsWith("col-") && n.endsWith("=" + value)), + s"did not expect a logical partition dir ending in '=$value' under $tableRoot") + } + } + + test("partitioned writing into a column-mapped table") { + // Runs on V1 here and, via DeltaV2SinkSuite's classification, on the V2 Kernel sink. + Seq("name", "id").foreach { mode => + withClue(s"columnMappingMode=$mode") { + withColumnMappingConf(mode) { + withSinkTarget { (target, checkpointDir) => + val inputData = MemoryStream[Int] + val ds = inputData.toDS() + val query = startStream( + ds.map(i => (i, i * 1000)) + .toDF("id", "value") + .writeStream + .partitionBy("id") + .option("checkpointLocation", checkpointDir.getCanonicalPath) + .format("delta"), + target) + try { + inputData.addData(1, 2, 3) + failAfter(streamingTimeout) { + query.processAllAvailable() + } + + // Data round-trips through a batch read. + checkDatasetUnorderly( + readTarget(target).as[(Int, Int)], + (1, 1000), (2, 2000), (3, 3000)) + + // Guard that the auto-created table is actually column-mapped. + assert(deltaLogForTarget(target).update().metadata.columnMappingMode.name == mode) + + // Partition pruning returns the right rows. + checkDatasetUnorderly( + readTarget(target).filter("id = 1").as[(Int, Int)], (1, 1000)) + checkDatasetUnorderly( + readTarget(target).filter("id in (1, 2)").as[(Int, Int)], (1, 1000), (2, 2000)) + + val tableRoot = new File(deltaLogForTarget(target).dataPath.toUri) + // TODO(#7140): Implement randomized file prefixes in the V2 sink and remove + // this V1/V2 physical-layout distinction. + if (useDsv2) { + assertPhysicalColumnMappedPartitionDirs(tableRoot, Seq("1", "2", "3")) + } else { + val dataDirs = Option(tableRoot.listFiles()).getOrElse(Array.empty[File]) + .filter(_.isDirectory).map(_.getName).filterNot(_ == "_delta_log") + assert( + dataDirs.nonEmpty && !dataDirs.exists(_.contains("=")), + s"expected V1's random-prefix layout (data dirs, none of the form col=value), " + + s"got: ${dataDirs.mkString(", ")}") + } + } finally { + query.stop() + } + } + } + } + } + } + + test("streaming write shreds variant columns only when the table enables it") { + // Runs on V1 here and, via DeltaV2SinkSuite's classification, on the V2 Kernel sink. Where this + // Spark version cannot infer a shredding schema nothing shreds, whatever the property says. + val shreddingSupported = VariantShreddingTestShims.variantInferShreddingSchemaSupported + withSQLConf("spark.sql.variant.writeShredding.enabled" -> "true") { + Seq(true, false).foreach { shreddingEnabled => + withClue(s"${DeltaConfigs.ENABLE_VARIANT_SHREDDING.key}=$shreddingEnabled") { + withSinkTarget { (target, checkpointDir) => + createPreexistingTableAtSinkTarget( + target, + Seq( + IODeltaTable.columnBuilder(spark, "id").dataType("int").build(), + IODeltaTable.columnBuilder(spark, "v").dataType("variant").build()), + Map(DeltaConfigs.ENABLE_VARIANT_SHREDDING.key -> shreddingEnabled.toString)) + val inputData = MemoryStream[(Int, String)] + val query = startStream( + inputData.toDF() + .toDF("id", "json") + .selectExpr("id", "parse_json(json) AS v") + .writeStream + .option("checkpointLocation", checkpointDir.getCanonicalPath) + .format("delta"), + target) + try { + inputData.addData((1, """{"a":1,"b":"xy"}""")) + failAfter(streamingTimeout) { + query.processAllAvailable() + } + } finally { + query.stop() + } + + assert( + targetHasShreddedVariant(target) == (shreddingEnabled && shreddingSupported), + "the shredded layout must follow the table property") + checkAnswer( + readTarget(target).selectExpr( + "id", "variant_get(v, '$.a', 'int')", "variant_get(v, '$.b', 'string')"), + Row(1, 1, "xy")) + } + } + } + } + } + + /** Whether any data file in `target`'s current snapshot stores variant column `v` shredded. */ + private def targetHasShreddedVariant(target: String): Boolean = { + val deltaLog = deltaLogForTarget(target) + val snapshot = deltaLog.update() + // Parquet footers carry physical column names, which differ from `v` under column mapping. + val physicalName = DeltaColumnMapping.getPhysicalName(snapshot.schema("v")) + val files = snapshot.allFiles.collect().map(_.absolutePath(deltaLog).toString) + assert(files.nonEmpty, s"expected at least one data file in $target") + files.exists { file => + val reader = + ParquetFileReader.open(deltaLog.newDeltaHadoopConf(), new org.apache.hadoop.fs.Path(file)) + try { + val schema = reader.getFooter.getFileMetaData.getSchema + schema.getType(schema.getFieldIndex(physicalName)).asGroupType() + .containsField("typed_value") + } finally { + reader.close() + } + } + } + test("work with aggregation + watermark") { withSinkTarget { (target, checkpointDir) => val inputData = MemoryStream[Long] @@ -783,11 +926,13 @@ class DeltaSinkSuite */ private def createPreexistingTableAtSinkTarget( target: String, - columns: Seq[StructField] + columns: Seq[StructField], + properties: Map[String, String] = Map.empty ): Unit = { val builder = IODeltaTable.create(spark) if (useNameBasedAccess) builder.tableName(target) else builder.location(target) columns.foreach(builder.addColumn) + properties.foreach { case (key, value) => builder.property(key, value) } builder.execute() } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceColumnMappingSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceColumnMappingSuite.scala index ee637023b7c..f04ad2c18d9 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceColumnMappingSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceColumnMappingSuite.scala @@ -27,6 +27,7 @@ import org.apache.spark.sql.delta.Relocated.StreamExecution import org.apache.spark.sql.delta.coordinatedcommits.CatalogOwnedTestBaseSuite import org.apache.spark.sql.delta.sources.{DeltaSource, DeltaSQLConf} import org.apache.spark.sql.delta.test.DeltaColumnMappingSelectedTestMixin +import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.util.JsonUtils import org.apache.commons.io.FileUtils @@ -104,17 +105,10 @@ trait ColumnMappingStreamingTestUtils extends StreamTest with DeltaColumnMapping } trait ColumnMappingStreamingBlockedWorkflowSuiteBase extends ColumnMappingStreamingTestUtils { - self: DeltaSourceConnectorTrait => + self: DeltaSourceConnectorTrait with DeltaSQLCommandTest => import testImplicits._ - /** - * Executes a DML SQL statement (DELETE, INSERT, etc.). - * Overridable so that V2 suites can route DML through the V1 connector, - * since SparkTable (V2) is read-only and does not support writes. - */ - protected def executeDml(sqlText: String): Unit = sql(sqlText) - // Start a streaming read on the configured connector (V1 vs V2). MAX_FILES_PER_TRIGGER=1 // ensures we catch failures ASAP; CDC_READ_OPTION is layered in for CDC variants. private def loadStream( @@ -752,10 +746,6 @@ class DeltaSourceIdColumnMappingSuite extends DeltaSourceSuite with DeltaSourceColumnMappingSuiteBase { override protected def isCdcTest: Boolean = false - - // Disambiguates the `executeDml` inherited from both DeltaSourceSuite and - // ColumnMappingStreamingBlockedWorkflowSuiteBase (Scala requires an explicit override). - override protected def executeDml(sqlText: String): Unit = sql(sqlText) } class DeltaSourceNameColumnMappingSuite extends DeltaSourceSuite @@ -764,10 +754,6 @@ class DeltaSourceNameColumnMappingSuite extends DeltaSourceSuite with DeltaSourceColumnMappingSuiteBase { override protected def isCdcTest: Boolean = false - - // Disambiguates the `executeDml` inherited from both DeltaSourceSuite and - // ColumnMappingStreamingBlockedWorkflowSuiteBase (Scala requires an explicit override). - override protected def executeDml(sqlText: String): Unit = sql(sqlText) } // Batch sizes 1, 2, and 100 exercise different backfill behaviors in the commit coordinator. diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceDeletionVectorsSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceDeletionVectorsSuite.scala index 7210e4d9e13..eea686465d3 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceDeletionVectorsSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceDeletionVectorsSuite.scala @@ -26,22 +26,18 @@ import org.apache.hadoop.fs.Path import org.scalatest.concurrent.Eventually import org.scalatest.concurrent.PatienceConfiguration.Timeout +import org.apache.spark.sql.QueryTest import org.apache.spark.sql.streaming.{StreamTest, Trigger} import org.apache.spark.sql.streaming.util.StreamManualClock -trait DeltaSourceDeletionVectorTests extends StreamTest - with DeletionVectorsTestUtils { - self: DeltaSourceConnectorTrait => +trait DeltaSourceDeletionVectorTests + extends QueryTest + with DeltaSQLCommandTest + with DeletionVectorsTestUtils + with StreamTest { self: DeltaSourceConnectorTrait => import testImplicits._ - /** - * Executes a DML SQL statement (DELETE, INSERT, etc.). - * Overridable so that V2 suites can route DML through the V1 connector, - * since DeltaV2Table (V2) is read-only and does not support writes. - */ - protected def executeDml(sqlText: String): Unit = sql(sqlText) - test("allow to delete files before starting a streaming query") { withTempDir { inputDir => val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceFastDropFeatureSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceFastDropFeatureSuite.scala index 8c74e68d3fc..5e6cdc7e381 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceFastDropFeatureSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceFastDropFeatureSuite.scala @@ -42,8 +42,6 @@ class DeltaSourceFastDropFeatureSuite spark.conf.set(DeltaSQLConf.FAST_DROP_FEATURE_ENABLED.key, "true") } - protected def executeDml(sqlText: String): Unit = sql(sqlText) - protected def dropUnsupportedFeature(dir: File): Unit = executeDml( s"""ALTER TABLE delta.`${dir.getCanonicalPath}` diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceRowTrackingSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceRowTrackingSuite.scala index dac5388de62..59351f7281a 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceRowTrackingSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceRowTrackingSuite.scala @@ -20,6 +20,7 @@ import java.util.Locale import org.apache.spark.sql.delta.test.DeltaSQLCommandTest +import org.apache.spark.sql.QueryTest import org.apache.spark.sql.streaming.StreamTest /** @@ -28,9 +29,12 @@ import org.apache.spark.sql.streaming.StreamTest * The base suite runs through the DSv1 connector; DeltaV2SourceRowTrackingSuite * re-runs the same tests through the DSv2 connector via V2ForceTest. */ -trait DeltaSourceRowTrackingSuiteBase extends StreamTest +trait DeltaSourceRowTrackingSuiteBase + extends QueryTest + with DeltaSQLCommandTest with DeltaSourceSuiteBase - with DeltaColumnMappingTestUtils { + with DeltaColumnMappingTestUtils + with StreamTest { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceSchemaEvolutionSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceSchemaEvolutionSuite.scala index 0bf5a1bf45d..99053fe8a1f 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceSchemaEvolutionSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceSchemaEvolutionSuite.scala @@ -33,7 +33,7 @@ import org.apache.hadoop.fs.Path import org.apache.logging.log4j.Level import org.apache.spark.SparkConf -import org.apache.spark.sql.{DataFrame, Row} +import org.apache.spark.sql.{DataFrame, QueryTest, Row} import org.apache.spark.sql.catalyst.{InternalRow, TableIdentifier} import org.apache.spark.sql.execution.streaming.Offset import org.apache.spark.sql.functions.lit @@ -41,8 +41,11 @@ import org.apache.spark.sql.streaming.{StreamingQueryException, Trigger} import org.apache.spark.sql.types.{StringType, StructType} import org.apache.spark.util.Utils -trait StreamingSchemaEvolutionSuiteBase extends ColumnMappingStreamingTestUtils - with DeltaSourceSuiteBase with DeltaColumnMappingSelectedTestMixin with DeltaSQLCommandTest { +trait StreamingSchemaEvolutionSuiteBase + extends QueryTest + with DeltaColumnMappingSelectedTestMixin + with DeltaSourceSuiteBase + with ColumnMappingStreamingTestUtils { override protected def runOnlyTests: Seq[String] = Seq( "schema log initialization with additive schema changes", @@ -251,12 +254,6 @@ trait StreamingSchemaEvolutionSuiteBase extends ColumnMappingStreamingTestUtils protected def getDefaultSchemaLocation(implicit log: DeltaLog): Path = new Path(getDefaultCheckpoint, "_schema_location") - /** - * Executes a DDL/DML SQL statement. Overridable so that V2 suites can route it through the V1 - * connector, since DeltaV2Table (V2) is read-only and does not support writes/DDL. - */ - protected def executeDml(sqlText: String): Unit = sql(sqlText) - protected def addColumn(column: String, dt: String = "STRING")(implicit log: DeltaLog): Unit = { executeDml(s"ALTER TABLE delta.`${log.dataPath}` ADD COLUMN ($column $dt)") } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceSuite.scala index c63ac87207a..f49d54abc49 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceSuite.scala @@ -16,7 +16,7 @@ package org.apache.spark.sql.delta -import java.io.{File, FileInputStream, OutputStream, PrintWriter, StringWriter} +import java.io.{File, FileInputStream, FileNotFoundException, OutputStream, PrintWriter, StringWriter} import java.net.URI import java.sql.Timestamp import java.util.UUID @@ -29,6 +29,7 @@ import org.apache.spark.sql.delta.DataFrameUtils import org.apache.spark.sql.delta.DeltaTestUtils.{modifyCommitTimestamp, modifyCommitTimestamps} import org.apache.spark.sql.delta.Relocated import org.apache.spark.sql.delta.actions.{AddFile, Protocol} +import org.apache.spark.sql.delta.coordinatedcommits.CatalogManagedMaintenanceIncompatible import org.apache.spark.sql.delta.sources.{DeltaDataSource, DeltaSQLConf, DeltaSource, DeltaSourceOffset} import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaTestImplicits._ @@ -39,7 +40,7 @@ import org.apache.commons.lang3.exception.ExceptionUtils import org.apache.hadoop.fs.{FileStatus, Path, RawLocalFileSystem} import org.scalatest.time.{Seconds, Span} -import org.apache.spark.{SparkConf, SparkThrowable} +import org.apache.spark.{SparkConf, SparkException, SparkThrowable} import org.apache.spark.sql.{AnalysisException, DataFrame, Dataset, Row} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.util.IntervalUtils @@ -454,10 +455,17 @@ class DeltaSourceSuite extends DeltaSourceSuiteBase .start() .processAllAvailable() } - assert(e.getCause.isInstanceOf[IllegalArgumentException]) - for (msg <- Seq("Invalid", DeltaOptions.MAX_FILES_PER_TRIGGER_OPTION, "positive")) { - assert(e.getCause.getMessage.contains(msg)) + val cause = e.getCause.asInstanceOf[DeltaIllegalArgumentException] + val condition = invalidMaxFilesPerTrigger match { + case _: Int => "DELTA_ILLEGAL_OPTION.MUST_BE_POSITIVE_NUMBER" + case _ => "DELTA_ILLEGAL_OPTION.MUST_BE_INTEGER" } + checkError( + cause, + condition, + parameters = Map( + "input" -> invalidMaxFilesPerTrigger.toString, + "name" -> DeltaOptions.MAX_FILES_PER_TRIGGER_OPTION)) } } } @@ -1086,7 +1094,8 @@ class DeltaSourceSuite extends DeltaSourceSuiteBase } test( - "can delete old files of a snapshot without update" + "can delete old files of a snapshot without update", + CatalogManagedMaintenanceIncompatible ) { withTempDir { inputDir => val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) @@ -1363,13 +1372,6 @@ class DeltaSourceSuite extends DeltaSourceSuiteBase .select($"ts".cast("string")).as[String].head() } - /** - * Executes a DML SQL statement (DELETE, INSERT, etc.). - * Overridable so that V2 suites can route DML through the V1 connector, - * since SparkTable (V2) is read-only and does not support writes. - */ - protected def executeDml(sqlText: String): Unit = sql(sqlText) - /** Disable log cleanup to avoid deleting logs we are testing. */ protected def disableLogCleanup(tablePath: String): Unit = { executeDml(s"alter table delta.`$tablePath` " + @@ -2280,15 +2282,18 @@ class DeltaSourceSuite extends DeltaSourceSuiteBase assert(e.getMessage.contains( "unexpectedly still requires additional file-system listing")) } else { - // getBatch reads versions 0-1 (version 2 is gone), so maxVersionSeen = 1. - // lastExpectedVersion = 2 in both cases: - // mid-version: endOffset=(v2, index=0) -> lastExpectedVersion = 2 - // fully consumed: endOffset=(v3, BASE_INDEX) -> lastExpectedVersion = 3 - 1 = 2 - val e = intercept[DeltaIllegalStateException] { + // With the parallel CommitInfo read, getBatch's `update()` call eagerly reads the last + // commit's file during snapshot construction to source the AMT manifest reference. + // Because v2's commit file was deleted, that read fails as a SparkException wrapping a + // FileNotFoundException before the streaming's DELTA_STREAMING_TRAILING_COMMIT_MISSING + // check fires. + val e = intercept[SparkException] { source.getBatch(startOffsetOption = None, endOffset) } - checkError(e, "DELTA_STREAMING_TRAILING_COMMIT_MISSING", "42K03", - Map("expectedVersion" -> "2", "seenVersion" -> "1")) + assert( + Iterator.iterate[Throwable](e)(_.getCause).takeWhile(_ != null) + .exists(_.isInstanceOf[FileNotFoundException]), + s"expected a FileNotFoundException in the cause chain, got: $e") } } } @@ -2692,7 +2697,7 @@ class DeltaSourceSuite extends DeltaSourceSuiteBase Seq(1, 2, 3).toDF().write.delta(inputDir.toString) - val df = spark.readStream.delta(inputDir.toString) + val df = loadStreamWithOptions(inputDir.getCanonicalPath, Map.empty) val stream = df.writeStream .option("checkpointLocation", checkpointDir.toString) @@ -3374,20 +3379,19 @@ class DeltaSourceSuite extends DeltaSourceSuiteBase withTable("srcTable") { withTempDirs { (srcTblDir, checkpointDir, checkpointDir2) => def readStream(startingVersion: Option[Long] = None): DataFrame = { - var dsr = spark.readStream - startingVersion.foreach { v => - dsr = dsr.option("startingVersion", v) - } - dsr.table("srcTable") + val options = startingVersion + .map(version => Map("startingVersion" -> version.toString)) + .getOrElse(Map.empty) + loadStreamWithOptions(srcTblDir.getCanonicalPath, options) } - sql(s""" + executeDml(s""" |CREATE TABLE srcTable ( | a STRING NOT NULL, | b STRING NOT NULL |) USING DELTA LOCATION '${srcTblDir.getCanonicalPath}' |""".stripMargin) - sql(""" + executeDml(""" |INSERT INTO srcTable | VALUES ("a", "b") |""".stripMargin) @@ -3409,12 +3413,12 @@ class DeltaSourceSuite extends DeltaSourceSuiteBase // Write more data and drop NOT NULL constraint Execute { _ => // A batch of Delta actions - sql(""" + executeDml(""" |INSERT INTO srcTable |VALUES ("c", "d") |""".stripMargin) - sql("ALTER TABLE srcTable ALTER COLUMN a DROP NOT NULL") - sql(""" + executeDml("ALTER TABLE srcTable ALTER COLUMN a DROP NOT NULL") + executeDml(""" |INSERT INTO srcTable |VALUES ("e", "f") |""".stripMargin) @@ -3445,7 +3449,7 @@ class DeltaSourceSuite extends DeltaSourceSuiteBase txn.commit(txn.metadata.copy(schemaString = newSchema.json) :: Nil, DeltaOperations.ManualUpdate) } - sql(""" + executeDml(""" |INSERT INTO srcTable |VALUES ("g", "h") |""".stripMargin) diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceSuiteBase.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceSuiteBase.scala index e2e34ff611e..4fe77d19fa6 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceSuiteBase.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceSuiteBase.scala @@ -22,8 +22,9 @@ import org.apache.spark.sql.delta.actions.Format import org.apache.spark.sql.delta.coordinatedcommits.CatalogOwnedTestBaseSuite import org.apache.spark.sql.delta.schema.{SchemaMergingUtils, SchemaUtils} import org.apache.spark.sql.delta.test.DeltaSQLTestUtils +import org.apache.spark.sql.delta.test.DeltaSQLCommandTest -import org.apache.spark.sql.{Column, DataFrame} +import org.apache.spark.sql.{Column, DataFrame, QueryTest} import org.apache.spark.sql.streaming.StreamTest import org.apache.spark.sql.types.StructType import org.scalactic.source.Position @@ -49,10 +50,13 @@ trait DeltaSourceConnectorTrait { } } -trait DeltaSourceSuiteBase extends StreamTest +trait DeltaSourceSuiteBase + extends QueryTest + with DeltaSQLCommandTest with DeltaSQLTestUtils with CatalogOwnedTestBaseSuite - with DeltaSourceConnectorTrait { + with DeltaSourceConnectorTrait + with StreamTest { /** * Creates 3 temporary directories for use within a function. diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceTableAPISuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceTableAPISuite.scala index 14d6e7412cd..f1da294f9d9 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceTableAPISuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceTableAPISuite.scala @@ -24,29 +24,32 @@ import org.apache.spark.sql.delta.coordinatedcommits.CatalogOwnedTestBaseSuite import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.shims.StreamingTestShims.MemoryStream -import org.apache.spark.sql.{AnalysisException, Dataset} +import org.apache.spark.sql.{AnalysisException, Dataset, QueryTest} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.catalog.SessionCatalog.DEFAULT_DATABASE import org.apache.spark.sql.execution.streaming._ import org.apache.spark.sql.streaming.{StreamingQuery, StreamTest} import org.apache.spark.util.Utils -class DeltaSourceTableAPISuite extends StreamTest +class DeltaSourceTableAPISuite + extends QueryTest with DeltaSQLCommandTest - with CatalogOwnedTestBaseSuite { + with CatalogOwnedTestBaseSuite + with StreamTest { - override def beforeAll(): Unit = { - super.beforeAll() + import testImplicits._ - } + /** Runs batch writes that prepare data for a streaming source test through V1. */ + protected def withV1Mode(f: => Unit): Unit = f - import testImplicits._ test("table API") { withTempDir { tempDir => val tblName = "my_table" val dir = tempDir.getAbsolutePath withTable(tblName) { - spark.range(3).write.format("delta").option("path", dir).saveAsTable(tblName) + withV1Mode { + spark.range(3).write.format("delta").option("path", dir).saveAsTable(tblName) + } testStream(spark.readStream.table(tblName))( ProcessAllAvailable(), @@ -63,7 +66,9 @@ class DeltaSourceTableAPISuite extends StreamTest withTempDatabase { db => withTable(tblName) { spark.sql(s"USE $db") - spark.range(3).write.format("delta").option("path", dir).saveAsTable(tblName) + withV1Mode { + spark.range(3).write.format("delta").option("path", dir).saveAsTable(tblName) + } spark.sql(s"USE $DEFAULT_DATABASE") testStream(spark.readStream.table(s"$db.$tblName"))( diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSuite.scala index f85352e02cb..5ed5451ef62 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSuite.scala @@ -25,7 +25,7 @@ import org.apache.spark.sql.delta.DeltaTestUtils.BOOLEAN_DOMAIN import org.apache.spark.sql.delta.actions.{Action, TableFeatureProtocolUtils} import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.commands.cdc.CDCReader -import org.apache.spark.sql.delta.coordinatedcommits.{CatalogOwnedTableUtils, CatalogOwnedTestBaseSuite} +import org.apache.spark.sql.delta.coordinatedcommits.{CatalogManagedMaintenanceIncompatible, CatalogOwnedTableUtils, CatalogOwnedTestBaseSuite} import org.apache.spark.sql.delta.files.TahoeLogFileIndex import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest @@ -265,8 +265,26 @@ class DeltaSuite extends QueryTest .mode("overwrite") .option(DeltaOptions.REPLACE_WHERE_OPTION, "is_odd = true") .save(tempDir.toString) - }.getMessage - assert(e1.contains("does not conform to partial table overwrite condition or constraint")) + } + if (enabled) { + checkError( + exception = e1, + condition = "DELTA_REPLACE_WHERE_MISMATCH.INVARIANT_VIOLATION", + sqlState = Some("44000"), + parameters = Map( + "replaceWhere" -> "is_odd = true", + "invariantViolationMessage" -> "(?s).*"), + matchPVals = true) + } else { + checkError( + exception = e1, + condition = "DELTA_REPLACE_WHERE_MISMATCH.INVALID_PARTITIONS", + sqlState = Some("44000"), + parameters = Map( + "replaceWhere" -> "is_odd = true", + "badPartitions" -> "(?s).*=false"), + matchPVals = true) + } val e2 = intercept[AnalysisException] { Seq(true).toDF("is_odd") @@ -307,14 +325,20 @@ class DeltaSuite extends QueryTest .mode("overwrite") .option(DeltaOptions.REPLACE_WHERE_OPTION, "value = 1") .save(tempDir.toString) - }.getMessage + } if (enabled) { - assert(e4.contains( - "Written data does not conform to partial table overwrite condition " + - "or constraint 'value = 1'")) + checkError( + exception = e4, + condition = "DELTA_REPLACE_WHERE_MISMATCH.INVARIANT_VIOLATION", + sqlState = Some("44000"), + parameters = Map( + "replaceWhere" -> "value = 1", + "invariantViolationMessage" -> "(?s).*"), + matchPVals = true) } else { - assert(e4.contains("Predicate references non-partition column 'value'. Only the " + - "partition columns may be referenced: [is_odd]")) + assert(e4.getMessage.contains( + "Predicate references non-partition column 'value'. Only the " + + "partition columns may be referenced: [is_odd]")) } val e5 = intercept[AnalysisException] { @@ -331,7 +355,7 @@ class DeltaSuite extends QueryTest } } - test("replaceWhere with rearrangeOnly") { + test("replaceWhere with rearrangeOnly", CatalogManagedMaintenanceIncompatible) { withTempDir { dir => Seq(1, 2, 3, 4).toDF() .withColumn("is_odd", $"value" % 2 =!= 0) @@ -368,6 +392,32 @@ class DeltaSuite extends QueryTest } } + test("rearrange-only write is blocked for a catalog-managed table") { + withCatalogManagedTable() { tableName => + spark.sql(s"INSERT INTO $tableName VALUES (1)") + val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) + val snapshotBefore = deltaLog.update() + + checkError( + intercept[DeltaUnsupportedOperationException] { + Seq(2).toDF("id") + .write + .format("delta") + .mode("overwrite") + .option(DeltaOptions.REPLACE_WHERE_OPTION, "id = 1") + .option(DeltaOptions.DATA_CHANGE_OPTION, "false") + .save(deltaLog.dataPath.toString) + }, + "DELTA_UNSUPPORTED_CATALOG_MANAGED_TABLE_OPERATION", + parameters = Map("operation" -> "DATA_REORGANIZATION")) + + val snapshotAfter = deltaLog.update() + assert(snapshotAfter.version === snapshotBefore.version) + assert(snapshotAfter.allFiles.collect().toSet === snapshotBefore.allFiles.collect().toSet) + checkAnswer(spark.table(tableName), Row(1)) + } + } + test("valid replaceWhere") { Seq(true, false).foreach { enabled => withSQLConf(DeltaSQLConf.REPLACEWHERE_DATACOLUMNS_ENABLED.key -> enabled.toString) { @@ -2224,8 +2274,14 @@ class DeltaSuite extends QueryTest .mode("overwrite") .saveAsTable(table) } - assert(e.getMessage.startsWith("[DELTA_REPLACE_WHERE_MISMATCH] " + - "Written data does not conform to partial table overwrite condition or constraint")) + checkError( + exception = e, + condition = "DELTA_REPLACE_WHERE_MISMATCH.INVARIANT_VIOLATION", + sqlState = Some("44000"), + parameters = Map( + "replaceWhere" -> "a\\.b = 'a' AND `a\\.b` = 'a'", + "invariantViolationMessage" -> "(?s).*"), + matchPVals = true) Seq(("a", "b", "c"), ("d", "e", "f")) .toDF("a.b", "c.d", "ab") @@ -2795,10 +2851,8 @@ class DeltaSuite extends QueryTest val e1 = intercept[DeltaIllegalArgumentException] { spark.sql(s"INSERT INTO $tableName (col1, col2) VALUES (4, 0)") } - checkError(e1, "DELTA_INVALID_IDEMPOTENT_WRITES_OPTIONS", "42616", Map("reason" -> ( - "Both spark.databricks.delta.write.txnAppId and spark.databricks.delta.write.txnVersion " + - "must be specified for idempotent Delta writes") - )) + checkError(e1, "DELTA_INVALID_IDEMPOTENT_WRITES_OPTIONS.MISSING_SESSION_CONFS", "42616", + Map.empty[String, String]) // this write should succeed as it's using a newer version than the latest spark.conf.set("spark.databricks.delta.write.txnVersion", "10") spark.sql(s"INSERT INTO $tableName (col1, col2) VALUES (2, 0)") @@ -2806,10 +2860,8 @@ class DeltaSuite extends QueryTest val e2 = intercept[DeltaIllegalArgumentException] { spark.sql(s"INSERT INTO $tableName (col1, col2) VALUES (3, 0)") } - checkError(e2, "DELTA_INVALID_IDEMPOTENT_WRITES_OPTIONS", "42616", Map("reason" -> ( - "Both spark.databricks.delta.write.txnAppId and spark.databricks.delta.write.txnVersion " + - "must be specified for idempotent Delta writes") - )) + checkError(e2, "DELTA_INVALID_IDEMPOTENT_WRITES_OPTIONS.MISSING_SESSION_CONFS", "42616", + Map.empty[String, String]) val res = spark.sql(s"SELECT col1 FROM $tableName") .orderBy(asc("col1")) @@ -2881,7 +2933,7 @@ class DeltaSuite extends QueryTest idempotentWrite(mode, appId2, df, path, name, 5, 12, 4, isSaveAsTable) // Verify that specifying only one of the options -- either appId or version -- fails. - val e1 = intercept[Exception] { + val e1 = intercept[DeltaIllegalArgumentException] { val stage = df.write.format("delta").option(DeltaOptions.TXN_APP_ID, 1).mode(mode) if (isSaveAsTable) { stage.option("path", path).saveAsTable(name) @@ -2889,8 +2941,10 @@ class DeltaSuite extends QueryTest stage.save(path) } } - assert(e1.getMessage.contains("Invalid options for idempotent Dataframe writes")) - val e2 = intercept[Exception] { + checkError(e1, + "DELTA_INVALID_IDEMPOTENT_WRITES_OPTIONS.MISSING_DATAFRAME_WRITE_OPTIONS", "42616", + Map.empty[String, String]) + val e2 = intercept[DeltaIllegalArgumentException] { val stage = df.write.format("delta").option(DeltaOptions.TXN_VERSION, 1).mode(mode) if (isSaveAsTable) { stage.option("path", path).saveAsTable(name) @@ -2898,7 +2952,9 @@ class DeltaSuite extends QueryTest stage.save(path) } } - assert(e2.getMessage.contains("Invalid options for idempotent Dataframe writes")) + checkError(e2, + "DELTA_INVALID_IDEMPOTENT_WRITES_OPTIONS.MISSING_DATAFRAME_WRITE_OPTIONS", "42616", + Map.empty[String, String]) } } } @@ -2927,16 +2983,20 @@ class DeltaSuite extends QueryTest idempotentWrite(mode, appId2, df, path, name, 5, 3, 4, isSaveAsTable) // Verify that specifying only one of the options -- either appId or version -- fails. - val e1 = intercept[Exception] { + val e1 = intercept[DeltaIllegalArgumentException] { val stage = df.write.format("delta").option(DeltaOptions.TXN_APP_ID, 1).mode(mode) if (isSaveAsTable) stage.option("path", path).saveAsTable(name) else stage.save(path) } - assert(e1.getMessage.contains("Invalid options for idempotent Dataframe writes")) - val e2 = intercept[Exception] { + checkError(e1, + "DELTA_INVALID_IDEMPOTENT_WRITES_OPTIONS.MISSING_DATAFRAME_WRITE_OPTIONS", "42616", + Map.empty[String, String]) + val e2 = intercept[DeltaIllegalArgumentException] { val stage = df.write.format("delta").option(DeltaOptions.TXN_VERSION, 1).mode(mode) if (isSaveAsTable) stage.option("path", path).saveAsTable(name) else stage.save(path) } - assert(e2.getMessage.contains("Invalid options for idempotent Dataframe writes")) + checkError(e2, + "DELTA_INVALID_IDEMPOTENT_WRITES_OPTIONS.MISSING_DATAFRAME_WRITE_OPTIONS", "42616", + Map.empty[String, String]) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaTableCreationTests.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaTableCreationTests.scala index ea0611997c7..aff1c06c564 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaTableCreationTests.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaTableCreationTests.scala @@ -28,6 +28,7 @@ import org.apache.spark.sql.delta.DeltaOperations.ManualUpdate import org.apache.spark.sql.delta.actions.Metadata import org.apache.spark.sql.delta.actions.Protocol import org.apache.spark.sql.delta.actions.TableFeatureProtocolUtils +import org.apache.spark.sql.delta.commands.CreateDeltaTableCommand import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.{DeltaColumnMappingSelectedTestMixin, DeltaSQLCommandTest} import org.apache.spark.sql.delta.test.DeltaTestImplicits._ @@ -45,7 +46,7 @@ import org.apache.spark.sql.connector.catalog.{CatalogV2Util, Identifier, Table, import org.apache.spark.sql.functions.col import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SharedSparkSession -import org.apache.spark.sql.types.{MetadataBuilder, StructType} +import org.apache.spark.sql.types.{ArrayType, DataType, IntegerType, MapType, MetadataBuilder, StringType, StructField, StructType} import org.apache.spark.util.Utils trait DeltaTableCreationTests @@ -2262,16 +2263,18 @@ class DeltaTableCreationSuite } test("Replace a table without comment") { - withTempDir { dir => - val table = "replace_table_without_comment" - val location = dir.getAbsolutePath - withTable(table) { - sql(s"CREATE TABLE $table (col string) USING delta COMMENT 'Table' LOCATION '$location'") - sql(s"REPLACE TABLE $table (col string) USING delta LOCATION '$location'") - checkResult( - sql(s"DESCRIBE DETAIL $table"), - Seq("delta", null), - Seq("format", "description")) + withSQLConf(DeltaSQLConf.RETAIN_COMMENTS_DURING_REPLACE_TABLE.key -> "false") { + withTempDir { dir => + val table = "replace_table_without_comment" + val location = dir.getAbsolutePath + withTable(table) { + sql(s"CREATE TABLE $table (col string) USING delta COMMENT 'Table' LOCATION '$location'") + sql(s"REPLACE TABLE $table (col string) USING delta LOCATION '$location'") + checkResult( + sql(s"DESCRIBE DETAIL $table"), + Seq("delta", null), + Seq("format", "description")) + } } } } @@ -2325,19 +2328,21 @@ class DeltaTableCreationSuite } test("Replace CTAS a table without comment") { - val table = "replace_ctas_without_comment" - withTable(table) { - sql( - s"""CREATE TABLE $table - |USING delta - |COMMENT 'This table is created with existing data' - |AS SELECT * FROM range(10) - """.stripMargin) - sql(s"REPLACE TABLE $table USING delta AS SELECT * FROM range(10)") - checkResult( - sql(s"DESCRIBE DETAIL $table"), - Seq("delta", null), - Seq("format", "description")) + withSQLConf(DeltaSQLConf.RETAIN_COMMENTS_DURING_REPLACE_TABLE.key -> "false") { + val table = "replace_ctas_without_comment" + withTable(table) { + sql( + s"""CREATE TABLE $table + |USING delta + |COMMENT 'This table is created with existing data' + |AS SELECT * FROM range(10) + """.stripMargin) + sql(s"REPLACE TABLE $table USING delta AS SELECT * FROM range(10)") + checkResult( + sql(s"DESCRIBE DETAIL $table"), + Seq("delta", null), + Seq("format", "description")) + } } } @@ -2358,6 +2363,464 @@ class DeltaTableCreationSuite } } + test("Replace table retains column comments when not re-specified") { + withTempDir { dir => + val table = "retain_col_comment" + val location = dir.getAbsolutePath + withTable(table) { + sql(s"""CREATE TABLE $table (a STRING COMMENT 'column a', b INT COMMENT 'column b') + |USING delta LOCATION '$location'""".stripMargin) + sql(s"REPLACE TABLE $table (a STRING, b INT) USING delta LOCATION '$location'") + val catalogTable = spark.sessionState.catalog.getTableMetadata(TableIdentifier(table)) + val deltaSchema = DeltaLog.forTable(spark, catalogTable).update().schema + Seq( + deltaSchema + ).foreach { schema => + assert(schema("a").getComment() === Some("column a")) + assert(schema("b").getComment() === Some("column b")) + } + } + } + } + + test("Replace table allows explicit column comment override") { + withTempDir { dir => + val table = "override_col_comment" + val location = dir.getAbsolutePath + withTable(table) { + sql(s"""CREATE TABLE $table (a STRING COMMENT 'original') + |USING delta LOCATION '$location'""".stripMargin) + sql(s"""REPLACE TABLE $table (a STRING COMMENT 'new comment') + |USING delta LOCATION '$location'""".stripMargin) + val catalogTable = spark.sessionState.catalog.getTableMetadata(TableIdentifier(table)) + val deltaSchema = DeltaLog.forTable(spark, catalogTable).update().schema + Seq( + deltaSchema + ).foreach { schema => + assert(schema("a").getComment() === Some("new comment")) + } + } + } + } + + test("Replace table allows explicitly clearing column comment") { + withTempDir { dir => + val table = "clear_col_comment" + val location = dir.getAbsolutePath + withTable(table) { + sql(s"""CREATE TABLE $table (a STRING COMMENT 'original') + |USING delta LOCATION '$location'""".stripMargin) + sql(s"""REPLACE TABLE $table (a STRING COMMENT '') + |USING delta LOCATION '$location'""".stripMargin) + val catalogTable = spark.sessionState.catalog.getTableMetadata(TableIdentifier(table)) + val deltaSchema = DeltaLog.forTable(spark, catalogTable).update().schema + Seq( + deltaSchema + ).foreach { schema => + assert(schema("a").getComment() === Some("")) + } + } + } + } + + test("Replace table retains comments only for matching columns") { + withTempDir { dir => + val table = "retain_matching_col_comments" + val location = dir.getAbsolutePath + withTable(table) { + sql(s"""CREATE TABLE $table (a STRING COMMENT 'ca', b INT COMMENT 'cb') + |USING delta LOCATION '$location'""".stripMargin) + sql(s"REPLACE TABLE $table (a STRING, c DOUBLE) USING delta LOCATION '$location'") + val catalogTable = spark.sessionState.catalog.getTableMetadata(TableIdentifier(table)) + val deltaSchema = DeltaLog.forTable(spark, catalogTable).update().schema + Seq( + deltaSchema + ).foreach { schema => + assert(schema("a").getComment() === Some("ca")) + assert(schema("c").getComment() === None) + } + } + } + } + + test("Replace table retains a matching column's comment even when its data type changes") { + withTempDir { dir => + val table = "retain_comment_changed_type" + val location = dir.getAbsolutePath + withTable(table) { + sql(s"""CREATE TABLE $table (a STRING COMMENT 'ca') + |USING delta LOCATION '$location'""".stripMargin) + sql(s"REPLACE TABLE $table (a INT) USING delta LOCATION '$location'") + val catalogTable = spark.sessionState.catalog.getTableMetadata(TableIdentifier(table)) + val deltaSchema = DeltaLog.forTable(spark, catalogTable).update().schema + Seq( + deltaSchema + ).foreach { schema => + assert(schema("a").dataType === IntegerType) + assert(schema("a").getComment() === Some("ca")) + } + } + } + } + + test("Replace CTAS retains column comments") { + val table = "rtas_retain_col_comments" + withTable(table) { + sql(s"CREATE TABLE $table USING delta AS SELECT * FROM range(10)") + sql(s"ALTER TABLE $table ALTER COLUMN id COMMENT 'my identifier'") + assert(spark.table(table).schema("id").getComment() === Some("my identifier")) + sql(s"REPLACE TABLE $table USING delta AS SELECT * FROM range(5)") + val catalogTable = spark.sessionState.catalog.getTableMetadata(TableIdentifier(table)) + val deltaSchema = DeltaLog.forTable(spark, catalogTable).update().schema + Seq( + deltaSchema + ).foreach { schema => + assert(schema("id").getComment() === Some("my identifier")) + } + } + } + + test("Replace table retains nested struct column comments") { + withTempDir { dir => + val table = "retain_nested_comments" + val location = dir.getAbsolutePath + withTable(table) { + sql(s"""CREATE TABLE $table + |(info STRUCT) + |USING delta LOCATION '$location'""".stripMargin) + sql(s"""REPLACE TABLE $table + |(info STRUCT) + |USING delta LOCATION '$location'""".stripMargin) + val catalogTable = spark.sessionState.catalog.getTableMetadata(TableIdentifier(table)) + val deltaSchema = DeltaLog.forTable(spark, catalogTable).update().schema + Seq( + deltaSchema + ).foreach { schema => + val innerSchema = schema("info").dataType.asInstanceOf[StructType] + assert(innerSchema("x").getComment() === Some("nested x")) + assert(innerSchema("y").getComment() === Some("nested y")) + } + } + } + } + + test("Replace table retains table comment when not re-specified") { + withTempDir { dir => + val table = "retain_table_comment" + val location = dir.getAbsolutePath + withTable(table) { + sql(s"""CREATE TABLE $table (col STRING) + |USING delta COMMENT 'my table comment' LOCATION '$location'""".stripMargin) + sql(s"REPLACE TABLE $table (col STRING) USING delta LOCATION '$location'") + val catalogTable = spark.sessionState.catalog.getTableMetadata(TableIdentifier(table)) + val deltaLog = DeltaLog.forTable(spark, catalogTable) + // Retained in the Delta log metadata and catalog table definition. + assert(deltaLog.update().metadata.description === "my table comment") + assert(catalogTable.comment === Some("my table comment")) + } + } + } + + test("Replace table allows explicit table comment override") { + withTempDir { dir => + val table = "override_table_comment" + val location = dir.getAbsolutePath + withTable(table) { + sql(s"""CREATE TABLE $table (col STRING) + |USING delta COMMENT 'original' LOCATION '$location'""".stripMargin) + sql(s"""REPLACE TABLE $table (col STRING) + |USING delta COMMENT 'new comment' LOCATION '$location'""".stripMargin) + val catalogTable = spark.sessionState.catalog.getTableMetadata(TableIdentifier(table)) + val deltaLog = DeltaLog.forTable(spark, catalogTable) + assert(deltaLog.update().metadata.description === "new comment") + assert(catalogTable.comment === Some("new comment")) + } + } + } + + test("Replace table via DataFrameWriterV2 retains comments") { + withTempDir { dir => + val table = "retain_comments_df_v2" + val location = dir.getAbsolutePath + withTable(table) { + sql(s"""CREATE TABLE $table (a STRING COMMENT 'my col a', b INT COMMENT 'my col b') + |USING delta COMMENT 'my table comment' LOCATION '$location'""".stripMargin) + spark.range(1).selectExpr("cast(id as string) as a", "cast(id as int) as b") + .writeTo(table) + .using("delta") + .replace() + val catalogTable = spark.sessionState.catalog.getTableMetadata(TableIdentifier(table)) + val deltaLog = DeltaLog.forTable(spark, catalogTable) + val deltaSchema = deltaLog.update().schema + Seq( + deltaSchema + ).foreach { schema => + assert(schema("a").getComment() === Some("my col a")) + assert(schema("b").getComment() === Some("my col b")) + } + assert(deltaLog.update().metadata.description === "my table comment") + assert(catalogTable.comment === Some("my table comment")) + } + } + } + + test("Replace table retains comments in array of structs") { + withTempDir { dir => + val table = "retain_array_struct_comments" + val location = dir.getAbsolutePath + withTable(table) { + sql(s"""CREATE TABLE $table + |(items ARRAY>) + |USING delta LOCATION '$location'""".stripMargin) + sql(s"""REPLACE TABLE $table + |(items ARRAY>) + |USING delta LOCATION '$location'""".stripMargin) + val catalogTable = spark.sessionState.catalog.getTableMetadata(TableIdentifier(table)) + val deltaSchema = DeltaLog.forTable(spark, catalogTable).update().schema + Seq( + deltaSchema + ).foreach { schema => + val elemType = schema("items").dataType + .asInstanceOf[ArrayType].elementType.asInstanceOf[StructType] + assert(elemType("x").getComment() === Some("my arr x")) + assert(elemType("y").getComment() === Some("my arr y")) + } + } + } + } + + test("Replace table retains comments in map value structs") { + withTempDir { dir => + val table = "retain_map_struct_comments" + val location = dir.getAbsolutePath + withTable(table) { + sql(s"""CREATE TABLE $table + |(lookup MAP>) + |USING delta LOCATION '$location'""".stripMargin) + sql(s"""REPLACE TABLE $table + |(lookup MAP>) + |USING delta LOCATION '$location'""".stripMargin) + val catalogTable = spark.sessionState.catalog.getTableMetadata(TableIdentifier(table)) + val deltaSchema = DeltaLog.forTable(spark, catalogTable).update().schema + Seq( + deltaSchema + ).foreach { schema => + val valType = schema("lookup").dataType + .asInstanceOf[MapType].valueType.asInstanceOf[StructType] + assert(valType("a").getComment() === Some("my map a")) + assert(valType("b").getComment() === Some("my map b")) + } + } + } + } + + test("Replace table retains comments in map key structs") { + withTempDir { dir => + val table = "retain_map_key_struct_comments" + val location = dir.getAbsolutePath + withTable(table) { + sql(s"""CREATE TABLE $table + |(lookup MAP, INT>) + |USING delta LOCATION '$location'""".stripMargin) + sql(s"""REPLACE TABLE $table + |(lookup MAP, INT>) + |USING delta LOCATION '$location'""".stripMargin) + val catalogTable = spark.sessionState.catalog.getTableMetadata(TableIdentifier(table)) + val deltaSchema = DeltaLog.forTable(spark, catalogTable).update().schema + Seq( + deltaSchema + ).foreach { schema => + val keyType = schema("lookup").dataType + .asInstanceOf[MapType].keyType.asInstanceOf[StructType] + assert(keyType("k1").getComment() === Some("my key1")) + assert(keyType("k2").getComment() === Some("my key2")) + } + } + } + } + + test("Replace table retains comments in deeply nested types") { + withTempDir { dir => + val table = "retain_deep_nested_comments" + val location = dir.getAbsolutePath + withTable(table) { + // MAP>> + sql(s"""CREATE TABLE $table + |(col MAP>>) + |USING delta LOCATION '$location'""".stripMargin) + sql(s"""REPLACE TABLE $table + |(col MAP>>) + |USING delta LOCATION '$location'""".stripMargin) + val catalogTable = spark.sessionState.catalog.getTableMetadata(TableIdentifier(table)) + val deltaSchema = DeltaLog.forTable(spark, catalogTable).update().schema + Seq( + deltaSchema + ).foreach { schema => + val innerStruct = schema("col").dataType + .asInstanceOf[MapType].valueType + .asInstanceOf[ArrayType].elementType.asInstanceOf[StructType] + assert(innerStruct("x").getComment() === Some("my deep")) + } + } + } + } + + // Direct unit tests for the pure comment carry-over helpers. These complement the end-to-end + // REPLACE tests above by exercising the schema transform in isolation, including edge cases that + // are awkward or impossible to express through SQL DDL (type-shape mismatches, explicit empty + // comments, non-comment metadata preservation, and asymmetric nested schemas). + private def field( + name: String, + dataType: DataType, + comment: Option[String] = None): StructField = { + val base = StructField(name, dataType) + comment.map(base.withComment).getOrElse(base) + } + + private def carryOver( + oldSchema: StructType, + newSchema: StructType): StructType = { + CreateDeltaTableCommand.carryOverStructTypeComments(oldSchema, newSchema) + } + + test("carryOverStructTypeComments: fills missing comments, keeps explicit ones") { + val oldSchema = StructType(Seq( + field("a", StringType, Some("old a")), + field("b", IntegerType, Some("old b")), + field("c", StringType, Some("old c")))) + val newSchema = StructType(Seq( + field("a", StringType), // no comment -> carried over + field("b", IntegerType, Some("new b")), // explicit comment -> kept + field("c", StringType, Some("")), // explicit empty comment -> kept (wins over carry-over) + field("d", StringType))) // no match in old -> stays uncommented + val result = carryOver(oldSchema, newSchema) + assert(result("a").getComment() === Some("old a")) + assert(result("b").getComment() === Some("new b")) + assert(result("c").getComment() === Some("")) + assert(result("d").getComment() === None) + } + + test("carryOverStructTypeComments: matches column names case-insensitively") { + val oldSchema = StructType(Seq(field("Name", StringType, Some("old name")))) + val newSchema = StructType(Seq(field("name", StringType))) + val result = carryOver(oldSchema, newSchema) + assert(result("name").getComment() === Some("old name")) + } + + test("carryOverStructTypeComments: preserves non-comment field metadata") { + val extraMetadata = new MetadataBuilder().putString("custom", "keep-me").build() + val oldSchema = StructType(Seq(field("a", StringType, Some("old a")))) + val newSchema = StructType(Seq(StructField("a", StringType, metadata = extraMetadata))) + val result = carryOver(oldSchema, newSchema) + assert(result("a").getComment() === Some("old a")) + assert(result("a").metadata.getString("custom") === "keep-me") + } + + test("carryOverStructTypeComments: nested struct, array, and map recursion") { + val oldSchema = StructType(Seq( + field("s", StructType(Seq(field("x", StringType, Some("old x")))), Some("old s")), + field("arr", + ArrayType(StructType(Seq(field("e", StringType, Some("old e"))))), Some("old a")), + field("m", + MapType( + StructType(Seq(field("k", StringType, Some("old k")))), + StructType(Seq(field("v", StringType, Some("old v"))))), + Some("old m")))) + val newSchema = StructType(Seq( + field("s", StructType(Seq(field("x", StringType)))), + field("arr", ArrayType(StructType(Seq(field("e", StringType))))), + field("m", + MapType( + StructType(Seq(field("k", StringType))), + StructType(Seq(field("v", StringType))))))) + val result = carryOver(oldSchema, newSchema) + assert(result("s").getComment() === Some("old s")) + assert(result("s").dataType.asInstanceOf[StructType]("x").getComment() === Some("old x")) + val arrElem = + result("arr").dataType.asInstanceOf[ArrayType].elementType.asInstanceOf[StructType] + assert(arrElem("e").getComment() === Some("old e")) + val mapType = result("m").dataType.asInstanceOf[MapType] + assert(mapType.keyType.asInstanceOf[StructType]("k").getComment() === Some("old k")) + assert(mapType.valueType.asInstanceOf[StructType]("v").getComment() === Some("old v")) + } + + test("carryOverStructTypeComments: type-shape mismatch is a no-op, not a crash") { + // Old field is a struct, new field is a scalar (and vice versa): recursion must fall through + // and leave the new type untouched rather than throwing. + val oldSchema = StructType(Seq( + field("a", StructType(Seq(field("x", StringType, Some("old x")))), Some("old a")), + field("b", ArrayType(StringType), Some("old b")))) + val newSchema = StructType(Seq( + field("a", StringType), // struct -> scalar + field("b", MapType(StringType, StringType)))) // array -> map + val result = carryOver(oldSchema, newSchema) + // Top-level comments are still carried over by name; only the nested recursion no-ops. + assert(result("a").getComment() === Some("old a")) + assert(result("a").dataType === StringType) + assert(result("b").getComment() === Some("old b")) + assert(result("b").dataType === MapType(StringType, StringType)) + } + + test("carryOverStructTypeComments: preserves nullability from the new schema") { + // containsNull / valueContainsNull must be taken from the NEW type, not the old. + val oldSchema = StructType(Seq( + field("arr", ArrayType(StringType, containsNull = true)), + field("m", MapType(StringType, StringType, valueContainsNull = true)))) + val newSchema = StructType(Seq( + field("arr", ArrayType(StringType, containsNull = false)), + field("m", MapType(StringType, StringType, valueContainsNull = false)))) + val result = carryOver(oldSchema, newSchema) + assert(!result("arr").dataType.asInstanceOf[ArrayType].containsNull) + assert(!result("m").dataType.asInstanceOf[MapType].valueContainsNull) + } + + test("Replace table allows explicitly clearing table comment") { + withTempDir { dir => + val table = "clear_table_comment" + val location = dir.getAbsolutePath + withTable(table) { + sql(s"""CREATE TABLE $table (col STRING) + |USING delta COMMENT 'original' LOCATION '$location'""".stripMargin) + sql(s"""REPLACE TABLE $table (col STRING) + |USING delta COMMENT '' LOCATION '$location'""".stripMargin) + // An explicit empty comment must win over carry-over: the description is set to the empty + // string rather than the retained old comment. + assert(getDeltaLog(new Path(location)).update().metadata.description === "") + } + } + } + + test("Replace via DataFrameWriterV2 with overwriteSchema=true drops column comments but " + + "retains the table comment") { + // Scoped to NoMapping: in column-mapping modes the reused column-mapping metadata already + // makes the data schema differ from txn.metadata.schema regardless of comments, so the + // overwriteSchema path diverges for reasons unrelated to this carry-over. Skip the body + // (rather than cancel) under the column-mapping suite variants. + if (columnMappingMode == NoMapping.name) { + withTempDir { dir => + val table = "overwrite_schema_drops_col_comments" + val location = dir.getAbsolutePath + withTable(table) { + sql(s"""CREATE TABLE $table (a STRING COMMENT 'my col a', b INT COMMENT 'my col b') + |USING delta COMMENT 'my table comment' LOCATION '$location'""".stripMargin) + spark.range(1).selectExpr("cast(id as string) as a", "cast(id as int) as b") + .writeTo(table) + .using("delta") + .option("overwriteSchema", "true") + .replace() + val schema = spark.table(table).schema + assert(schema("a").getComment() === None) + assert(schema("b").getComment() === None) + checkResult( + sql(s"DESCRIBE DETAIL $table"), + Seq("delta", "my table comment"), + Seq("format", "description")) + } + } + } + } + /** * Verifies that the correct table properties are stored in the transaction log as well as the * catalog. diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaTableFeatureSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaTableFeatureSuite.scala index 4d819a46634..f84bc11b153 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaTableFeatureSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaTableFeatureSuite.scala @@ -474,6 +474,34 @@ class DeltaTableFeatureSuite } } + test("fileType-preview is a registered, removable, and addable feature") { + // Registered and recognized by name. + val feature = TableFeature.featureNameToFeature(FileTypePreviewTableFeature.name) + assert(feature.contains(FileTypePreviewTableFeature)) + // Removable, so it can be dropped once a table no longer contains `file` columns. + assert(feature.exists(_.isInstanceOf[RemovableFeature])) + // There is no `file` type in the schema yet, so no action references the feature. + assert(!FileTypePreviewTableFeature.actionUsesFeature(Metadata())) + + // The feature can be added to a table and then shows up in the protocol. + val table = "tbl" + withTable(table) { + spark.range(0).write.format("delta").saveAsTable(table) + val log = DeltaLog.forTable(spark, TableIdentifier(table)) + assert(!log.update().protocol.readerAndWriterFeatureNames.contains( + FileTypePreviewTableFeature.name)) + + sql(buildTablePropertyModifyingCommand( + "ALTER", + targetTableName = table, + sourceTableName = table, + Seq(s"'$FEATURE_PROP_PREFIX${FileTypePreviewTableFeature.name}' = 'supported', " + + s"'delta.minWriterVersion' = $TABLE_FEATURES_MIN_WRITER_VERSION"))) + assert(log.update().protocol.readerAndWriterFeatureNames.contains( + FileTypePreviewTableFeature.name)) + } + } + test("drop table feature works with coordinated commits") { val table = "tbl" withTable(table) { @@ -515,7 +543,7 @@ class DeltaTableFeatureSuite val e = intercept[DeltaTableFeatureException] { sql(s"ALTER TABLE $table DROP FEATURE $featureName") } - assert(e.getMessage.contains("DELTA_FEATURE_DROP_HISTORICAL_VERSIONS_EXIST"), e) + assert(e.getCondition == "DELTA_FEATURE_DROP_HISTORICAL_VERSIONS_EXIST", e) // Add in a checkpoint and cleanUp up older logs containing feature traces log.startTransaction().commitManually() diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaTableProvider.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaTableProvider.scala new file mode 100644 index 00000000000..64f0c3b1a7c --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaTableProvider.scala @@ -0,0 +1,69 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta + +/** + * Indirection over the table format used by a test suite so that a single suite body can run + * against different table formats. + * + * A suite mixes this in (directly or transitively) and uses [[tableProvider]] / [[writeFormat]] + * instead of hard-coded `"delta"` literals in `USING ` clauses and + * `df.write.format(...)` calls. A variant of the suite can override these to target another + * format. Because the defaults here are `"delta"`, mixing this into an existing Delta suite is + * behavior-preserving. + */ +trait DeltaTableProvider { + /** Provider name for `USING ` in CREATE TABLE statements. Defaults to Delta. */ + protected def tableProvider: String = "delta" + + /** + * Format name for `df.write.format(...)` / `spark.read.format(...)`. Defaults to + * [[tableProvider]]. + */ + protected def writeFormat: String = tableProvider + + /** + * Table properties applied to every table created via [[createTableSQL]]. Empty by default; a + * variant can override this to add format-specific properties. + */ + protected def defaultTableProperties: Map[String, String] = Map.empty + + /** + * Build a `CREATE TABLE` statement for the table format under test. Merges + * [[defaultTableProperties]] with any per-call `props` (the latter win on key conflicts). + * + * @param partitionBy passed through verbatim when non-empty, so it must already include its + * keyword, e.g. `"PARTITIONED BY (p)"`. + * @param clusterBy passed through verbatim when non-empty, so it must already include its + * keyword, e.g. `"CLUSTER BY (c)"`. + * @param location a bare path (no keyword); it is wrapped as `LOCATION ''`. + */ + protected def createTableSQL( + name: String, + schema: String, + partitionBy: String = "", + clusterBy: String = "", + location: String = "", + props: Map[String, String] = Map.empty): String = { + val allProps = defaultTableProperties ++ props + val propStr = + if (allProps.isEmpty) "" + else s"TBLPROPERTIES (${allProps.map { case (k, v) => s"'$k'='$v'" }.mkString(",")})" + val locStr = if (location.isEmpty) "" else s"LOCATION '$location'" + s"CREATE TABLE $name ($schema) USING $tableProvider $partitionBy $clusterBy $locStr $propStr" + } +} diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaTestUtils.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaTestUtils.scala index ea623588625..adff3ead518 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaTestUtils.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaTestUtils.scala @@ -30,7 +30,9 @@ import scala.util.matching.Regex import com.databricks.spark.util.{Log4jUsageLogger, UsageRecord} import org.apache.spark.sql.delta.DeltaTestUtils.Plans import org.apache.spark.sql.delta.actions._ +import org.apache.spark.sql.delta.amt.AMTUtils import org.apache.spark.sql.delta.commands.cdc.CDCReader +import org.apache.spark.sql.delta.coordinatedcommits.{CatalogOwnedTableUtils, CoordinatedCommitsUtils} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.{DeltaSQLCommandTest, DeltaSQLTestUtils} import org.apache.spark.sql.delta.test.DeltaTestImplicits._ @@ -66,9 +68,36 @@ object DeltaTestUtilsBase { * Used to gate NullType tests so they run on Spark 4.1+ and are skipped on Spark 4.0. */ def nullTypeColumnsSupported: Boolean = !org.apache.spark.SPARK_VERSION.startsWith("4.0") + + /** + * Collects `t` and every throwable in its cause chain (cycle-guarded) and asserts that at + * least one of them satisfies `f`. + */ + def assertThrowableInCauseChain(t: Throwable)(f: Throwable => Boolean): Unit = { + val throwables = scala.collection.mutable.ArrayBuffer.empty[Throwable] + var current: Throwable = t + while (current != null && !throwables.contains(current)) { + throwables += current + current = current.getCause + } + assert(throwables.exists(f), + s"No throwable in the cause chain matched the predicate. Chain: " + + throwables.map(_.getClass.getName).mkString("[", ", ", "]")) + } + + /** + * Asserts that at least one throwable in `t`'s cause chain is a `SparkThrowable` whose error + * condition equals `condition`. + */ + def assertThrowableWithConditionInCauseChain(t: Throwable, condition: String): Unit = { + assertThrowableInCauseChain(t) { + case st: org.apache.spark.SparkThrowable => st.getCondition == condition + case _ => false + } + } } -trait CDCTestMixin extends SharedSparkSession { +trait CDCTestMixin { // Setting the spark Conf is left to the test implementation. def computeCDC( @@ -81,14 +110,17 @@ trait CDCTestMixin extends SharedSparkSession { } } -trait ChangelogV2CDCUtilMixin extends CDCTestMixin with ChangelogSyntaxSupportedShim { +trait ChangelogV2CDCUtilMixin + extends SharedSparkSession + with CDCTestMixin + with ChangelogSyntaxSupportedShim { // Tests skipped on the V2 changelog read path. protected def excludedV2Exact: Set[String] = Set( // Read-CDF does not write any files. "usage metrics", // VOID is not a supported Delta data type in the Kernel schema parser. On the V2 changelog - // read path (DeltaChangelogScan -> JvmSnapshot.getSchema) a table whose schema retains a VOID + // read path (DeltaV2ChangelogScan -> JvmSnapshot.getSchema) a table whose schema retains a VOID // field throws KernelException. These tests build a struct that keeps a lit(null) VOID field. // V1 CDCReader does not parse the schema through the Kernel, so these are excluded only on the // V2 path. @@ -474,6 +506,51 @@ trait DeltaCheckpointTestUtils } } +trait DeltaMinorCompactionTestUtils extends DeltaTestUtilsBase { + self: SparkFunSuite with SharedSparkSession => + + /** Helper method to do minor compaction of [[DeltaLog]] from [startVersion, endVersion] */ + protected def minorCompactDeltaLog( + tablePath: String, + startVersion: Long, + endVersion: Long, + tableName: Option[String] = None): Unit = { + val deltaLog = DeltaLog.forTable(spark, tablePath) + val snapshotForReplay = deltaLog.update() + val logReplay = new InMemoryLogReplay( + minFileRetentionTimestamp = None, + minSetTransactionRetentionTimestamp = None, + tableRoot = deltaLog.dataPath, + useDeletionVectorObjectIdentity = FileAction.useDeletionVectorObjectIdentity( + snapshotForReplay.metadata, snapshotForReplay.protocol, spark), + retainFirstBackreference = AMTUtils.amtEnabled(snapshotForReplay)) + val hadoopConf = deltaLog.newDeltaHadoopConf() + val catalogTable = tableName + .map(name => spark.sessionState.catalog.getTableMetadata(new TableIdentifier(name))) + CatalogOwnedTableUtils.populateTableCommitCoordinatorFromCatalog( + spark, catalogTable, snapshotForReplay).foreach { + tableCommitCoordinatorClient => + CoordinatedCommitsUtils.ensureCommitFilesBackfilled( + version = endVersion, + deltaLog = deltaLog, + tableCommitCoordinatorClient = tableCommitCoordinatorClient, + deltaCommitFileProvider = DeltaCommitFileProvider(snapshotForReplay), + catalogTableOpt = catalogTable) + } + + (startVersion to endVersion).foreach { versionToRead => + val file = FileNames.unsafeDeltaFile(deltaLog.logPath, versionToRead) + val actionsIterator = deltaLog.store.readAsIterator(file, hadoopConf).map(Action.fromJson) + logReplay.append(versionToRead, actionsIterator) + } + deltaLog.store.write( + path = FileNames.compactedDeltaFile(deltaLog.logPath, startVersion, endVersion), + actions = logReplay.checkpoint.map(_.json).toIterator, + overwrite = true, + hadoopConf = hadoopConf) + } +} + object DeltaTestUtils extends DeltaTestUtilsBase { sealed trait TableIdentifierOrPath @@ -502,6 +579,25 @@ object DeltaTestUtils extends DeltaTestUtilsBase { AddFile(encodedPath, partitionValues, size, modificationTime, dataChange, stats) } + /** + * Rewrites the `dataChange` an existing commit records in its [[CommitInfo]], leaving its file + * actions untouched so that the two can be made to disagree -- which no write path produces, and + * which is the only way to observe which of the two a reader consulted. + */ + def recordDataChangeInCommitInfo( + deltaLog: DeltaLog, version: Long, dataChange: Option[Boolean]): Unit = { + val conf = deltaLog.newDeltaHadoopConf() + val commitFile = FileNames.unsafeDeltaFile(deltaLog.logPath, version) + val rewritten = deltaLog.store.read(commitFile, conf).map { line => + Action.fromJson(line) match { + case commitInfo: CommitInfo => commitInfo.copy(dataChange = dataChange).json + case _ => line + } + } + deltaLog.store.write(commitFile, rewritten.toIterator, overwrite = true, conf) + DeltaLog.clearCache() + } + /** * Discovers all DeltaOperations.Operation subclasses using reflection. @@ -809,9 +905,9 @@ trait DeltaSQLInMemoryTestUtils trait DeltaDMLTestUtils extends DeltaSQLTestUtils with DeltaTestUtilsBase + with DeltaTableProvider with BeforeAndAfterEach with CDCTestMixin { - self: SharedSparkSession => import testImplicits._ @@ -868,7 +964,7 @@ trait DeltaDMLTestUtils } protected def append(df: DataFrame, partitionBy: Seq[String] = Nil): Unit = { - val dfw = df.write.format("delta").mode("append") + val dfw = df.write.format(writeFormat).mode("append") if (partitionBy.nonEmpty) { dfw.partitionBy(partitionBy: _*) } @@ -902,11 +998,11 @@ trait DeltaDMLTestUtils spark.read .schema(schema) .option("mode", FailFastMode.name) - .json(data.toDS) + .json(spark.createDataset(data)) } else { spark.read .option("mode", FailFastMode.name) - .json(data.toDS) + .json(spark.createDataset(data)) } } @@ -1031,7 +1127,6 @@ trait DeltaDMLInMemoryTestUtils } trait DeltaDMLTestUtilsPathBased extends DeltaDMLTestUtils { - self: SharedSparkSession => protected var tempDir: File = _ @@ -1064,8 +1159,7 @@ trait DeltaDMLTestUtilsPathBased extends DeltaDMLTestUtils { */ case object NameBasedAccessIncompatible extends Tag("NameBasedAccessIncompatible") -trait DeltaDMLTestUtilsNameBased extends DeltaDMLTestUtils { - self: SharedSparkSession => +trait DeltaDMLTestUtilsNameBased extends DeltaDMLTestUtils with SharedSparkSession { override protected def test(testName: String, testTags: Tag*)(testFun: => Any)( implicit pos: Position): Unit = { diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaVacuumSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaVacuumSuite.scala index 6ce740dab21..af78217ad9a 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaVacuumSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaVacuumSuite.scala @@ -1540,20 +1540,27 @@ class DeltaVacuumSuite extends DeltaVacuumSuiteBase with DeltaSQLCommandTest { test("running vacuum on a catalog managed table should fail") { withCatalogManagedTable() { tableName => + val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) + val untrackedFile = new Path(deltaLog.dataPath, "untracked-file") + val fs = untrackedFile.getFileSystem(deltaLog.newDeltaHadoopConf()) + fs.create(untrackedFile).close() + checkError( intercept[DeltaUnsupportedOperationException] { spark.sql(s"VACUUM $tableName") }, "DELTA_UNSUPPORTED_CATALOG_MANAGED_TABLE_OPERATION", - parameters = Map("operation" -> "VACUUM") + parameters = Map("operation" -> "DATA_CLEANUP") ) + assert(fs.exists(untrackedFile)) checkError( intercept[DeltaUnsupportedOperationException] { spark.sql(s"VACUUM $tableName DRY RUN") }, "DELTA_UNSUPPORTED_CATALOG_MANAGED_TABLE_OPERATION", - parameters = Map("operation" -> "VACUUM") + parameters = Map("operation" -> "DATA_CLEANUP") ) + assert(fs.exists(untrackedFile)) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/GeneratedColumnSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/GeneratedColumnSuite.scala index a1244d652c8..d1108190da1 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/GeneratedColumnSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/GeneratedColumnSuite.scala @@ -2229,7 +2229,47 @@ trait GeneratedColumnSuiteBase } } } + + test("validateGeneratedColumns: NULLIF with regexp_extract") { + Seq(false, true).foreach { alwaysInlineCommonExpr => + withSQLConf(SQLConf.ALWAYS_INLINE_COMMON_EXPR.key -> alwaysInlineCommonExpr.toString) { + val body = StructField("body", StringType) + val shortUrlId = withGenerationExpression( + StructField("short_url_id", StringType), + "NULLIF(regexp_extract(body, 'short/([^ ]+)', 1), '')") + validateGeneratedColumns(spark, StructType(Seq(body, shortUrlId))) + } + } + } + + test("OPTIMIZE rewrites files with a NULLIF generated column") { + withTableName("nullif_generated_column") { table => + withSQLConf( + DeltaSQLConf.DELTA_OPTIMIZE_WRITE_ENABLED.key -> "false", + DeltaSQLConf.DELTA_AUTO_COMPACT_ENABLED.key -> "false") { + createTable( + table, + None, + "body STRING, short_url_id STRING", + Map("short_url_id" -> "NULLIF(regexp_extract(body, 'short/([^ ]+)', 1), '')"), + Nil) + + sql(s"INSERT INTO $table (body) VALUES ('short/abc')") + sql(s"INSERT INTO $table (body) VALUES ('no-match')") + + val deltaLog = DeltaLog.forTableWithSnapshot(spark, TableIdentifier(table))._1 + assert(deltaLog.update().numOfFiles > 1) + val numFilesAdded = sql(s"OPTIMIZE $table") + .select($"metrics.numFilesAdded") + .as[Long] + .head() + assert(numFilesAdded > 0) + checkAnswer( + sql(s"SELECT body, short_url_id FROM $table"), + Seq(Row("short/abc", "abc"), Row("no-match", null))) + } + } + } } class GeneratedColumnSuite extends GeneratedColumnSuiteBase - diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/IdentityColumnAdmissionSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/IdentityColumnAdmissionSuite.scala index 3729bf98562..6544913ec69 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/IdentityColumnAdmissionSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/IdentityColumnAdmissionSuite.scala @@ -272,6 +272,9 @@ trait IdentityColumnAdmissionSuiteBase withTempDir { checkpointDir => val ex = intercept[StreamingQueryException] { val stream = MemoryStream[Int] + // AvailableNow snapshots the source data when the query starts, so enqueue data first. + // Otherwise, the query can finish before exercising the identity column write. + stream.addData(1 to 10) val q = stream .toDF .map(_ => Tuple2(1L, 1)) @@ -282,7 +285,6 @@ trait IdentityColumnAdmissionSuiteBase .option("checkpointLocation", checkpointDir.getCanonicalPath) .trigger(Trigger.AvailableNow) .start(path) - stream.addData(1 to 10) q.processAllAvailable() q.stop() } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/InCommitTimestampSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/InCommitTimestampSuite.scala index 2bd1e5a7273..b6fda123b64 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/InCommitTimestampSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/InCommitTimestampSuite.scala @@ -189,7 +189,8 @@ class InCommitTimestampSuite newProtocolOpt = None, DeltaOperations.ManualUpdate, context = Map.empty, - metrics = Map.empty) + metrics = Map.empty, + dataChange = Some(true)) } else { deltaLog.startTransaction().commit( Seq(createTestAddFile("1")), diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/LastCheckpointInfoSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/LastCheckpointInfoSuite.scala index 9fb20fa012a..aeea97ea774 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/LastCheckpointInfoSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/LastCheckpointInfoSuite.scala @@ -17,6 +17,8 @@ package org.apache.spark.sql.delta // scalastyle:off import.ordering.noEmptyLine +import org.apache.spark.sql.delta.actions.{Checkpoint, ContentRoot, Metadata, Protocol} +import org.apache.spark.sql.delta.amt.{AMTLeafComparisons, AMTSingleAction, AMTWriteResult, DataManifestEntry, ManifestInfo, SingleAMTWriteMetrics, Tracking} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaTestImplicits._ @@ -234,6 +236,247 @@ class LastCheckpointInfoSuite extends SharedSparkSession === LastCheckpointInfo.serializeToJson(ci2, addChecksum = true)) } + /** A [[Checkpoint]] action describing `version`, with a root manifest at `rootPath`. */ + private def sampleCheckpointAction( + version: Long = 1L, + rootPath: String = "metadata/root-abc.parquet", + rootSizeInBytes: Long = 4096L): Checkpoint = Checkpoint( + version = version, + contentRoot = ContentRoot(path = rootPath, sizeInBytes = rootSizeInBytes, version = version), + protocol = Protocol(minReaderVersion = 3, minWriterVersion = 7), + metaData = Metadata(id = "metadata-id", name = "t"), + domainMetadata = Nil, + txns = Nil, + sidecars = Nil) + + private def sampleAMTCheckpoint( + version: Long = 1L, + leaves: Seq[DataManifestEntry] = + Seq(sampleLeaf("metadata/leaf-1.parquet"))): LastAMTCheckpoint = + LastAMTCheckpoint( + manifestCommitVersion = version, + checkpoint = Some(sampleCheckpointAction(version)), + leaves = Some(leaves)) + + /** + * A leaf pointer carrying `recordCount` entries and occupying `sizeInBytes` on disk. Every + * optional field is populated with a distinct non-empty value so the round-trip tests actually + * exercise their serialization -- in particular the byte-array fields (`key_metadata`, + * `deleted_positions`, `dv`) whose base64 encoding is easy to get wrong. + */ + private def sampleLeaf( + location: String, + recordCount: Long = 1L, + sizeInBytes: Long = 1024L): DataManifestEntry = DataManifestEntry( + location = location, + file_format = AMTSingleAction.FileFormatParquet, + tracking = Tracking( + status = Tracking.Status.Added, + snapshot_id = Some(11L), + dv_snapshot_id = Some(12L), + sequence_number = Some(13L), + file_sequence_number = Some(14L), + first_row_id = Some(15L), + deleted_positions = Some(Array[Byte](1, 2, 3)), + replaced_positions = Some(Array[Byte](4, 5))), + record_count = recordCount, + file_size_in_bytes = sizeInBytes, + manifest_info = ManifestInfo( + added_files_count = recordCount.toInt, + existing_files_count = 2, + deleted_files_count = 3, + replaced_files_count = 4, + modified_files_count = 5, + added_rows_count = 100L, + existing_rows_count = 200L, + deleted_rows_count = 300L, + replaced_rows_count = 400L, + modified_rows_count = 500L, + min_sequence_number = 7L, + dv = Some(Array[Byte](9, 8, 7)), + dv_cardinality = Some(5L)), + partition = Some(Map("part_col" -> "part_val")), + spec_id = Some(1), + content_stats = Some("""{"minValues":{"col-1":1}}"""), + key_metadata = Some(Array[Byte](16, 17, 18)), + split_offsets = Some(Seq(0L, 128L, 256L))) + + private def sampleAMTWriteResult( + contentRootVersion: Long = 1L, + leaves: Seq[DataManifestEntry] = Seq(sampleLeaf("metadata/leaf-1.parquet"))): AMTWriteResult = + AMTWriteResult( + contentRootVersion = contentRootVersion, + checkpoint = sampleCheckpointAction(contentRootVersion), + leaves = leaves, + includeActionsInCommitJson = true, + amtWriteMetrics = sampleAMTWriteMetrics) + + private val sampleAMTWriteMetrics = SingleAMTWriteMetrics( + trigger = "TEST", + incremental = "false", + materializeDurationMs = 0L) + + /** Reads `_last_checkpoint` back as raw json. */ + private def readLastCheckpointFileAsJson(log: DeltaLog): String = { + val fs = log.LAST_CHECKPOINT.getFileSystem(log.newDeltaHadoopConf()) + val is = fs.open(log.LAST_CHECKPOINT) + try { + IOUtils.toString(is, "UTF-8") + } finally { + is.close() + } + } + + //////////////////////////////////////// + // writeLastCheckpointFileForAMT + //////////////////////////////////////// + + test("writeLastCheckpointFileForAMT - writes an AMT-format _last_checkpoint") { + withTempDir { dir => + spark.range(10).write.format("delta").save(dir.getAbsolutePath) + val log = DeltaLog.forTable(spark, dir) + val writeResult = sampleAMTWriteResult(contentRootVersion = 3) + + log.writeLastCheckpointFileForAMT(manifestCommitVersion = 4, writeResult) + + val info = log.readLastCheckpointFile().getOrElse( + fail("writeLastCheckpointFileForAMT must produce a readable _last_checkpoint.")) + // The pointer describes the manifest tree's content-root version, and records the leaf count + // in `parts`. + assert(info.version == 3) + assert(info.parts == Some(writeResult.leaves.size)) + // `size` (action count) and `numOfAddFiles` are intentionally unavailable for AMT. + assert(info.size == -1) + assert(info.numOfAddFiles.isEmpty) + // The embedded Checkpoint action survives the round trip, so a cold reader can locate the + // root manifest without reading the commit json. + val amt = info.amtCheckpoint.getOrElse(fail("The written info must carry an amtCheckpoint.")) + assert(amt.checkpoint == Some(writeResult.checkpoint)) + assert(amt.manifestCommitVersion == 4) + assert(amt.checkpoint.map(_.contentRoot.path) == Some("metadata/root-abc.parquet")) + AMTLeafComparisons.assertLeavesEqual(amt.leaves.get, writeResult.leaves) + } + } + + test("writeLastCheckpointFileForAMT - records the leaf count") { + withTempDir { dir => + spark.range(10).write.format("delta").save(dir.getAbsolutePath) + val log = DeltaLog.forTable(spark, dir) + val leaves = Seq(sampleLeaf("metadata/leaf-1.parquet"), sampleLeaf("metadata/leaf-2.parquet")) + log.writeLastCheckpointFileForAMT( + manifestCommitVersion = 4, sampleAMTWriteResult(contentRootVersion = 3, leaves = leaves)) + + val info = log.readLastCheckpointFile().get + assert(info.parts == Some(2), "`parts` carries the number of leaves for an AMT checkpoint.") + assert(info.version == 3, "`version` carries the content root version.") + assert(info.amtCheckpoint.map(_.manifestCommitVersion) == Some(4L)) + } + } + + test("writeLastCheckpointFileForAMT - overwrites the previous pointer") { + withTempDir { dir => + spark.range(10).write.format("delta").save(dir.getAbsolutePath) + val log = DeltaLog.forTable(spark, dir) + + log.writeLastCheckpointFileForAMT( + manifestCommitVersion = 4, sampleAMTWriteResult(contentRootVersion = 3)) + assert(log.readLastCheckpointFile().map(_.version) == Some(3L)) + + // A later manifest commit replaces the pointer wholesale: `_last_checkpoint` names only the + // latest checkpoint, so the newer content root must win. + val newer = AMTWriteResult( + contentRootVersion = 5, + checkpoint = sampleCheckpointAction(version = 5, rootPath = "metadata/root-def.parquet"), + leaves = Seq(sampleLeaf("metadata/leaf-9.parquet")), + includeActionsInCommitJson = true, + amtWriteMetrics = sampleAMTWriteMetrics) + log.writeLastCheckpointFileForAMT(manifestCommitVersion = 6, newer) + + val info = log.readLastCheckpointFile().get + assert(info.version == 5) + assert(info.amtCheckpoint.flatMap(_.checkpoint).map(_.contentRoot.path) + == Some("metadata/root-def.parquet")) + assert(info.amtCheckpoint.map(_.manifestCommitVersion) == Some(6)) + } + } + + test("writeLastCheckpointFileForAMT - honors the checksum config") { + withTempDir { dir => + spark.range(10).write.format("delta").save(dir.getAbsolutePath) + val log = DeltaLog.forTable(spark, dir) + val writeResult = sampleAMTWriteResult(contentRootVersion = 1) + + withSQLConf(DeltaSQLConf.LAST_CHECKPOINT_CHECKSUM_ENABLED.key -> "true") { + log.writeLastCheckpointFileForAMT(manifestCommitVersion = 1, writeResult) + assert(readLastCheckpointFileAsJson(log).contains("checksum")) + } + + withSQLConf(DeltaSQLConf.LAST_CHECKPOINT_CHECKSUM_ENABLED.key -> "false") { + log.writeLastCheckpointFileForAMT(manifestCommitVersion = 1, writeResult) + assert(!readLastCheckpointFileAsJson(log).contains("checksum")) + } + } + } + + test("LastCheckpointInfo - amtCheckpoint serialize/deserialize round-trip") { + val ci = LastCheckpointInfo(version = 1, size = -1, parts = None, sizeInBytes = None, + numOfAddFiles = None, checkpointSchema = None, + checkpointType = Some(LastCheckpointInfo.CheckpointType.AMT), + amtCheckpoint = Some(sampleAMTCheckpoint())) + val json = LastCheckpointInfo.serializeToJson(ci, addChecksum = true) + val deserialized = LastCheckpointInfo.deserializeFromJson(json, validate = true) + val amt = deserialized.amtCheckpoint.getOrElse(fail("round-trip dropped the amtCheckpoint.")) + val expectedAmt = ci.amtCheckpoint.get + assert(amt.manifestCommitVersion == expectedAmt.manifestCommitVersion) + assert(amt.checkpoint == expectedAmt.checkpoint) // Checkpoint carries no Array[Byte] fields. + AMTLeafComparisons.assertLeavesEqual(amt.leaves.get, expectedAmt.leaves.get) + assert(deserialized.checkpointType == Some(LastCheckpointInfo.CheckpointType.AMT)) + // Compare the rest non-array fields by `==`. + assert(deserialized.copy(amtCheckpoint = None, checkpointType = None, checksum = None) + == ci.copy(amtCheckpoint = None, checkpointType = None)) + } + + test("LastCheckpointInfo - checkpointType serializes as a bare string") { + val ci = LastCheckpointInfo(version = 1, size = -1, parts = None, sizeInBytes = None, + numOfAddFiles = None, checkpointSchema = None, + checkpointType = Some(LastCheckpointInfo.CheckpointType.AMT), + amtCheckpoint = Some(sampleAMTCheckpoint())) + val json = LastCheckpointInfo.serializeToJson(ci, addChecksum = false) + // The type must land as its bare `name`, not as a nested `{"name":...}` object. Jackson would + // bean-serialize the sealed type without `@JsonValue`, which also changes the checksum shape + // (one `"checkpointType"` entry instead of a `"checkpointType"+"name"` entry) and breaks + // deserialization. + assert(json.contains(s""""checkpointType":"${LastCheckpointInfo.CheckpointType.AMT.name}""""), + s"Expected a bare-string checkpointType in: $json") + assert(!json.contains(""""checkpointType":{"""), + s"checkpointType must not serialize as an object: $json") + + // And it deserializes back to the same singleton, with the checksum validating over it. + val withChecksum = LastCheckpointInfo.serializeToJson(ci, addChecksum = true) + val deserialized = LastCheckpointInfo.deserializeFromJson(withChecksum, validate = true) + assert(deserialized.checkpointType == Some(LastCheckpointInfo.CheckpointType.AMT)) + } + + test("LastCheckpointInfo - an unknown checkpointType decodes to UNKNOWN") { + val ci = LastCheckpointInfo(version = 1, size = -1, parts = None, sizeInBytes = None, + numOfAddFiles = None, checkpointSchema = None, + checkpointType = Some(LastCheckpointInfo.CheckpointType.UNKNOWN), + amtCheckpoint = Some(sampleAMTCheckpoint())) + val json = LastCheckpointInfo.serializeToJson(ci, addChecksum = false) + val forged = json.replace( + s""""checkpointType":"${LastCheckpointInfo.CheckpointType.UNKNOWN.name}"""", + """"checkpointType":"SomeFutureTree"""") + val deserialized = LastCheckpointInfo.deserializeFromJson(forged, validate = false) + assert(deserialized.checkpointType == Some(LastCheckpointInfo.CheckpointType.UNKNOWN), + s"An unrecognized checkpointType must decode to UNKNOWN, got ${deserialized.checkpointType}.") + // The amtCheckpoint alongside the unknown type must still be decoded, not dropped. + val amt = deserialized.amtCheckpoint.getOrElse( + fail("The amtCheckpoint must survive an unknown checkpointType.")) + assert(amt.manifestCommitVersion == ci.amtCheckpoint.get.manifestCommitVersion) + assert(amt.checkpoint == ci.amtCheckpoint.get.checkpoint) + AMTLeafComparisons.assertLeavesEqual(amt.leaves.get, ci.amtCheckpoint.get.leaves.get) + } + test("LastCheckpointInfo - json with duplicate keys should fail") { val jsonString = """{"version":1,"size":3,"parts":3,"checksum":"d84a0aa11c93304d57feca6acaceb7fb","size":2}""" @@ -250,25 +493,15 @@ class LastCheckpointInfoSuite extends SharedSparkSession spark.range(10).write.format("delta").save(dir.getAbsolutePath) val log = DeltaLog.forTable(spark, dir) - def readLastCheckpointFile(): String = { - val fs = log.LAST_CHECKPOINT.getFileSystem(log.newDeltaHadoopConf()) - val is = fs.open(log.LAST_CHECKPOINT) - try { - IOUtils.toString(is, "UTF-8") - } finally { - is.close() - } - } - withSQLConf(DeltaSQLConf.LAST_CHECKPOINT_CHECKSUM_ENABLED.key -> "true") { DeltaLog.forTable(spark, dir).checkpoint() - assert(readLastCheckpointFile().contains("checksum")) + assert(readLastCheckpointFileAsJson(log).contains("checksum")) } spark.range(10).write.mode("append").format("delta").save(dir.getAbsolutePath) withSQLConf(DeltaSQLConf.LAST_CHECKPOINT_CHECKSUM_ENABLED.key -> "false") { DeltaLog.forTable(spark, dir).checkpoint() - assert(!readLastCheckpointFile().contains("checksum")) + assert(!readLastCheckpointFileAsJson(log).contains("checksum")) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/MergeIntoMaterializeSourceSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/MergeIntoMaterializeSourceSuite.scala index 8e3216b6a4c..f6b1f0fdc94 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/MergeIntoMaterializeSourceSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/MergeIntoMaterializeSourceSuite.scala @@ -50,7 +50,7 @@ trait MergeIntoMaterializeSourceMixin with DeltaSQLTestUtils with DeltaTestUtilsBase { - override def beforeAll(): Unit = { + override protected def beforeAll(): Unit = { super.beforeAll() // trigger source materialization in all tests spark.conf.set(DeltaSQLConf.MERGE_MATERIALIZE_SOURCE.key, "all") @@ -69,10 +69,6 @@ trait MergeIntoMaterializeSourceMixin // Data does not need to be big; there is enough latency to unpersist even with small data. val targetDF = spark.range(100).toDF("id") targetDF.write.format("delta").saveAsTable(tblName) - spark.range(90, 120).toDF("id").createOrReplaceTempView("s") - val mergeQuery = - s"MERGE INTO $tblName t USING s ON t.id = s.id " + - "WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT *" // Killer thread tries to unpersist any persisted mergeMaterializedSource RDDs, // until it has seen more than numKills distinct ones (from distinct Merge retries) @@ -138,7 +134,13 @@ trait MergeIntoMaterializeSourceMixin val events = Log4jUsageLogger.track { try { - sql(mergeQuery) + withTempView("s") { + spark.range(90, 120).toDF("id").createOrReplaceTempView("s") + val mergeQuery = + s"MERGE INTO $tblName t USING s ON t.id = s.id " + + "WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT *" + sql(mergeQuery) + } } catch { case NonFatal(ex) => if (numKills < maxAttempts) { @@ -179,6 +181,28 @@ trait MergeIntoMaterializeSourceErrorTests extends MergeIntoMaterializeSourceMix sparkEx.getMessageParameters.get("rddBlockId").contains(s"rdd_${rdd.id}")) } + test("checkpoint block error without rddBlockId does not throw NullPointerException") { + val injectEx = new SparkException( + message = "Checkpoint block not found", + cause = null, + errorClass = Some("CHECKPOINT_RDD_BLOCK_ID_NOT_FOUND"), + messageParameters = Map.empty) + + testWithCustomErrorInjected[SparkException](injectEx) { (thrownEx, error) => + val checkpointError = Iterator + .iterate(thrownEx: Throwable)(_.getCause) + .takeWhile(_ != null) + .collectFirst { + case e: SparkException + if e.getErrorClass == "CHECKPOINT_RDD_BLOCK_ID_NOT_FOUND" => e + } + + assert(checkpointError.isDefined, thrownEx) + assert(!checkpointError.get.getMessageParameters.containsKey("rddBlockId")) + assert(error.isEmpty) + } + } + for { materialized <- BOOLEAN_DOMAIN } test(s"merge logs out of disk errors - materialized=$materialized") { @@ -224,30 +248,32 @@ trait MergeIntoMaterializeSourceErrorTests extends MergeIntoMaterializeSourceMix withTable(tblName) { val targetDF = spark.range(10).toDF("id").withColumn("value", rand()) targetDF.write.format("delta").saveAsTable(tblName) - spark - .range(10) - .mapPartitions { x => - throw inject - x - } - .toDF("id") - .withColumn("value", rand()) - .createOrReplaceTempView("s") - var thrownException: Intercept = null - val events = Log4jUsageLogger - .track { - thrownException = intercept[Intercept] { - sql(s"MERGE INTO $tblName t USING s ON t.id = s.id " + - s"WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT *") + withTempView("s") { + spark + .range(10) + .mapPartitions { x => + throw inject + x } - } - .filter { e => - e.metric == MetricDefinitions.EVENT_TAHOE.name && - e.tags.get("opType").contains(MergeIntoMaterializeSourceError.OP_TYPE) - } - val error = events.headOption - .map(e => JsonUtils.fromJson[MergeIntoMaterializeSourceError](e.blob)) - handle(thrownException, error) + .toDF("id") + .withColumn("value", rand()) + .createOrReplaceTempView("s") + var thrownException: Intercept = null + val events = Log4jUsageLogger + .track { + thrownException = intercept[Intercept] { + sql(s"MERGE INTO $tblName t USING s ON t.id = s.id " + + s"WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT *") + } + } + .filter { e => + e.metric == MetricDefinitions.EVENT_TAHOE.name && + e.tags.get("opType").contains(MergeIntoMaterializeSourceError.OP_TYPE) + } + val error = events.headOption + .map(e => JsonUtils.fromJson[MergeIntoMaterializeSourceError](e.blob)) + handle(thrownException, error) + } } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/MergeIntoMetricsBase.scala b/spark/src/test/scala/org/apache/spark/sql/delta/MergeIntoMetricsBase.scala index 36841308ec6..ffa8e99e733 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/MergeIntoMetricsBase.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/MergeIntoMetricsBase.scala @@ -1337,6 +1337,20 @@ trait MergeIntoMetricsBase testConfig = testConfig ) }} + + test("MERGE output metrics schema is not nullable") { + withTable("target", "source") { + spark.range(10).write.format("delta").saveAsTable("target") + spark.range(5, 15).write.format("delta").saveAsTable("source") + + val schema = spark.sql( + "MERGE INTO target t USING source s ON t.id = s.id " + + "WHEN MATCHED THEN UPDATE SET t.id = s.id " + + "WHEN NOT MATCHED THEN INSERT (id) VALUES (s.id)").schema + + assert(schema.forall(!_.nullable)) + } + } } object MergeIntoMetricsBase extends QueryTest with SharedSparkSession { diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/MergeIntoSQLSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/MergeIntoSQLSuite.scala index e38a659a597..163f7dbc3f0 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/MergeIntoSQLSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/MergeIntoSQLSuite.scala @@ -284,7 +284,13 @@ trait MergeIntoSQLTests extends MergeIntoSQLMixin { src = s"$sourceName s", cond = s"s.key = t.key AND $nonDeterministicCondition", update(condition = "s.key < 2", set = "key = s.key, value = s.value"))) - assert(e.getMessage.contains("DELTA_NON_DETERMINISTIC_FUNCTION_NOT_SUPPORTED")) + checkError( + e, + "DELTA_NON_DETERMINISTIC_FUNCTION_NOT_SUPPORTED", + matchPVals = true, + parameters = Map( + "operation" -> "search condition of MERGE operation", + "expression" -> "\\(condition = .*\\)\\.")) } } @@ -303,7 +309,13 @@ trait MergeIntoSQLTests extends MergeIntoSQLMixin { update( condition = nonDeterministicCondition, set = "key = s.key, value = s.value"))) - assert(e.getMessage.contains("DELTA_NON_DETERMINISTIC_FUNCTION_NOT_SUPPORTED")) + checkError( + e, + "DELTA_NON_DETERMINISTIC_FUNCTION_NOT_SUPPORTED", + matchPVals = true, + parameters = Map( + "operation" -> "UPDATE condition of MERGE operation", + "expression" -> "\\(condition = .*\\)\\.")) } } @@ -320,7 +332,13 @@ trait MergeIntoSQLTests extends MergeIntoSQLMixin { src = s"$sourceName s", cond = s"s.key = t.key", delete(condition = nonDeterministicCondition))) - assert(e.getMessage.contains("DELTA_NON_DETERMINISTIC_FUNCTION_NOT_SUPPORTED")) + checkError( + e, + "DELTA_NON_DETERMINISTIC_FUNCTION_NOT_SUPPORTED", + matchPVals = true, + parameters = Map( + "operation" -> "DELETE condition of MERGE operation", + "expression" -> "\\(condition = .*\\)\\.")) } } @@ -339,7 +357,13 @@ trait MergeIntoSQLTests extends MergeIntoSQLMixin { insert( condition = nonDeterministicCondition, values = "(key, value) VALUES (s.key, s.value)"))) - assert(e.getMessage.contains("DELTA_NON_DETERMINISTIC_FUNCTION_NOT_SUPPORTED")) + checkError( + e, + "DELTA_NON_DETERMINISTIC_FUNCTION_NOT_SUPPORTED", + matchPVals = true, + parameters = Map( + "operation" -> "INSERT condition of MERGE operation", + "expression" -> "\\(condition = .*\\)\\.")) } } @@ -358,7 +382,13 @@ trait MergeIntoSQLTests extends MergeIntoSQLMixin { updateNotMatched( condition = nonDeterministicCondition, set = "key = t.key, value = t.value + 1"))) - assert(e.getMessage.contains("DELTA_NON_DETERMINISTIC_FUNCTION_NOT_SUPPORTED")) + checkError( + e, + "DELTA_NON_DETERMINISTIC_FUNCTION_NOT_SUPPORTED", + matchPVals = true, + parameters = Map( + "operation" -> "UPDATE condition of MERGE operation", + "expression" -> "\\(condition = .*\\)\\.")) } } @@ -375,7 +405,13 @@ trait MergeIntoSQLTests extends MergeIntoSQLMixin { src = s"$sourceName s", cond = s"s.key = t.key", deleteNotMatched(condition = nonDeterministicCondition))) - assert(e.getMessage.contains("DELTA_NON_DETERMINISTIC_FUNCTION_NOT_SUPPORTED")) + checkError( + e, + "DELTA_NON_DETERMINISTIC_FUNCTION_NOT_SUPPORTED", + matchPVals = true, + parameters = Map( + "operation" -> "DELETE condition of MERGE operation", + "expression" -> "\\(condition = .*\\)\\.")) } } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/MergeIntoSchemaEvolutionSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/MergeIntoSchemaEvolutionSuite.scala index 4e1a62bc403..8562bb4d5be 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/MergeIntoSchemaEvolutionSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/MergeIntoSchemaEvolutionSuite.scala @@ -33,7 +33,7 @@ import org.apache.spark.util.Utils * Trait collecting schema evolution test runner methods and other helpers. */ trait MergeIntoSchemaEvolutionMixin extends QueryTest { - self: SharedSparkSession with MergeIntoTestUtils => + self: MergeIntoTestUtils => protected implicit def strToJsonSeq(str: String): Seq[String] = { str.split("\n").filter(_.trim.length > 0) diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/MergeIntoTestUtils.scala b/spark/src/test/scala/org/apache/spark/sql/delta/MergeIntoTestUtils.scala index beb5ac106ec..6fca24201ed 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/MergeIntoTestUtils.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/MergeIntoTestUtils.scala @@ -29,7 +29,6 @@ import org.apache.spark.sql.test.SharedSparkSession * or Scala API resp. */ trait MergeIntoTestUtils extends DeltaDMLTestUtils with MergeHelpers { - self: SharedSparkSession => protected def executeMerge( target: String, @@ -56,7 +55,6 @@ trait MergeIntoTestUtils extends DeltaDMLTestUtils with MergeHelpers { } trait MergeIntoSQLTestUtils extends DeltaSQLTestUtils with MergeIntoTestUtils { - self: SharedSparkSession => protected def basicMergeStmt( cte: Option[String] = None, @@ -120,7 +118,6 @@ trait MergeIntoSQLTestUtils extends DeltaSQLTestUtils with MergeIntoTestUtils { } trait MergeIntoScalaTestUtils extends MergeIntoTestUtils { - self: SharedSparkSession => override protected def executeMerge( target: String, diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/OptimisticTransactionSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/OptimisticTransactionSuite.scala index e68d7720e7c..87b2ff3dccd 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/OptimisticTransactionSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/OptimisticTransactionSuite.scala @@ -28,7 +28,8 @@ import scala.concurrent.duration._ import com.databricks.spark.util.{Log4jUsageLogger, UsageRecord} import org.apache.spark.sql.delta.DeltaOperations.{ManualUpdate, Truncate} import org.apache.spark.sql.delta.DeltaTestUtils.createTestAddFile -import org.apache.spark.sql.delta.actions.{Action, AddCDCFile, AddFile, CommitInfo, Metadata, Protocol, RemoveFile, SetTransaction} +import org.apache.spark.sql.delta.RowId.RowTrackingMetadataDomain +import org.apache.spark.sql.delta.actions.{Action, AddCDCFile, AddFile, CommitInfo, DomainMetadata, Metadata, Protocol, RemoveFile, SetTransaction} import org.apache.spark.sql.delta.coordinatedcommits.{CommitCoordinatorBuilder, CommitCoordinatorProvider, InMemoryCommitCoordinator, InMemoryCommitCoordinatorBuilder, TableCommitCoordinatorClient} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaTestImplicits._ @@ -37,6 +38,7 @@ import io.delta.storage.LogStore import io.delta.storage.commit.{CommitCoordinatorClient, CommitFailedException, CommitResponse, TableDescriptor, UpdatedActions} import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path +import org.mockito.Mockito.{mockingDetails, spy} import org.apache.spark.sql.{Row, SaveMode, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier @@ -227,7 +229,7 @@ class OptimisticTransactionSuite t => t.metadata ), concurrentWrites = Seq( - Metadata()), + testDefaultMetadata()), actions = Nil) check( @@ -289,6 +291,105 @@ class OptimisticTransactionSuite actions = Seq( AddFile("b", Map.empty, 1, 1, dataChange = true))) + check( + "multiple concurrent appends", + conflicts = false, + initialSetup = { log => + Seq(testDefaultMetadata(), testDefaultProtocol()).foreach { action => + log.startTransaction().commit(Seq(action), ManualUpdate) + } + }, + reads = Nil, + concurrentTxns = Seq( + t => t.commit(Seq(createTestAddFile(encodedPath = "winner-1")), Truncate()), + t => t.commit(Seq(createTestAddFile(encodedPath = "winner-2")), Truncate())), + actions = Seq(createTestAddFile(encodedPath = "loser")), + operation = Truncate(), + expectedErrorClass = None, + expectedErrorMessageParameters = None, + exceptionClass = None, + additionalSQLConfs = Seq.empty) + + private def withRowTrackingTable(f: Path => Unit): Unit = withTempDir { dir => + val dataPath = new Path(dir.getCanonicalPath) + withSQLConf("spark.databricks.delta.properties.defaults.enableRowTracking" -> "true") { + spark.range(0, 5).toDF("id").coalesce(1) + .write.format("delta").save(dataPath.toString) + } + f(dataPath) + } + + private def rowTrackingAddFile(path: String, numRecords: Long): AddFile = + AddFile( + path = path, + partitionValues = Map.empty, + size = 1L, + modificationTime = 1L, + dataChange = true, + stats = s"""{"numRecords":$numRecords}""") + + private def assertRowTrackingCommitActions( + txn: OptimisticTransaction, + expectedHighWaterMarks: Seq[Long]): Unit = { + val commitActions = mockingDetails(txn).getInvocations.asScala + .filter(_.getMethod.getName == "doCommit") + .map(_.getArgument[Seq[Action]](2)).toSeq + assert(commitActions.size === expectedHighWaterMarks.size) + commitActions.zip(expectedHighWaterMarks).foreach { case (actions, highWaterMark) => + val rowTrackingActions = actions.collect { + case domain: DomainMetadata if RowTrackingMetadataDomain.isSameDomain(domain) => domain + } + assert(rowTrackingActions === Seq(RowTrackingMetadataDomain(highWaterMark).toDomainMetadata)) + } + } + + test("row tracking high water mark advances with committed files") { + withRowTrackingTable { dataPath => + val txn = spy(startTestTransaction(dataPath)) + val initialHighWaterMark = txn.snapshot.getRowTrackingHighWaterMark().get + txn.commit(Seq(rowTrackingAddFile("row-tracking-file", numRecords = 3L)), ManualUpdate) + val expectedHighWaterMark = initialHighWaterMark + 3L + assertRowTrackingCommitActions(txn, Seq(expectedHighWaterMark)) + + val post = startTestTransaction(dataPath).snapshot + val committed = post.allFiles.collect().find(_.path == "row-tracking-file").get + assert(committed.baseRowId === Some(initialHighWaterMark + 1L)) + assert(post.getRowTrackingHighWaterMark() === Some(expectedHighWaterMark)) + assert(RowId.extractHighWatermark(post) === post.getRowTrackingHighWaterMark()) + } + } + + test("concurrent row tracking commits reassign row IDs and advance the high water mark") { + withRowTrackingTable { dataPath => + val firstTxn = spy(startTestTransaction(dataPath)) + val secondTxn = spy(startTestTransaction(dataPath)) + val initialHighWaterMark = firstTxn.snapshot.getRowTrackingHighWaterMark().get + assert(secondTxn.readVersion === firstTxn.readVersion) + + firstTxn.commit( + Seq(rowTrackingAddFile("first-row-tracking-file", numRecords = 2L)), ManualUpdate) + val firstExpectedHighWaterMark = initialHighWaterMark + 2L + assertRowTrackingCommitActions(firstTxn, Seq(firstExpectedHighWaterMark)) + val afterFirst = startTestTransaction(dataPath).snapshot + assert(afterFirst.getRowTrackingHighWaterMark() === Some(firstExpectedHighWaterMark)) + + secondTxn.commit( + Seq(rowTrackingAddFile("second-row-tracking-file", numRecords = 3L)), ManualUpdate) + val secondExpectedHighWaterMark = initialHighWaterMark + 5L + assertRowTrackingCommitActions( + secondTxn, Seq(initialHighWaterMark + 3L, secondExpectedHighWaterMark)) + + val post = startTestTransaction(dataPath).snapshot + assert(post.version === firstTxn.readVersion + 2L) + val filesByPath = post.allFiles.collect().map(file => file.path -> file).toMap + assert(filesByPath("first-row-tracking-file").baseRowId === + Some(initialHighWaterMark + 1L)) + assert(filesByPath("second-row-tracking-file").baseRowId === + Some(initialHighWaterMark + 3L)) + assert(post.getRowTrackingHighWaterMark() === Some(secondExpectedHighWaterMark)) + } + } + override def beforeEach(): Unit = { super.beforeEach() CommitCoordinatorProvider.clearNonDefaultBuilders() @@ -606,6 +707,25 @@ class OptimisticTransactionSuite } } + test("commit fails after exhausting the retry budget") { + withSQLConf(DeltaSQLConf.DELTA_MAX_RETRY_COMMIT_ATTEMPTS.key -> "0") { + withTempDir { dir => + val log = DeltaLog.forTable(spark, new Path(dir.getCanonicalPath)) + Seq(testDefaultMetadata(), testDefaultProtocol()).foreach { action => + log.startTransaction().commit(Seq(action), ManualUpdate) + } + val txn = startTestTransaction(log.dataPath) + log.startTransaction().commit( + Seq(createTestAddFile(encodedPath = "winner-1")), Truncate()) + + val e = intercept[DeltaIllegalStateException] { + txn.commit(Seq(createTestAddFile(encodedPath = "loser")), Truncate()) + } + assert(e.getErrorClass == "DELTA_MAX_COMMIT_RETRIES_EXCEEDED") + } + } + } + test("Limited retries for non-conflict retryable CommitFailedExceptions") { val commitCoordinatorName = "retryable-non-conflict-commit-coordinator" var commitAttempts = 0 @@ -1195,7 +1315,8 @@ class OptimisticTransactionSuite newProtocolOpt = None, op = DeltaOperations.Restore(Some(0), None), context = Map.empty, - metrics = Map.empty) + metrics = Map.empty, + dataChange = Some(true)) } if (conflict) { assert(e.isInstanceOf[ConcurrentWriteException]) @@ -1807,7 +1928,7 @@ class OptimisticTransactionSuite */ private def commitMixedDataChangeBatch( tempDir: File, - mode: DeltaSQLConf.ConsistentDataChangeValidationMode, + mode: DeltaSQLConf.DataChangeValidationMode, useCommitLarge: Boolean) : (Seq[UsageRecord], Option[Throwable]) = { withSQLConf( @@ -1830,7 +1951,8 @@ class OptimisticTransactionSuite newProtocolOpt = None, op = DeltaOperations.ManualUpdate, context = Map.empty, - metrics = Map.empty) + metrics = Map.empty, + dataChange = Some(true)) } else { log.startTransaction().commit(mixedActions, ManualUpdate) } @@ -1844,13 +1966,13 @@ class OptimisticTransactionSuite } for (useCommitLarge <- BOOLEAN_DOMAIN) { - val callerContext = if (useCommitLarge) "commitLarge" else "commit" + val callerContext = if (useCommitLarge) "commitLarge" else "doCommit" test(s"consistent dataChange validation: FATAL mode throws and logs payload" + s" ($callerContext)") { withTempDir { tempDir => val (records, thrown) = commitMixedDataChangeBatch( - tempDir, DeltaSQLConf.ConsistentDataChangeValidationMode.FATAL, useCommitLarge) + tempDir, DeltaSQLConf.DataChangeValidationMode.FATAL, useCommitLarge) assert(thrown.exists(_.isInstanceOf[IllegalStateException])) assert(records.size == 1) val payload = JsonUtils.fromJson[Map[String, Any]](records.head.blob) @@ -1868,7 +1990,7 @@ class OptimisticTransactionSuite s" ($callerContext)") { withTempDir { tempDir => val (records, thrown) = commitMixedDataChangeBatch( - tempDir, DeltaSQLConf.ConsistentDataChangeValidationMode.LOG, useCommitLarge) + tempDir, DeltaSQLConf.DataChangeValidationMode.LOG, useCommitLarge) assert(thrown.isEmpty) assert(records.size == 1) } @@ -1878,7 +2000,7 @@ class OptimisticTransactionSuite s" ($callerContext)") { withTempDir { tempDir => val (records, thrown) = commitMixedDataChangeBatch( - tempDir, DeltaSQLConf.ConsistentDataChangeValidationMode.OFF, useCommitLarge) + tempDir, DeltaSQLConf.DataChangeValidationMode.OFF, useCommitLarge) assert(thrown.isEmpty) assert(records.isEmpty) } @@ -1888,7 +2010,7 @@ class OptimisticTransactionSuite test("consistent dataChange validation: uniform commit passes in FATAL mode") { withTempDir { tempDir => withSQLConf(DeltaSQLConf.DELTA_COMMIT_VALIDATE_CONSISTENT_DATA_CHANGE_MODE.key -> - DeltaSQLConf.ConsistentDataChangeValidationMode.FATAL.toString) { + DeltaSQLConf.DataChangeValidationMode.FATAL.toString) { val log = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath)) val seedTxn = log.startTransaction() seedTxn.updateMetadataForNewTable(Metadata()) @@ -1906,7 +2028,7 @@ class OptimisticTransactionSuite test("consistent dataChange validation: AddCDCFile is excluded from the check") { withTempDir { tempDir => withSQLConf(DeltaSQLConf.DELTA_COMMIT_VALIDATE_CONSISTENT_DATA_CHANGE_MODE.key -> - DeltaSQLConf.ConsistentDataChangeValidationMode.FATAL.toString) { + DeltaSQLConf.DataChangeValidationMode.FATAL.toString) { val log = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath)) val seedTxn = log.startTransaction() seedTxn.updateMetadataForNewTable(Metadata()) @@ -1923,4 +2045,131 @@ class OptimisticTransactionSuite } } + for (consistentMode <- Seq("off", "log", "fatal")) { + test(s"dataChange validation preserves CommitInfo logging with consistency=$consistentMode") { + withTempDir { tempDir => + withSQLConf( + DeltaSQLConf.DELTA_COMMIT_VALIDATE_CONSISTENT_DATA_CHANGE_MODE.key -> consistentMode, + DeltaSQLConf.DELTA_COMMIT_VALIDATE_EXPECTED_DATA_CHANGE_MODE.key -> "log") { + val log = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath)) + for { + declared <- Seq(None, Some(false), Some(true)) + fileDataChange <- Seq(None, Some(false), Some(true)) + commitInfoFirst <- BOOLEAN_DOMAIN + } { + withClue(s"declared=$declared, files=$fileDataChange, infoFirst=$commitInfoFirst: ") { + val commitInfo = CommitInfo.empty().copy(dataChange = declared) + val otherActions: Seq[Action] = fileDataChange.toSeq.flatMap { dataChange => + Seq( + createTestAddFile(encodedPath = "a", dataChange = dataChange), + createTestAddFile(encodedPath = "b", dataChange = dataChange)) + } ++ Seq(Metadata(), AddCDCFile("_change_data/cdc-file", Map.empty, size = 1L)) + val actions = if (commitInfoFirst) { + commitInfo +: otherActions + } else { + otherActions :+ commitInfo + } + val allRecords = Log4jUsageLogger.track { + val validated = ConflictChecker.trackDataChange( + spark, actions.iterator, log, DeltaOperations.ComputeStats(Nil), "test") + assert(validated.toVector == actions) + } + val derivedDataChange = fileDataChange.getOrElse(false) + val expectMismatch = consistentMode != "off" && + declared.exists(_ != derivedDataChange) + val mismatches = filterUsageRecords( + allRecords, "delta.commit.commitInfoDataChangeMismatch") + assert(mismatches.size == (if (expectMismatch) 1 else 0)) + if (expectMismatch) { + val payload = JsonUtils.fromJson[Map[String, Any]](mismatches.head.blob) + assert(payload("declaredDataChange") == declared.get) + assert(payload("derivedDataChange") == derivedDataChange) + } + val unexpected = filterUsageRecords(allRecords, "delta.commit.unexpectedDataChange") + assert(unexpected.size == (if (derivedDataChange) 1 else 0)) + } + } + } + } + } + } + + private def commitAndCaptureUnexpectedDataChange( + tempDir: File, + op: DeltaOperations.Operation, + addFiles: Seq[AddFile], + dataChange: Boolean, + seedExisting: Boolean, + mode: String = "log"): Seq[UsageRecord] = { + val log = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath)) + val seedTxn = log.startTransaction() + seedTxn.updateMetadataForNewTable(Metadata()) + val seedActions = if (seedExisting) addFiles.map(_.copy(dataChange = true)) else Seq.empty + seedTxn.commit(seedActions, ManualUpdate) + + withSQLConf(DeltaSQLConf.DELTA_COMMIT_VALIDATE_EXPECTED_DATA_CHANGE_MODE.key -> mode) { + val allRecords = Log4jUsageLogger.track { + log.startTransaction().commit(addFiles.map(_.copy(dataChange = dataChange)), op) + } + filterUsageRecords(allRecords, "delta.commit.unexpectedDataChange") + } + } + + test("unexpected dataChange committing dataChange=true is captured") { + withTempDir { tempDir => + val operation = DeltaOperations.ComputeStats(Nil) + val records = commitAndCaptureUnexpectedDataChange( + tempDir, + operation, + Seq(createTestAddFile(encodedPath = "leaked")), + dataChange = true, + seedExisting = true) + assert(records.size == 1) + val payload = JsonUtils.fromJson[Map[String, Any]](records.head.blob) + assert(payload("callerContext") == "doCommit") + assert(payload("operation") == operation.name) + assert(payload.contains("operationParameters")) + assert(payload("actualDataChange") == true) + assert(payload("path") == "leaked") + } + } + + test("unexpected dataChange: an op with expectedFileDataChange=None is not checked (fatal)") { + withTempDir { tempDir => + val records = commitAndCaptureUnexpectedDataChange( + tempDir, + ManualUpdate, + Seq(createTestAddFile(encodedPath = "data")), + dataChange = true, + seedExisting = false, + mode = "fatal") + assert(records.isEmpty) + } + } + + test("unexpected dataChange: fatal mode throws on a no-data-change commit") { + withTempDir { tempDir => + val log = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath)) + val seedTxn = log.startTransaction() + seedTxn.updateMetadataForNewTable(Metadata()) + val leaked = createTestAddFile(encodedPath = "leaked") + val operation = DeltaOperations.ComputeStats(Nil) + seedTxn.commit(Seq(leaked.copy(dataChange = true)), ManualUpdate) + withSQLConf(DeltaSQLConf.DELTA_COMMIT_VALIDATE_EXPECTED_DATA_CHANGE_MODE.key -> "fatal") { + val e = intercept[DeltaIllegalStateException] { + log.startTransaction().commit( + Seq(leaked.copy(dataChange = true)), operation) + } + checkError( + exception = e, + condition = "DELTA_COMMIT_UNEXPECTED_DATA_CHANGE", + sqlState = "XXKDS", + parameters = Map( + "operation" -> operation.name, + "actionType" -> "AddFile", + "actualDataChange" -> "true", + "expectedDataChange" -> "false")) + } + } + } } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/OptimisticTransactionSuiteBase.scala b/spark/src/test/scala/org/apache/spark/sql/delta/OptimisticTransactionSuiteBase.scala index 870a6db9a94..5a18d20694f 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/OptimisticTransactionSuiteBase.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/OptimisticTransactionSuiteBase.scala @@ -21,7 +21,7 @@ import java.util.ConcurrentModificationException import scala.jdk.CollectionConverters._ import org.apache.spark.sql.delta.DeltaOperations.{ManualUpdate, Truncate} -import org.apache.spark.sql.delta.actions.{Action, AddFile, FileAction, Metadata, RemoveFile} +import org.apache.spark.sql.delta.actions.{Action, AddFile, FileAction, Metadata, Protocol, RemoveFile} import org.apache.spark.sql.delta.deletionvectors.RoaringBitmapArray import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaTestImplicits._ @@ -38,6 +38,20 @@ trait OptimisticTransactionSuiteBase with DeletionVectorsTestUtils { + /** + * Starts a new transaction for the test table at `dataPath`. Defaults to the table's V1 + * `OptimisticTransaction`. + */ + protected def startTestTransaction(dataPath: Path): OptimisticTransaction = + DeltaLog.forTable(spark, dataPath).startTransaction() + + /** + * Default metadata and protocol for the test table. + */ + protected def testDefaultMetadata(): Metadata = Metadata() + protected def testDefaultProtocol(): Protocol = Action.supportedProtocolVersion( + featuresToExclude = Seq(CatalogOwnedTableFeature, AdaptiveMetadataTableFeature)) + /** * Check whether the test transaction conflict with the concurrent writes by executing the * given params in the following order: @@ -62,8 +76,7 @@ trait OptimisticTransactionSuiteBase protected def check( name: String, conflicts: Boolean, - setup: Seq[Action] = Seq(Metadata(), Action.supportedProtocolVersion( - featuresToExclude = Seq(CatalogOwnedTableFeature, AdaptiveMetadataTableFeature))), + setup: Seq[Action] = Seq(testDefaultMetadata(), testDefaultProtocol()), reads: Seq[OptimisticTransaction => Unit], concurrentWrites: Seq[Action], actions: Seq[Action], @@ -140,7 +153,7 @@ trait OptimisticTransactionSuiteBase initialSetup(log) // Perform reads - val txn = log.startTransaction() + val txn = startTestTransaction(log.dataPath) reads.foreach(_ (txn)) // Execute concurrent txn while current transaction is active diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/RestoreTableSQLSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/RestoreTableSQLSuite.scala index 11c7df83497..33ef4aca661 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/RestoreTableSQLSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/RestoreTableSQLSuite.scala @@ -52,8 +52,7 @@ class RestoreTableSQLSuite extends RestoreTableSuiteBase { val ex = intercept[AnalysisException] { sql(s"RESTORE TABLE not_exists VERSION AS OF 0") } - assert(ex.getMessage.contains("Table not found") - || ex.getMessage.contains("TABLE_OR_VIEW_NOT_FOUND")) + assert(ex.getCondition == "TABLE_OR_VIEW_NOT_FOUND") } test("restoring a view") { diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/SnapshotManagementSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/SnapshotManagementSuite.scala index 70d6669067a..f11c7d38e1f 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/SnapshotManagementSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/SnapshotManagementSuite.scala @@ -37,6 +37,8 @@ import io.delta.storage.commit.{Commit, CommitCoordinatorClient, GetCommitsRespo import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.FileStatus import org.apache.hadoop.fs.Path +import org.mockito.ArgumentMatchers.{any, anyLong} +import org.mockito.Mockito.{doReturn, spy} import org.apache.spark.SparkConf import org.apache.spark.SparkException @@ -623,6 +625,62 @@ class SnapshotManagementSuite extends QueryTest with DeltaSQLTestUtils with Shar } } + private def hasDeltaEvent(records: Seq[UsageRecord], opType: String): Boolean = { + records.exists(_.tags.get("opType").contains(opType)) + } + + test("updateAfterCommit retries a stale listing: throws when exhausted, else resolves") { + withTempDir { dir => + val path = dir.getCanonicalPath + spark.range(1).write.format("delta").save(path) + val log = DeltaLog.forTable(spark, new Path(path)) + val staleSegment = log.unsafeVolatileSnapshot.logSegment + + // Spy now, while the in-memory snapshot is at v0. + val spyLog = spy(log) + spark.range(1).write.format("delta").mode("append").save(path) + + val freshSnapshot = log.unsafeVolatileSnapshot + val freshSegment = freshSnapshot.logSegment + val fs = log.logPath.getFileSystem(log.newDeltaHadoopConf()) + val commit = + new Commit(1, fs.getFileStatus(DeltaCommitFileProvider(freshSnapshot).deltaFile(1)), 0) + + // With retries disabled, a persistently stale listing throws and logs inconsistentList. + doReturn(staleSegment, Nil: _*).when(spyLog) + .getLogSegmentAfterCommit(anyLong(), any(), any(), any(), any(), any(), any(), any()) + withSQLConf(DeltaSQLConf.DELTA_COMMIT_INCONSISTENT_LIST_MAX_RETRIES.key -> "0") { + val records = Log4jUsageLogger.track { + val e = intercept[DeltaIllegalStateException] { + spyLog.updateAfterCommit( + committedVersion = 1, + commitOpt = Some(commit), + newChecksumOpt = None, + preCommitLogSegment = staleSegment, + catalogTableOpt = None) + } + assert(e.getErrorClass == "DELTA_INVALID_COMMITTED_VERSION") + } + assert(hasDeltaEvent(records, "delta.assertions.commit.inconsistentList")) + assert(!hasDeltaEvent(records, "delta.commit.mitigatedInconsistentList")) + } + + // A listing that catches up on retry succeeds and logs an event instead of throwing. + doReturn(staleSegment, freshSegment).when(spyLog) + .getLogSegmentAfterCommit(anyLong(), any(), any(), any(), any(), any(), any(), any()) + val records = Log4jUsageLogger.track { + spyLog.updateAfterCommit( + committedVersion = 1, + commitOpt = Some(commit), + newChecksumOpt = None, + preCommitLogSegment = staleSegment, + catalogTableOpt = None) + } + assert(hasDeltaEvent(records, "delta.commit.mitigatedInconsistentList")) + assert(!hasDeltaEvent(records, "delta.assertions.commit.inconsistentList")) + } + } + testQuietly("checkpoint/json not found when executor restart " + "after expired checkpoints in the snapshot cache are cleaned up") { // This test deletes a classic checkpoint file by version. @@ -894,4 +952,48 @@ class SnapshotManagementParallelListingSuite extends QueryTest } } } + + test("LogSegment.toString truncates deltas above the configured limit") { + val logPath = new Path("/tmp/fake-table/_delta_log") + def deltaStatus(version: Long): FileStatus = + new FileStatus(1L, false, 1, 1L, version, FileNames.unsafeDeltaFile(logPath, version)) + val segment = LogSegment( + logPath, + version = 19L, + deltas = (0L until 20L).map(deltaStatus), + nonCompactedDeltasOpt = None, + checkpointProviderOpt = None, + deltaAtCheckpointVersionOpt = None, + lastCommitTimestamp = 0L) + + // Each rendered delta path ends in ".json"; the truncation marker does not. + def numRenderedDeltas(str: String): Int = str.split("\\.json", -1).length - 1 + + withSQLConf(DeltaSQLConf.DELTA_LOG_SEGMENT_DELTAS_TO_STRING_LIMIT.key -> "5") { + val str = segment.toString + assert(str.contains("15 of 20 deltas omitted"), str) + assert(numRenderedDeltas(str) === 5, str) + } + + // A negative limit disables truncation. + withSQLConf(DeltaSQLConf.DELTA_LOG_SEGMENT_DELTAS_TO_STRING_LIMIT.key -> "-1") { + val str = segment.toString + assert(!str.contains("omitted"), str) + assert(numRenderedDeltas(str) === 20, str) + } + + // Below the limit, every delta is rendered with no truncation marker. + withSQLConf(DeltaSQLConf.DELTA_LOG_SEGMENT_DELTAS_TO_STRING_LIMIT.key -> "100") { + val str = segment.toString + assert(!str.contains("omitted"), str) + assert(numRenderedDeltas(str) === 20, str) + } + + // Values below -1 are rejected. + intercept[IllegalArgumentException] { + withSQLConf(DeltaSQLConf.DELTA_LOG_SEGMENT_DELTAS_TO_STRING_LIMIT.key -> "-2") { + segment.toString + } + } + } } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/TruncateTableSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/TruncateTableSuite.scala index bdbd228d1f1..764637c4a27 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/TruncateTableSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/TruncateTableSuite.scala @@ -23,12 +23,13 @@ import org.apache.spark.sql.test.SharedSparkSession class TruncateTableSuite extends QueryTest with SharedSparkSession with DeltaSQLCommandTest - with DeltaTestUtilsForTempViews { + with DeltaTestUtilsForTempViews + with DeltaTableProvider { test("non-partitioned table") { withTable("t1") { - sql("CREATE TABLE t1 (a int) USING delta") + sql(createTableSQL("t1", "a int")) sql("INSERT into t1 VALUES (3)") sql("INSERT into t1 VALUES (4)") @@ -44,7 +45,7 @@ class TruncateTableSuite extends QueryTest test("partitioned table") { withTable("t1") { - sql("CREATE TABLE t1 (a int, b int) USING delta partitioned by (b)") + sql(createTableSQL("t1", "a int, b int", partitionBy = "partitioned by (b)")) sql("INSERT into t1 VALUES (1, 1)") sql("INSERT into t1 VALUES (2, 2)") @@ -63,7 +64,7 @@ class TruncateTableSuite extends QueryTest import testImplicits._ val path = dir.getCanonicalPath Seq((1, 2)).toDF("key", "value") - .write.format("delta").partitionBy("key").save(path) + .write.format(writeFormat).partitionBy("key").save(path) sql(s"TRUNCATE TABLE delta.`$path`") checkAnswer(sql(s"select * from delta.`$path`"), Nil) val ae = intercept[AnalysisException] { @@ -75,7 +76,7 @@ class TruncateTableSuite extends QueryTest test("truncate should refresh DF cache - table") { withTable("t1") { - sql("CREATE TABLE t1 (a int, b int) USING delta partitioned by (b)") + sql(createTableSQL("t1", "a int, b int", partitionBy = "partitioned by (b)")) sql("INSERT into t1 VALUES (1, 1)") sql("INSERT into t1 VALUES (2, 2)") @@ -91,7 +92,7 @@ class TruncateTableSuite extends QueryTest import testImplicits._ val path = dir.getCanonicalPath Seq((1, 2)).toDF("key", "value") - .write.format("delta").partitionBy("key").save(path) + .write.format(writeFormat).partitionBy("key").save(path) val df = sql(s"select * from delta.`$path`").cache() sql(s"TRUNCATE TABLE delta.`$path`") checkAnswer(df, Nil) @@ -101,7 +102,7 @@ class TruncateTableSuite extends QueryTest testWithTempView("truncate on temp views") { isSQLTempView => withTable("t1") { - sql("CREATE TABLE t1 (a int) USING delta") + sql(createTableSQL("t1", "a int")) sql("INSERT into t1 VALUES (3)") sql("INSERT into t1 VALUES (4)") diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/UniversalFormatSuiteBase.scala b/spark/src/test/scala/org/apache/spark/sql/delta/UniversalFormatSuiteBase.scala index 9c9d61d0f3c..fb354c4d473 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/UniversalFormatSuiteBase.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/UniversalFormatSuiteBase.scala @@ -227,7 +227,7 @@ trait UniversalFormatSuiteBase extends IcebergCompatUtilsBase executeSql(s"ALTER TABLE $id SET TBLPROPERTIES " + s"('delta.universalFormat.enabledFormats' = 'iceberg')") } - assert(e.getErrorClass === "DELTA_UNIVERSAL_FORMAT_VIOLATION") + assert(e.getErrorClass === "DELTA_UNIVERSAL_FORMAT_VIOLATION.ICEBERG_COMPAT_REQUIRED") } } } @@ -453,7 +453,7 @@ trait UniversalFormatMiscSuiteBase extends IcebergCompatUtilsBase with Universal val e = intercept[DeltaUnsupportedOperationException] { updatedConfiguration = getUpdatedConfiguration(configurationUnderTest) } - assert(e.getErrorClass == "DELTA_UNIVERSAL_FORMAT_VIOLATION") + assert(e.getErrorClass == "DELTA_UNIVERSAL_FORMAT_VIOLATION.ICEBERG_COMPAT_REQUIRED") for (icv <- allCompatObjects.map(_.version)) { configurationUnderTest = Map( @@ -571,7 +571,7 @@ trait UniversalFormatMiscSuiteBase extends IcebergCompatUtilsBase with Universal | 'delta.minWriterVersion' = $w |)""".stripMargin) } - assert(e.getErrorClass == "DELTA_UNIVERSAL_FORMAT_VIOLATION") + assert(e.getErrorClass == "DELTA_UNIVERSAL_FORMAT_VIOLATION.ICEBERG_COMPAT_REQUIRED") val e1 = intercept[DeltaUnsupportedOperationException] { executeSql(s""" @@ -581,7 +581,7 @@ trait UniversalFormatMiscSuiteBase extends IcebergCompatUtilsBase with Universal | 'delta.minWriterVersion' = $w |) AS SELECT 1""".stripMargin) } - assert(e1.getErrorClass == "DELTA_UNIVERSAL_FORMAT_VIOLATION") + assert(e1.getErrorClass == "DELTA_UNIVERSAL_FORMAT_VIOLATION.ICEBERG_COMPAT_REQUIRED") } } } @@ -599,7 +599,7 @@ trait UniversalFormatMiscSuiteBase extends IcebergCompatUtilsBase with Universal executeSql(s"ALTER TABLE $id SET TBLPROPERTIES " + s"('delta.universalFormat.enabledFormats' = 'iceberg')") } - assert(e.getErrorClass === "DELTA_UNIVERSAL_FORMAT_VIOLATION") + assert(e.getErrorClass === "DELTA_UNIVERSAL_FORMAT_VIOLATION.ICEBERG_COMPAT_REQUIRED") } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/UpdateMetricsSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/UpdateMetricsSuite.scala index d8bc64bfa65..6656abbb06e 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/UpdateMetricsSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/UpdateMetricsSuite.scala @@ -350,4 +350,14 @@ class UpdateMetricsSuite extends QueryTest ) } + test("UPDATE output metrics schema is not nullable") { + withTable("target") { + spark.range(10).write.format("delta").saveAsTable("target") + + val schema = spark.sql("UPDATE target SET id = id + 100 WHERE id < 5").schema + + assert(schema.forall(!_.nullable)) + } + } + } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/UpdateSuiteBase.scala b/spark/src/test/scala/org/apache/spark/sql/delta/UpdateSuiteBase.scala index bd84bdf6b38..d30c9c9be98 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/UpdateSuiteBase.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/UpdateSuiteBase.scala @@ -140,7 +140,7 @@ trait UpdateBaseTempViewTests extends UpdateBaseMixin { expectedErrorClassForDataSetTempView: String = null): Unit = { testWithTempView(s"test update on temp view - $name") { isSQLTempView => withTable("tab") { - Seq((0, 3), (1, 2)).toDF("key", "value").write.format("delta").saveAsTable("tab") + Seq((0, 3), (1, 2)).toDF("key", "value").write.format(writeFormat).saveAsTable("tab") createTempViewFromSelect(text, isSQLTempView) val ex = intercept[AnalysisException] { executeUpdate( @@ -176,14 +176,14 @@ trait UpdateBaseTempViewTests extends UpdateBaseMixin { private def testComplexTempViews(name: String)(text: String, expectedResult: Seq[Row]) = { testWithTempView(s"test update on temp view - $name") { isSQLTempView => withTable("tab") { - Seq((0, 3), (1, 2)).toDF("key", "value").write.format("delta").saveAsTable("tab") + Seq((0, 3), (1, 2)).toDF("key", "value").write.format(writeFormat).saveAsTable("tab") createTempViewFromSelect(text, isSQLTempView) executeUpdate( "v", where = "key >= 1 and value < 3", set = "value = key + value, key = key + 1" ) - checkAnswer(spark.read.format("delta").table("v"), expectedResult) + checkAnswer(spark.read.format(writeFormat).table("v"), expectedResult) } } } @@ -226,14 +226,14 @@ trait UpdateBaseMiscTests extends UpdateBaseMixin { test(s"basic update - Delta table by name - Partition=$isPartitioned") { withTable("delta_table") { val partitionByClause = if (isPartitioned) "PARTITIONED BY (key)" else "" - sql(s""" - |CREATE TABLE delta_table(key INT, value INT) USING delta - |$partitionByClause - """.stripMargin) + sql(createTableSQL( + "delta_table", + "key INT, value INT", + partitionBy = partitionByClause)) Seq((2, 2), (1, 4), (1, 1), (0, 3)).toDF("key", "value") .write - .format("delta") + .format(writeFormat) .mode("append") .saveAsTable("delta_table") @@ -513,7 +513,7 @@ trait UpdateBaseMiscTests extends UpdateBaseMixin { spark.read.json(""" {"a": {"b.1": 1, "c.e": 'random'}, "d": 1} {"a": {"b.1": 3, "c.e": 'string'}, "d": 2}""" - .split("\n").toSeq.toDS()).write.format("delta").saveAsTable("`target`") + .split("\n").toSeq.toDS()).write.format(writeFormat).saveAsTable("`target`") executeUpdate( target = "target", @@ -556,7 +556,7 @@ trait UpdateBaseMiscTests extends UpdateBaseMixin { test("Negative case - check target columns during analysis") { withTable("table") { - sql("CREATE TABLE table (s int, t string) USING delta PARTITIONED BY (s)") + sql(createTableSQL("table", "s int, t string", partitionBy = "PARTITIONED BY (s)")) var ae = intercept[AnalysisException] { executeUpdate("table", set = "column_doesnt_exist = 'San Francisco'", where = "t = 'a'") } @@ -603,7 +603,7 @@ trait UpdateBaseMiscTests extends UpdateBaseMixin { withTempDir { dir => val tempPath = dir.getCanonicalPath val df = Seq((2, 2), (3, 2)).toDF("key", "value") - df.write.format("delta").partitionBy("key").save(tempPath) + df.write.format(writeFormat).partitionBy("key").save(tempPath) val e = intercept[AnalysisException] { executeUpdate( @@ -925,7 +925,7 @@ trait UpdateBaseMiscTests extends UpdateBaseMixin { customErrorRegex: Option[String] = None) { test(s"$functionType functions in update - expect exception: $expectException") { withTable("deltaTable") { - data.write.format("delta").saveAsTable("deltaTable") + data.write.format(writeFormat).saveAsTable("deltaTable") val expectedErrorRegex = "(?s).*(?i)unsupported.*(?i).*Invalid expressions.*" @@ -942,7 +942,7 @@ trait UpdateBaseMiscTests extends UpdateBaseMixin { if (catchException) { - val dataBeforeException = spark.read.format("delta").table("deltaTable").collect() + val dataBeforeException = spark.read.format(writeFormat).table("deltaTable").collect() val e = intercept[Exception] { executeUpdate( "deltaTable", @@ -953,7 +953,7 @@ trait UpdateBaseMiscTests extends UpdateBaseMixin { e.getCause.getMessage } else e.getMessage assert(message.matches(errorRegex)) - checkAnswer(spark.read.format("delta").table("deltaTable"), dataBeforeException) + checkAnswer(spark.read.format(writeFormat).table("deltaTable"), dataBeforeException) } else { executeUpdate( "deltaTable", diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/actions/AddFileSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/actions/AddFileSuite.scala index 6b9dd5f827c..7ab72e8a348 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/actions/AddFileSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/actions/AddFileSuite.scala @@ -18,6 +18,7 @@ package org.apache.spark.sql.delta.actions import org.apache.spark.sql.delta.{DeltaConfigs, DeltaLog, DeltaRuntimeException} import org.apache.spark.sql.delta.DeltaTestUtils.BOOLEAN_DOMAIN +import org.apache.spark.sql.delta.amt.AMTPassthrough import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest @@ -67,6 +68,13 @@ class AddFileSuite extends SparkFunSuite with SharedSparkSession with DeltaSQLCo TimestampNTZType) } + test("effectiveFileSequenceNumber uses defaultRowCommitVersion") { + val add = createAddFileWithPartitionValue(Map.empty) + .copy(defaultRowCommitVersion = Some(10L)) + assert(add.effectiveFileSequenceNumber.contains(10L)) + } + + test("normalizedPartitionValues for non-timestamp partitions returns typed literals") { withSQLConf(DeltaSQLConf.DELTA_NORMALIZE_PARTITION_VALUES_ON_READ.key -> "true") { withTempDir { tempDir => diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/actions/DeletionVectorDescriptorSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/actions/DeletionVectorDescriptorSuite.scala index 68e3345cd4e..41d8e674403 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/actions/DeletionVectorDescriptorSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/actions/DeletionVectorDescriptorSuite.scala @@ -47,7 +47,7 @@ class DeletionVectorDescriptorSuite extends SparkFunSuite { assert(dv.offset.isEmpty) // There shouldn't be an offset for inline DV // Unique id to identify the DV - assert(dv.uniqueId === s"i$encodedDVData") + assert(dv.legacyUniqueId === s"i$encodedDVData") assert(dv.uniqueFileId === s"i$encodedDVData") // There is no on-disk file name for an inline DV @@ -57,7 +57,7 @@ class DeletionVectorDescriptorSuite extends SparkFunSuite { // expect the returned DV is same as input, since this is inline // so paths are irrelevant. assert(dv.copyWithAbsolutePath(testTablePath) === dv) - assert(dv.copyWithNewRelativePath(UUID.randomUUID(), "predix2") === dv) + assert(dv.copyWithNewUuidRelativePath(UUID.randomUUID(), "predix2") === dv) } for (offset <- Seq(None, Some(25))) { @@ -76,7 +76,7 @@ class DeletionVectorDescriptorSuite extends SparkFunSuite { // Unique id to identify the DV val offsetSuffix = offset.map(o => s"@$o").getOrElse("") - assert(dv.uniqueId === s"p$testDVAbsPath$offsetSuffix") + assert(dv.legacyUniqueId === s"p$testDVAbsPath$offsetSuffix") assert(dv.uniqueFileId === s"p$testDVAbsPath") // Given the input already has an absolute path, it should return the path in DV @@ -87,17 +87,17 @@ class DeletionVectorDescriptorSuite extends SparkFunSuite { // Copy DV as a relative path DV val uuid = UUID.randomUUID() - val dvCopyWithRelativePath = dv.copyWithNewRelativePath(uuid, "prefix") - assert(dvCopyWithRelativePath.isRelative) - assert(dvCopyWithRelativePath.isOnDisk) - assert(dvCopyWithRelativePath.pathOrInlineDv === encodeUUID(uuid, "prefix")) + val dvCopyWithUuidRelativePath = dv.copyWithNewUuidRelativePath(uuid, "prefix") + assert(dvCopyWithUuidRelativePath.isUuidRelative) + assert(dvCopyWithUuidRelativePath.isOnDisk) + assert(dvCopyWithUuidRelativePath.pathOrInlineDv === encodeUUID(uuid, "prefix")) } } for (offset <- Seq(None, Some(25))) { test(s"On-disk DV with relative path with offset=$offset") { val uuid = UUID.randomUUID() - val dv = onDiskWithRelativePath( + val dv = onDiskWithUuidRelativePath( uuid, randomPrefix = "prefix", sizeInBytes = 15, cardinality = 25, offset) // Make sure the metadata (type, size etc.) in the DV is as expected @@ -113,7 +113,7 @@ class DeletionVectorDescriptorSuite extends SparkFunSuite { // Unique id to identify the DV val offsetSuffix = offset.map(o => s"@$o").getOrElse("") val encodedUUID = encodeUUID(uuid, "prefix") - assert(dv.uniqueId === s"u$encodedUUID$offsetSuffix") + assert(dv.legacyUniqueId === s"u$encodedUUID$offsetSuffix") assert(dv.uniqueFileId === s"u$encodedUUID") // Expect the DV final path to be under the given table path @@ -131,7 +131,90 @@ class DeletionVectorDescriptorSuite extends SparkFunSuite { // Copy DV as a relative path DV - expect to return the same DV as the current // DV already contains relative path. - assert(dv.copyWithNewRelativePath(UUID.randomUUID(), "predix2") === dv) + assert(dv.copyWithNewUuidRelativePath(UUID.randomUUID(), "predix2") === dv) + } + } + + for (offset <- Seq(None, Some(25))) { + test(s"On-disk DV with un-encoded relative path with offset=$offset") { + val dv = createRelativePathDVDescriptor( + testDVRelPath, sizeInBytes = 15, cardinality = 7, offset) + + // Make sure the metadata (type, size etc.) in the DV is as expected + assert(dv.isOnDisk && !dv.isInline, s"Incorrect DV storage type: $dv") + assertCardinality(dv, 7) + + assert(dv.isUnencodedRelative) + assert(!dv.isUuidRelative) + assert(!dv.isAbsolute) + // It carries no UUID, so there is no prefix/UUID pair to recover. + assert(dv.getRandomPrefixAndUuid.isEmpty) + + assert(dv.pathOrInlineDv === testDVRelPath) + assert(dv.sizeInBytes === 15) + intercept[Exception] { dv.inlineData } + assert(dv.offset === offset) + + // Unique id to identify the DV + val offsetSuffix = offset.map(o => s"@$o").getOrElse("") + assert(dv.legacyUniqueId === s"r$testDVRelPath$offsetSuffix") + assert(dv.uniqueFileId === s"r$testDVRelPath") + + // Expect the DV path to resolve under the given table path. + assert(dv.absolutePath(testTablePath) === new Path(s"$testTablePath/$testDVRelPath")) + // And to relativize back to exactly what we started with. + assert(dv.urlEncodedRelativePathIfExists(testTablePath).contains(testDVRelPath)) + + val dvCopyWithAbsPath = dv.copyWithAbsolutePath(testTablePath) + assert(dvCopyWithAbsPath.isAbsolute) + assert(dvCopyWithAbsPath.isOnDisk) + assert(dvCopyWithAbsPath.pathOrInlineDv === + SparkPath.fromPath(dv.absolutePath(testTablePath)).urlEncoded) + assert(dvCopyWithAbsPath.absolutePath(testTablePath) === dv.absolutePath(testTablePath)) + assert(dvCopyWithAbsPath.sizeInBytes === dv.sizeInBytes) + assert(dvCopyWithAbsPath.cardinality === dv.cardinality) + assert(dvCopyWithAbsPath.offset === dv.offset) + } + } + + test("absolutePath() resolves p, u, r DVs with space (%20) to the same unencoded path") { + // Following paths are all equivalent and follow Delta's path encoding rules. + val uuid = UUID.fromString("f92b8939-98dc-41c0-ae52-ffb7df72a37f") + val fileName = assembleDeletionVectorFileName(uuid) + val relativePath = s"dv dir/$fileName" + val absolutePath = s"s3a://table/test/dv%20dir/${fileName.replace("%", "%25")}" + val expectedPath = s"s3a://table/test/dv dir/$fileName" + + val testCases = Seq( + (PATH_DV_MARKER, + onDiskWithAbsolutePath(absolutePath, sizeInBytes = 15, cardinality = 10)), + (UUID_DV_MARKER, + onDiskWithUuidRelativePath( + uuid, randomPrefix = "dv dir", sizeInBytes = 15, cardinality = 10)), + (RELATIVE_DV_MARKER, + createRelativePathDVDescriptor(relativePath, sizeInBytes = 15, cardinality = 10))) + + for ((storageType, dv) <- testCases) { + withClue(s"$storageType DV: ") { + assert(dv.absolutePath(testTablePath).toString === expectedPath) + } + } + } + + test("r DV copyWithAbsolutePath emits URL-encoded path") { + val path = "data/test%dv%prefix-deletes file.puffin" + val dv = createRelativePathDVDescriptor(path, sizeInBytes = 15, cardinality = 7) + val dvCopyWithAbsPath = dv.copyWithAbsolutePath(testTablePath) + + assert(dvCopyWithAbsPath.isAbsolute) + assert(dvCopyWithAbsPath.pathOrInlineDv.contains("test%25dv%25prefix-deletes%20file")) + assert(dvCopyWithAbsPath.absolutePath(testTablePath) === dv.absolutePath(testTablePath)) + } + + test("r DV rejects an absolute path") { + intercept[IllegalArgumentException] { + createRelativePathDVDescriptor( + "s3a://other/dv.bin", sizeInBytes = 15, cardinality = 7) } } @@ -144,14 +227,16 @@ class DeletionVectorDescriptorSuite extends SparkFunSuite { for { offset <- Seq(None, Some(0), Some(25)) - label <- Seq("relative path", "absolute path") + label <- Seq("uuid relative path", "absolute path", "unencoded relative path") } { test(s"base64 round-trip for $label DV with offset=$offset") { val dv = label match { - case "relative path" => onDiskWithRelativePath( + case "uuid relative path" => onDiskWithUuidRelativePath( UUID.randomUUID(), randomPrefix = "prefix", sizeInBytes = 15, cardinality = 25, offset) case "absolute path" => onDiskWithAbsolutePath( testDVAbsPath, sizeInBytes = 15, cardinality = 10, offset) + case "unencoded relative path" => createRelativePathDVDescriptor( + testDVRelPath, sizeInBytes = 15, cardinality = 7, offset) } val encoded = dv.serializeToBase64() val decoded = DeletionVectorDescriptor.deserializeFromBase64(encoded) @@ -159,6 +244,133 @@ class DeletionVectorDescriptorSuite extends SparkFunSuite { } } + /** The `r` form of the `u` DV that `uuid`/`prefix` describe. */ + private def rFormOf(uuid: UUID, prefix: String, offset: Option[Int]) = { + val path = if (prefix.isEmpty) { + s"${DELETION_VECTOR_FILE_NAME_CORE}_$uuid.bin" + } else { + s"$prefix/${DELETION_VECTOR_FILE_NAME_CORE}_$uuid.bin" + } + createRelativePathDVDescriptor(path, sizeInBytes = 15, cardinality = 25, offset = offset) + } + + test("config-aware uniqueId uses descriptor identity when object identity is disabled") { + val uuid = UUID.randomUUID() + val uDv = onDiskWithUuidRelativePath( + uuid, randomPrefix = "prefix", sizeInBytes = 15, cardinality = 25, offset = Some(8)) + val pDv = uDv.copyWithAbsolutePath(testTablePathWithSpace) + val rDv = rFormOf(uuid, "prefix", offset = Some(8)) + + assert(pDv.pathOrInlineDv.contains("test%20path")) + assert(Set(uDv.legacyUniqueId, pDv.legacyUniqueId, rDv.legacyUniqueId).size === 3) + assert(uDv.uniqueId(testTablePathWithSpace, useObjectIdentity = false) === uDv.legacyUniqueId) + assert(pDv.uniqueId(testTablePathWithSpace, useObjectIdentity = false) === pDv.legacyUniqueId) + assert(rDv.uniqueId(testTablePathWithSpace, useObjectIdentity = false) === rDv.legacyUniqueId) + } + + test("config-aware uniqueId collapses u, r, and in-root p with object identity") { + val uuid = UUID.randomUUID() + val uDv = onDiskWithUuidRelativePath( + uuid, randomPrefix = "prefix", sizeInBytes = 15, cardinality = 25, offset = Some(8)) + val pDv = uDv.copyWithAbsolutePath(testTablePathWithSpace) + val rDv = rFormOf(uuid, "prefix", offset = Some(8)) + + assert(pDv.pathOrInlineDv.contains("test%20path")) + assert(uDv.uniqueId(testTablePathWithSpace, useObjectIdentity = true) === + rDv.uniqueId(testTablePathWithSpace, useObjectIdentity = true)) + assert(pDv.uniqueId(testTablePathWithSpace, useObjectIdentity = true) === + rDv.uniqueId(testTablePathWithSpace, useObjectIdentity = true)) + } + + test("object identity keeps out-of-root p distinct") { + val uuid = UUID.randomUUID() + val uDv = onDiskWithUuidRelativePath( + uuid, randomPrefix = "prefix", sizeInBytes = 15, cardinality = 25, offset = Some(8)) + val pElsewhere = onDiskWithAbsolutePath( + SparkPath.fromPath( + new Path(s"s3a://other-bucket/tbl/prefix/${DELETION_VECTOR_FILE_NAME_CORE}_$uuid.bin")) + .urlEncoded, + sizeInBytes = 15, + cardinality = 25, + offset = Some(8)) + + assert(uDv.uniqueId(testTablePath, useObjectIdentity = true) !== + pElsewhere.uniqueId(testTablePath, useObjectIdentity = true)) + } + + test("object identity decodes absolute p path before relativizing") { + val pDv = onDiskWithAbsolutePath( + "file:///tmp/test%20path/temp_dv.bin", sizeInBytes = 15, cardinality = 25, offset = Some(8)) + + assert(pDv.normalizedTableRelativeObjectId(new Path("file:///tmp")) === + // "$storageType$pathOrInlineDv@$offsetValue" + "rtest path/temp_dv.bin@8") + } + + test("object identity strips table root from decoded absolute p path") { + val pDv = onDiskWithAbsolutePath( + "file:///tmp/test%20path/temp_dv.bin", sizeInBytes = 15, cardinality = 25, offset = Some(8)) + + assert(pDv.normalizedTableRelativeObjectId(new Path("file:///tmp/test path")) === + // "$storageType$pathOrInlineDv@$offsetValue" + "rtemp_dv.bin@8") + } + + test("object identity includes offset") { + val base = createRelativePathDVDescriptor( + testDVRelPath, sizeInBytes = 15, cardinality = 25, offset = Some(8)) + assert(base.uniqueId(testTablePath, useObjectIdentity = true) !== + base.copy(offset = Some(9)).uniqueId(testTablePath, useObjectIdentity = true)) + } + + test("log replay keeps descriptor identity when object identity is disabled") { + val uuid = UUID.randomUUID() + val uDv = onDiskWithUuidRelativePath( + uuid, randomPrefix = "prefix", sizeInBytes = 15, cardinality = 25) + val rDv = rFormOf(uuid, "prefix", offset = None) + val replay = new InMemoryLogReplay( + minFileRetentionTimestamp = None, + minSetTransactionRetentionTimestamp = None, + tableRoot = testTablePath, + useDeletionVectorObjectIdentity = false) + + replay.append(0, Iterator(addFileWithDv(uDv))) + replay.append(1, Iterator(removeFileWithDv(rDv))) + + assert(replay.allFiles.map(_.deletionVector) === Seq(uDv)) + } + + test("log replay uses object identity when enabled") { + val uuid = UUID.randomUUID() + val uDv = onDiskWithUuidRelativePath( + uuid, randomPrefix = "prefix", sizeInBytes = 15, cardinality = 25) + val rDv = rFormOf(uuid, "prefix", offset = None) + val replay = new InMemoryLogReplay( + minFileRetentionTimestamp = None, + minSetTransactionRetentionTimestamp = None, + tableRoot = testTablePath, + useDeletionVectorObjectIdentity = true) + + replay.append(0, Iterator(addFileWithDv(uDv))) + replay.append(1, Iterator(removeFileWithDv(rDv))) + + assert(replay.allFiles.isEmpty) + } + + private def addFileWithDv(dv: DeletionVectorDescriptor): AddFile = { + AddFile( + path = "part-000.parquet", + partitionValues = Map.empty, + size = 1, + modificationTime = 1, + dataChange = true, + deletionVector = dv) + } + + private def removeFileWithDv(dv: DeletionVectorDescriptor): RemoveFile = { + addFileWithDv(dv).removeWithTimestamp(timestamp = 2L) + } + private def assertCardinality(dv: DeletionVectorDescriptor, expSize: Int): Unit = { if (expSize == 0) { assert(dv.isEmpty, s"Expected DV to be empty: $dv") @@ -168,6 +380,8 @@ class DeletionVectorDescriptorSuite extends SparkFunSuite { } private val testTablePath = new Path("s3a://table/test") + private val testTablePathWithSpace = new Path("s3a://table/test path") private val testDVAbsPath = "s3a://table/test/dv1.bin" + private val testDVRelPath = "data/deletes-abc.puffin" private val testDVData: Array[Byte] = Array(1, 2, 3, 4) } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTAllFilesInCrcSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTAllFilesInCrcSuite.scala new file mode 100644 index 00000000000..e58200e2741 --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTAllFilesInCrcSuite.scala @@ -0,0 +1,177 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.amt + +import org.apache.spark.sql.delta.{DeletionVectorsTestUtils, DeltaOperations} +import org.apache.spark.sql.delta.actions.DeletionVectorDescriptor +import org.apache.spark.sql.delta.deletionvectors.RoaringBitmapArray +import org.apache.spark.sql.delta.sources.DeltaSQLConf + +import org.apache.spark.SparkConf + +class AMTAllFilesInCrcSuite extends AMTCheckpointTestBase with DeletionVectorsTestUtils { + + override protected def sparkConf: SparkConf = super.sparkConf + .set(DeltaSQLConf.DELTA_ALL_FILES_IN_CRC_ENABLED.key, "true") + .set(DeltaSQLConf.DELTA_ALL_FILES_IN_CRC_VERIFICATION_MODE_ENABLED.key, "true") + + private def crcThreshold: Int = + spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_ALL_FILES_IN_CRC_THRESHOLD_FILES) + + testAcrossAMTCheckpointScenarios( + "root-only AMT persists allFiles in the CRC without a back reference", + "amt_crc_rootonly")( + setup = name => sql(s"INSERT INTO $name VALUES (1)"), + inlineCheckpointTriggerActionsOrSQL = Some(name => Right( + s"INSERT INTO $name VALUES (2)"))) { context => + val snapshot = context.postCheckpointSnapshot + assert(context.provider.leaves.isEmpty, + "a small AMT tree must be root-only (root-resident or single-manifest promoted).") + + // State reconstruction leaves every root-resident file unstamped. + val live = snapshot.allFiles.collect() + assert(live.nonEmpty && live.forall(_.backReference.isEmpty), + "root-resident AMT files must carry no back reference.") + + // The CRC written for this version must carry those same files, still unstamped. + val crc = snapshot.deltaLog.readChecksum(snapshot.version).getOrElse( + fail(s"expected a CRC at version ${snapshot.version}.")) + val crcAdds = crc.allFiles.getOrElse( + fail("a root-only AMT tree must persist allFiles in its CRC.")) + assert(crcAdds.nonEmpty && crcAdds.forall(_.backReference.isEmpty), + "no AddFile persisted in a root-only AMT CRC may carry a back reference.") + assert(crcAdds.map(_.path).toSet == live.map(_.path).toSet, + "CRC allFiles must match state reconstruction exactly.") + + // And the CRC verifies clean (byte-identical) against a fresh state reconstruction. + assert(snapshot.validateFileListAgainstCRC(crc, contextOpt = Some("AMTAllFilesInCrcSuite")), + "a root-only AMT CRC must agree with state reconstruction.") + } + + testAcrossAMTCheckpointScenarios( + "CRC validation canonicalizes equivalent deletion-vector path encodings", + "amt_crc_dv", + sqlConfs = Seq(DeltaSQLConf.DELTA_HISTORY_METRICS_ENABLED.key -> "false"))( + setup = name => appendRowsAsSeparateFiles(name, numFiles = 1, rowsPerFile = 2), + inlineCheckpointTriggerActionsOrSQL = Some { name => + val log = deltaLogForName(name) + val files = log.update().allFiles.collect() + assert(files.length == 1) + Left(( + writeFileWithDVOnDisk(log, files.head, RoaringBitmapArray(0L)), + DeltaOperations.Delete(predicate = Seq.empty))) + }) { context => + val snapshot = context.postCheckpointSnapshot + assert(context.provider.leaves.isEmpty, "the AMT must be root-only.") + + val crc = snapshot.deltaLog.readChecksum(snapshot.version).getOrElse( + fail(s"expected a CRC at version ${snapshot.version}.")) + val crcFiles = crc.allFiles.getOrElse( + fail("a root-only AMT must persist allFiles in its CRC.")) + val reconstructedFiles = snapshot.allFilesViaStateReconstruction.collect() + assert(crcFiles.length == 1 && reconstructedFiles.length == 1) + + val crcDv = crcFiles.head.deletionVector + val reconstructedDv = reconstructedFiles.head.deletionVector + assert(crcDv.storageType == DeletionVectorDescriptor.UUID_DV_MARKER) + assert(reconstructedDv.storageType == DeletionVectorDescriptor.RELATIVE_DV_MARKER) + assert(snapshot.validateFileListAgainstCRC( + crc, contextOpt = Some("AMTAllFilesInCrcSuite"))) + } + + test("AMT tree with leaf manifests keeps its files out of the CRC") { + withSQLConf(leafPackingConfs: _*) { + withTable("amt_crc_leaves") { + createAMTTable("amt_crc_leaves", checkpointInterval = Int.MaxValue) + val deltaLog = deltaLogForName("amt_crc_leaves") + commitCheckpoint(deltaLog, incremental = false) // bootstrap full checkpoint + appendRowsAsSeparateFiles("amt_crc_leaves", numFiles = leafPackedFiles) + commitCheckpoint(deltaLog, incremental = false) + val snapshot = deltaLog.update() + val provider = amtProvider(snapshot).getOrElse(fail("table must be AMT-backed.")) + assertLeafCount(provider.leaves) + assert(snapshot.numOfFiles <= crcThreshold, + "exclusion must be driven by leaves, not by the file-count threshold.") + + val live = snapshot.allFiles.collect() + assert(live.nonEmpty && live.forall(_.backReference.isDefined), + "leaf-resident AMT files must be stamped with a back reference.") + + assert(deltaLog.readChecksum(snapshot.version).flatMap(_.allFiles).isEmpty, + "a leaf-bearing AMT tree must not persist allFiles in its CRC.") + } + } + } + + test("non-AMT CRC never carries a back reference") { + withTable("plain_crc") { + sql("CREATE TABLE plain_crc (id INT) USING DELTA") + sql("INSERT INTO plain_crc VALUES (1)") + sql("INSERT INTO plain_crc VALUES (2)") + + val deltaLog = deltaLogForName("plain_crc") + val snapshot = deltaLog.update() + assert(amtProvider(snapshot).isEmpty, "control table must NOT be AMT-backed.") + + val crc = deltaLog.readChecksum(snapshot.version).getOrElse( + fail(s"expected a CRC at version ${snapshot.version}.")) + val crcAdds = crc.allFiles.getOrElse( + fail("a small non-AMT table should still persist allFiles in its CRC.")) + assert(crcAdds.nonEmpty && crcAdds.forall(_.backReference.isEmpty), + "a non-AMT table must never persist a back reference in its CRC.") + } + } + + test("shrink from multi-leaf to root-only strips stale back references from the CRC") { + withSQLConf(leafPackingConfs: _*) { + withTable("amt_crc_shrink") { + createAMTTable("amt_crc_shrink", checkpointInterval = Int.MaxValue) + val deltaLog = deltaLogForName("amt_crc_shrink") + commitCheckpoint(deltaLog, incremental = false) // bootstrap full checkpoint + + appendRowsAsSeparateFiles("amt_crc_shrink", numFiles = leafPackedFiles) + commitCheckpoint(deltaLog, incremental = false) + val snapshotBackedByMultiLeafAMT = deltaLog.update() + assert(amtProvider(snapshotBackedByMultiLeafAMT).exists(_.leaves.size >= 2), + "the tree must have multiple leaf manifests before the shrink.") + assert(snapshotBackedByMultiLeafAMT.allFiles.collect().forall(_.backReference.isDefined), + "leaf-resident files must be stamped with a back reference.") + + // Delete every row but id 0, leaving a single live file, then re-materialize. The full + // checkpoint promotes to a root-only tree. + sql("DELETE FROM amt_crc_shrink WHERE id >= 1") + commitCheckpoint(deltaLog, incremental = false) + val snapshotBackedByNoLeafAMT = deltaLog.update() + assert(amtProvider(snapshotBackedByNoLeafAMT).exists(_.leaves.isEmpty), + "the tree must be root-only after the shrink.") + val live = snapshotBackedByNoLeafAMT.allFiles.collect() + assert(live.nonEmpty && live.forall(_.backReference.isEmpty), + "root-resident files must not be stamped after the shrink.") + + val crc = deltaLog.readChecksum(snapshotBackedByNoLeafAMT.version).getOrElse( + fail(s"expected a CRC at version ${snapshotBackedByNoLeafAMT.version}.")) + val crcAdds = crc.allFiles.getOrElse( + fail("a root-only AMT tree must persist allFiles in its CRC.")) + assert(crcAdds.nonEmpty && crcAdds.forall(_.backReference.isEmpty), + "the shrink must strip stale leaf back references from the CRC.") + assert(snapshotBackedByNoLeafAMT.validateFileListAgainstCRC( + crc, contextOpt = Some("amt_crc_shrink")), + "a root-only CRC produced by a shrink must verify clean.") + } + } + } +} diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTBackReferenceSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTBackReferenceSuite.scala index 45dffd1a714..0e675f9115f 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTBackReferenceSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTBackReferenceSuite.scala @@ -16,59 +16,51 @@ package org.apache.spark.sql.delta.amt -import org.apache.spark.sql.delta.{DeletionVectorsTestUtils, DeltaLog, DeltaOperations, Snapshot} -import org.apache.spark.sql.delta.actions.{Action, AddFile, BackReference, RemoveFile} +import org.apache.spark.sql.delta.{CurrentTransactionInfo, DeletionVectorsTestUtils, DeltaLog, DeltaMinorCompactionTestUtils, DeltaOperations, IsolationLevel, OptimisticTransaction, Snapshot, WinningCommitMetrics} +import org.apache.spark.sql.delta.actions.{Action, AddFile, BackReference, InMemoryLogReplay, RemoveFile} import org.apache.spark.sql.delta.deletionvectors.RoaringBitmapArray import org.apache.spark.sql.delta.sources.DeltaSQLConf -import org.apache.hadoop.fs.Path +import org.apache.spark.sql.delta.util.FileNames +import org.apache.hadoop.fs.{FileStatus, Path} +import org.apache.spark.paths.SparkPath import org.apache.spark.sql.functions.col -class AMTBackReferenceSuite extends AMTCheckpointTestBase with DeletionVectorsTestUtils { +class AMTBackReferenceSuite + extends AMTCheckpointTestBase + with DeletionVectorsTestUtils + with DeltaMinorCompactionTestUtils { import testImplicits._ - /** - * A back reference identifies a data file by (leaf manifest, position), so a back reference only - * exists for an entry that lives in a leaf: an entry that stays resident in the root is - * reconstructed with `backReference = None`. The incremental writer spills live adds into leaves - * only once the root would exceed `entriesPerLeaf`, and the default (50000) never trips for these - * small tables -- so without this every entry would stay in the root and carry no back reference. - * A cap of one entry forces every live add into its own leaf, matching the leaf-resident layout a - * full rewrite produces, so all checkpoint scenarios stamp back references. - */ - private val leafResidentConfs = Seq(DeltaSQLConf.AMT_ENTRIES_PER_LEAF.key -> "1") - /** Relativizes an absolute leaf manifest path the same way the stamped `backReference.manifest` * is, so test assertions compare against the identical value. */ private def relativeManifest(snapshot: Snapshot, absLeaf: Path): String = { val tableRoot = snapshot.deltaLog.dataPath - val fs = tableRoot.getFileSystem(snapshot.deltaLog.newDeltaHadoopConf()) - AMTUtils.relativizeManifestPathToTableRoot(fs, tableRoot, absLeaf) + AMTUtils.relativizeLocation(tableRoot.toString, absLeaf.toString) } /** - * The (relative leaf manifest, position) -> data-file location map for every live DATA entry - * across the snapshot's leaves, read straight from the leaf parquet via `_metadata.row_index`. + * The (relative leaf manifest, position) -> Delta AddFile path map for every live DATA entry + * across the snapshot's leaves, read from leaf parquet via `_metadata.row_index`. */ private def leafLocationByBackRef(snapshot: Snapshot): Map[(String, Long), String] = { val provider = amtProvider(snapshot).getOrElse(fail("expected AMTCheckpointProvider")) - provider.leafManifestAbsolutePaths.flatMap { leafPath => + allowReadWithinDeltaLog { + provider.liveLeafManifestAbsolutePaths.flatMap { leafPath => val relManifest = relativeManifest(snapshot, leafPath) spark.read.parquet(leafPath.toString) .where(col("content_type") === AMTSingleAction.ContentType.Type.Data) .select(col("location"), col("_metadata.row_index").as("pos")) .collect() - .map(row => (relManifest, row.getLong(1)) -> row.getString(0)) + .map { row => + val deltaPath = SparkPath.fromPathString(row.getString(0)).urlEncoded + (relManifest, row.getLong(1)) -> deltaPath + } }.toMap + } } - /** - * All live [[AddFile]]s of snapshot. - */ - private def liveAddFiles(snapshot: Snapshot): Seq[AddFile] = - snapshot.allFiles.collect().toSeq - /** All actions committed after `afterVersion`, up to the latest version. */ private def actionsAfter(deltaLog: DeltaLog, afterVersion: Long): Seq[Action] = { val latest = deltaLog.update().version @@ -76,23 +68,23 @@ class AMTBackReferenceSuite extends AMTCheckpointTestBase with DeletionVectorsTe } /** - * Creates an AMT table and inserts two single-row files, the second insert landing on a - * checkpoint boundary so both files are captured in the leaves and stamped with back references. - * Returns the two reconstructed leaf-derived `AddFile`s (each carrying a back reference). + * Creates an AMT table, seeds it with single-row files, and checkpoints them into leaves. Returns + * the reconstructed leaf-derived `AddFile`s (each carrying a back reference). */ - private def emitTwoStampedAddFiles(name: String): Seq[AddFile] = { - // A back reference only exists for a leaf-resident entry, so cap the leaf capacity to force - // both live adds out of the root and into leaves (see `leafResidentConfs`). - withSQLConf(leafResidentConfs: _*) { - createAMTTable(name, checkpointInterval = 2) - sql(s"INSERT INTO $name VALUES (1)") - sql(s"INSERT INTO $name VALUES (2)") // Lands on a checkpoint boundary -> emit. + private def emitStampedAddFiles(name: String): Seq[AddFile] = { + // The interval is parked out of reach so the tree comes from the explicit OPTIMIZE CHECKPOINT + // below rather than from a commit happening to land on the interval grid. + createAMTTable(name, checkpointInterval = 100) + val deltaLog = deltaLogForName(name) + withSQLConf(leafPackingConfs: _*) { + appendRowsAsSeparateFiles(name, numFiles = leafPackedFiles) + commitCheckpoint(deltaLog, incremental = false) } - val snapshot = deltaLogForName(name).update() + val snapshot = deltaLog.update() assert(amtProvider(snapshot).isDefined, "table must be AMT-backed after the emit.") val adds = liveAddFiles(snapshot) - assert(adds.size == 2 && adds.forall(_.backReference.isDefined), - "both emitted files must be reconstructed from the leaves with a back reference.") + assert(adds.size == leafPackedFiles && adds.forall(_.backReference.isDefined), + "every emitted file must be reconstructed from the leaves with a back reference.") adds } @@ -106,23 +98,32 @@ class AMTBackReferenceSuite extends AMTCheckpointTestBase with DeletionVectorsTe /** * Asserts that every file action committed after `afterVersion` that reuses a pre-command - * leaf-derived file's path carries exactly that file's back reference. + * leaf-derived file's path carries exactly that file's back reference. A file superseded under + * the same path with a new DV is the one exception: the RemoveFile keeps the back reference + * and the superseding AddFile must carry none. */ private def assertBackRefsPropagated( deltaLog: DeltaLog, afterVersion: Long, backRefByPath: Map[String, Option[BackReference]]): Int = { + val actions = actionsAfter(deltaLog, afterVersion) + val supersededPaths = actions.collect { + case r: RemoveFile if r.backReference.isDefined => r.path + }.toSet var matched = 0 - actionsAfter(deltaLog, afterVersion).foreach { + actions.foreach { case r: RemoveFile if backRefByPath.contains(r.path) => val expected = backRefByPath(r.path) assert(r.backReference == expected, s"RemoveFile ${r.path} back-ref ${r.backReference} must equal source $expected.") matched += 1 + case a: AddFile if supersededPaths.contains(a.path) => + assert(a.backReference.isEmpty, + s"Superseding AddFile ${a.path} must carry no back reference, was ${a.backReference}.") case a: AddFile if backRefByPath.contains(a.path) => val expected = backRefByPath(a.path) assert(a.backReference == expected, - s"Superseding AddFile ${a.path} back-ref ${a.backReference} must equal source $expected.") + s"Re-added AddFile ${a.path} back-ref ${a.backReference} must equal source $expected.") matched += 1 case _ => // A freshly written AddFile (new path) or a non-file action: nothing to inherit. } @@ -132,16 +133,17 @@ class AMTBackReferenceSuite extends AMTCheckpointTestBase with DeletionVectorsTe testAcrossAMTCheckpointScenarios( "reconstructed AddFiles are stamped with a back reference matching the leaf layout", "amt_back_ref_stamped", - sqlConfs = leafResidentConfs)( - setup = name => sql(s"INSERT INTO $name VALUES (1)"), + sqlConfs = leafPackingConfs)( + setup = name => appendRowsAsSeparateFiles(name, numFiles = leafPackedFiles - 1), inlineCheckpointTriggerActionsOrSQL = Some(name => Right( - s"INSERT INTO $name VALUES (2)"))) { context => + s"INSERT INTO $name VALUES (${leafPackedFiles - 1})"))) { context => val groundTruth = leafLocationByBackRef(context.postCheckpointSnapshot) val adds = liveAddFiles(context.postCheckpointSnapshot) - assert(adds.size == 2, - s"Both inserted files must be reconstructed from the leaves, got ${adds.size}.") + assert(adds.size == leafPackedFiles, + s"All $leafPackedFiles inserted files must be reconstructed from the leaves, " + + s"got ${adds.size}.") - val leafPaths = context.provider.leafManifestAbsolutePaths + val leafPaths = context.provider.liveLeafManifestAbsolutePaths .map(relativeManifest(context.postCheckpointSnapshot, _)).toSet adds.foreach { add => val br = add.backReference.getOrElse( @@ -149,18 +151,48 @@ class AMTBackReferenceSuite extends AMTCheckpointTestBase with DeletionVectorsTe assert(leafPaths.contains(br.manifest), s"Back-ref manifest ${br.manifest} must be one of the tree's leaves $leafPaths.") // The back-ref must point at exactly the leaf entry describing this data file. - assert(groundTruth.get((br.manifest, br.pos)).contains(add.path), + assert(groundTruth.get((br.manifest, br.pos.toLong)).contains(add.path), s"Back-ref (${br.manifest}, ${br.pos}) must resolve to the entry for ${add.path}.") } } + testAcrossAMTCheckpointScenarios( + "no back references are generated for a root without leaves", + "amt_back_ref_root_only")( + setup = name => { + sql(s"INSERT INTO $name VALUES (1)") + sql(s"INSERT INTO $name VALUES (2)") + }, + inlineCheckpointTriggerActionsOrSQL = Some(name => Right( + s"INSERT INTO $name VALUES (3)"))) { context => + assert(context.provider.leaves.isEmpty, + "precondition: the tree must hold no leaves for entries to stay root-resident.") + val adds = liveAddFiles(context.postCheckpointSnapshot) + assert(adds.size == 3, + s"All three live files must be reconstructed from the root; got ${adds.size}.") + adds.foreach { add => + assert(add.backReference.isEmpty, + s"Root-resident ${add.path} must carry no back reference, but was ${add.backReference}.") + } + + // A DELETE of a root-resident file likewise has no back reference to propagate. + val deltaLog = context.postCheckpointSnapshot.deltaLog + val vBefore = deltaLog.update().version + sql(s"DELETE FROM ${context.tableName} WHERE id = 1") + val removes = actionsAfter(deltaLog, vBefore).collect { case r: RemoveFile => r } + assert(removes.size == 1, s"The file holding id=1 must be removed whole; got ${removes.size}.") + assert(removes.head.backReference.isEmpty, + s"Removing a root-resident file must emit an empty back reference, " + + s"but was ${removes.head.backReference}.") + } + testAcrossAMTCheckpointScenarios( "DELETE emits a RemoveFile carrying the removed file's back reference", "amt_back_ref_delete", - sqlConfs = leafResidentConfs)( - setup = name => sql(s"INSERT INTO $name VALUES (1)"), + sqlConfs = leafPackingConfs)( + setup = name => appendRowsAsSeparateFiles(name, numFiles = leafPackedFiles - 1), inlineCheckpointTriggerActionsOrSQL = Some(name => Right( - s"INSERT INTO $name VALUES (2)"))) { context => + s"INSERT INTO $name VALUES (${leafPackedFiles - 1})"))) { context => val backRefByPath = stampedBackRefs(context.postCheckpointSnapshot) val deltaLog = context.postCheckpointSnapshot.deltaLog sql(s"DELETE FROM ${context.tableName} WHERE id = 1") @@ -179,17 +211,17 @@ class AMTBackReferenceSuite extends AMTCheckpointTestBase with DeletionVectorsTe testAcrossAMTCheckpointScenarios( "a file added after the emit and removed before the next emit has no back reference", "amt_back_ref_post_emit", - sqlConfs = leafResidentConfs)( - setup = name => (1 to 2).foreach(i => sql(s"INSERT INTO $name VALUES ($i)")), + sqlConfs = leafPackingConfs)( + setup = name => appendRowsAsSeparateFiles(name, numFiles = leafPackedFiles - 1), inlineCheckpointTriggerActionsOrSQL = Some(name => Right( - s"INSERT INTO $name VALUES (3)"))) { context => + s"INSERT INTO $name VALUES (${leafPackedFiles - 1})"))) { context => val stamped = liveAddFiles(context.postCheckpointSnapshot) val stampedPaths = stamped.map(_.path).toSet - assert(stamped.size == 3 && stamped.forall(_.backReference.isDefined), - "all three emitted files must be stamped from the leaves.") + assert(stamped.size == leafPackedFiles && stamped.forall(_.backReference.isDefined), + s"all $leafPackedFiles emitted files must be stamped from the leaves.") // This file is added off a checkpoint boundary, so it never enters a leaf. - sql(s"INSERT INTO ${context.tableName} VALUES (4)") + sql(s"INSERT INTO ${context.tableName} VALUES ($leafPackedFiles)") val deltaLog = context.postCheckpointSnapshot.deltaLog val newFile = deltaLog.update().allFiles.collect() .find(add => !stampedPaths.contains(add.path)) @@ -199,7 +231,7 @@ class AMTBackReferenceSuite extends AMTCheckpointTestBase with DeletionVectorsTe // Removing the unstamped file must produce an unstamped tombstone. val vBefore = deltaLog.update().version - sql(s"DELETE FROM ${context.tableName} WHERE id = 4") + sql(s"DELETE FROM ${context.tableName} WHERE id = $leafPackedFiles") val remove = actionsAfter(deltaLog, vBefore).collect { case r: RemoveFile => r } .find(_.path == newFile.path) .getOrElse(fail(s"expected a RemoveFile for the post-emit file ${newFile.path}.")) @@ -208,12 +240,16 @@ class AMTBackReferenceSuite extends AMTCheckpointTestBase with DeletionVectorsTe } testAcrossAMTCheckpointScenarios( - "removeRows propagates the back reference to the superseding AddFile and the RemoveFile", + "removeRows keeps the back reference on the RemoveFile and drops it from the AddFile", "amt_back_ref_remove_rows", - sqlConfs = leafResidentConfs)( - setup = name => Seq(1, 2).toDF("id").coalesce(1).write.mode("append").insertInto(name), + sqlConfs = leafPackingConfs)( + setup = name => { + // A single file holding two rows. + Seq(1, 2).toDF("id").coalesce(1).write.mode("append").insertInto(name) + appendRowsAsSeparateFiles(name, numFiles = leafPackedFiles - 2, startId = 100) + }, inlineCheckpointTriggerActionsOrSQL = Some(name => Right( - s"INSERT INTO $name VALUES (3)"))) { + s"INSERT INTO $name VALUES (${leafPackedFiles - 1})"))) { context => val twoRowFile = liveAddFiles(context.postCheckpointSnapshot) .find(_.numPhysicalRecords.contains(2L)) @@ -226,8 +262,8 @@ class AMTBackReferenceSuite extends AMTCheckpointTestBase with DeletionVectorsTe twoRowFile.removeRows( deletionVector = dv, updateStats = false) - assert(supersedingAdd.backReference == twoRowFile.backReference, - "The superseding AddFile (new DV) must inherit the source file's back reference.") + assert(supersedingAdd.backReference.isEmpty, + "The superseding AddFile (new DV) is a net-new root entry and must carry no back reference.") assert(removeFile.backReference == twoRowFile.backReference, "The paired RemoveFile must inherit the source file's back reference.") } @@ -235,10 +271,14 @@ class AMTBackReferenceSuite extends AMTCheckpointTestBase with DeletionVectorsTe testAcrossAMTCheckpointScenarios( "DELETE via a persistent DV (removeRows) propagates through a real command", "amt_back_ref_dv_delete", - sqlConfs = leafResidentConfs)( - setup = name => Seq(1, 2).toDF("id").coalesce(1).write.mode("append").insertInto(name), + sqlConfs = leafPackingConfs)( + setup = name => { + // A single file holding two rows. + Seq(1, 2).toDF("id").coalesce(1).write.mode("append").insertInto(name) + appendRowsAsSeparateFiles(name, numFiles = leafPackedFiles - 2, startId = 100) + }, inlineCheckpointTriggerActionsOrSQL = Some(name => Right( - s"INSERT INTO $name VALUES (3)"))) { + s"INSERT INTO $name VALUES (${leafPackedFiles - 1})"))) { context => val backRefByPath = liveAddFiles(context.postCheckpointSnapshot).map(add => add.path -> add.backReference).toMap @@ -257,8 +297,8 @@ class AMTBackReferenceSuite extends AMTCheckpointTestBase with DeletionVectorsTe case r: RemoveFile if r.path == twoRowPath => r }.getOrElse(fail("expected a paired RemoveFile for the two-row file.")) assert(supersedingAdd.deletionVector != null, "the superseding AddFile must carry the DV.") - assert(supersedingAdd.backReference == backRefByPath(twoRowPath), - "the superseding AddFile must inherit the source file's back reference.") + assert(supersedingAdd.backReference.isEmpty, + "the superseding AddFile must not claim the source file's leaf position.") assert(removed.backReference == backRefByPath(twoRowPath), "the paired RemoveFile must inherit the source file's back reference.") } @@ -292,18 +332,21 @@ class AMTBackReferenceSuite extends AMTCheckpointTestBase with DeletionVectorsTe |WHEN MATCHED THEN UPDATE SET id = 200""".stripMargin), tombstones = 1, exact = false), PropagationCase("INSERT OVERWRITE", "amt_back_ref_insert_overwrite", - n => sql(s"INSERT OVERWRITE $n VALUES (99)"), tombstones = 2, exact = true), + n => sql(s"INSERT OVERWRITE $n VALUES (99)"), + tombstones = leafPackedFiles, exact = true), PropagationCase("RESTORE", "amt_back_ref_restore", - n => sql(s"RESTORE TABLE $n TO VERSION AS OF 1"), tombstones = 1, exact = false)) + n => sql(s"RESTORE TABLE $n TO VERSION AS OF 1"), tombstones = 1, exact = false), + PropagationCase("TRUNCATE", "amt_back_ref_truncate", + n => sql(s"TRUNCATE TABLE $n"), tombstones = leafPackedFiles, exact = true)) propagationCases.foreach { c => testAcrossAMTCheckpointScenarios( s"${c.label} tombstones carry the source files' back references", c.table, - sqlConfs = leafResidentConfs)( - setup = name => sql(s"INSERT INTO $name VALUES (1)"), + sqlConfs = leafPackingConfs)( + setup = name => appendRowsAsSeparateFiles(name, numFiles = leafPackedFiles - 1), inlineCheckpointTriggerActionsOrSQL = Some(name => Right( - s"INSERT INTO $name VALUES (2)"))) { context => + s"INSERT INTO $name VALUES (${leafPackedFiles - 1})"))) { context => val backRefByPath = stampedBackRefs(context.postCheckpointSnapshot) c.run(context.tableName) @@ -356,7 +399,7 @@ class AMTBackReferenceSuite extends AMTCheckpointTestBase with DeletionVectorsTe test("commit accepts a RemoveFile carrying the correct back reference") { withTable("amt_commit_ok") { - val adds = emitTwoStampedAddFiles("amt_commit_ok") + val adds = emitStampedAddFiles("amt_commit_ok") // Tombstoning a leaf-derived file keeps its (correct) back reference: the commit must pass. commitActions("amt_commit_ok", Seq(adds.head.removeWithTimestamp())) } @@ -364,7 +407,7 @@ class AMTBackReferenceSuite extends AMTCheckpointTestBase with DeletionVectorsTe test("commit accepts a fresh file that carries no back reference") { withTable("amt_commit_fresh") { - val adds = emitTwoStampedAddFiles("amt_commit_fresh") + val adds = emitStampedAddFiles("amt_commit_fresh") // A brand-new file not present in the tree legitimately carries no back reference. val fresh = adds.head.copy(path = "brand-new-file.parquet", backReference = None) commitActions("amt_commit_fresh", Seq(fresh)) @@ -373,7 +416,7 @@ class AMTBackReferenceSuite extends AMTCheckpointTestBase with DeletionVectorsTe test("commit fails when a leaf-derived file's tombstone is missing its back reference") { withTable("amt_commit_missing") { - val adds = emitTwoStampedAddFiles("amt_commit_missing") + val adds = emitStampedAddFiles("amt_commit_missing") // Strip the back reference off a tombstone for a file the tree says should carry one. val stripped = adds.head.removeWithTimestamp().copy(backReference = None) val ex = intercept[IllegalStateException] { @@ -386,7 +429,7 @@ class AMTBackReferenceSuite extends AMTCheckpointTestBase with DeletionVectorsTe test("commit fails when a fresh file carries a spurious back reference") { withTable("amt_commit_spurious") { - val adds = emitTwoStampedAddFiles("amt_commit_spurious") + val adds = emitStampedAddFiles("amt_commit_spurious") // A file not in the tree must not carry a back reference. val spurious = adds.head.copy(path = "brand-new-file.parquet") val ex = intercept[IllegalStateException] { @@ -398,8 +441,8 @@ class AMTBackReferenceSuite extends AMTCheckpointTestBase with DeletionVectorsTe test("commit fails when a present file's tombstone carries the wrong back reference") { withTable("amt_commit_wrong") { - val adds = emitTwoStampedAddFiles("amt_commit_wrong") - val wrongBr = adds.head.backReference.get.copy(pos = adds.head.backReference.get.pos + 1000L) + val adds = emitStampedAddFiles("amt_commit_wrong") + val wrongBr = adds.head.backReference.get.copy(pos = adds.head.backReference.get.pos + 1000) val wrong = adds.head.removeWithTimestamp().copy(backReference = Some(wrongBr)) val ex = intercept[IllegalStateException] { commitActions("amt_commit_wrong", Seq(wrong)) @@ -408,6 +451,344 @@ class AMTBackReferenceSuite extends AMTCheckpointTestBase with DeletionVectorsTe } } + test("reStampBackReferences drops an MDV-masked back reference and keeps a live one") { + withTable("amt_restamp_mdv") { + val name = "amt_restamp_mdv" + // A tree with three full leaves; every seeded file is stamped with a leaf back reference. + val adds = emitStampedAddFiles(name) + val deltaLog = deltaLogForName(name) + val originalLeafLocations = amtProvider(deltaLog.update()) + .getOrElse(fail("expected AMTCheckpointProvider")).leaves.map(_.location).toSet + val victim = adds.head + val keeper = adds.last + val victimBr = victim.backReference.getOrElse(fail("victim must be leaf-resident.")) + val keeperBr = keeper.backReference.getOrElse(fail("keeper must be leaf-resident.")) + + // Remove the victim whole, then fold the removal into the tree with an incremental + // checkpoint. Its leaf keeps nine live entries, so it is carried forward at the SAME + // location with the victim's position MDV-masked -- the case a leaf-location-only gate + // would wrongly skip. + commitActions(name, Seq(victim.removeWithTimestamp())) + commitCheckpoint(deltaLog, incremental = true) + + val winnerSnapshot = deltaLog.update() + val winner = amtProvider(winnerSnapshot).getOrElse(fail("expected AMTCheckpointProvider")) + assert(winner.leaves.map(_.location).toSet == originalLeafLocations, + "precondition: the winning tree must carry the leaves forward at the same locations.") + assert(winner.leaves.exists(_.location == victimBr.manifest), + "precondition: the victim's leaf must still be present (carried forward) in the winner.") + val livePaths = liveAddFiles(winnerSnapshot).map(_.path).toSet + assert(!livePaths.contains(victim.path), + "precondition: the victim's leaf entry must be MDV-masked (no longer live).") + assert(livePaths.contains(keeper.path), + "precondition: the keeper must still be live at its stamped leaf position.") + + // Re-stamping tombstones for both against the winner tree: the victim's stale back reference + // must be dropped (its entry moved out of the leaf), the keeper's must be preserved. + val restamped = winner.reStampBackReferences( + spark, deltaLog, Seq(victim.removeWithTimestamp(), keeper.removeWithTimestamp())).actions + val backRefByPath = + restamped.collect { case r: RemoveFile => r.path -> r.backReference }.toMap + assert(backRefByPath(victim.path).isEmpty, + s"the MDV-masked victim's stale back reference must be dropped, was " + + s"${backRefByPath(victim.path)}.") + assert(backRefByPath(keeper.path).contains(keeperBr), + s"the still-live keeper's back reference must be preserved as $keeperBr, was " + + s"${backRefByPath(keeper.path)}.") + + // When every back reference is still valid, the batch is returned untouched -- no live map is + // built and no action is re-stamped. + val keeperOnly = Seq(keeper.removeWithTimestamp()) + assert(winner.reStampBackReferences(spark, deltaLog, keeperOnly).actions eq keeperOnly, + "a batch whose back references are all still valid must be returned as-is.") + } + } + + test("conflict-resolution rebase preserves a leaf-derived tombstone's back reference") { + withTable("amt_commit_rebase") { + val adds = emitStampedAddFiles("amt_commit_rebase") + assert(adds.size >= 2, "the scenario needs two distinct leaf-derived files.") + val deltaLog = deltaLogForName("amt_commit_rebase") + val vBefore = deltaLog.update().version + + // T1 reads the current snapshot and plans to tombstone one leaf-derived file, carrying that + // file's stamped back reference. + val txn1 = deltaLog.startTransaction() + val leafRemove = adds.head.removeWithTimestamp() + assert(leafRemove.backReference.isDefined, + "precondition: the file T1 tombstones must be leaf-derived (carry a back reference).") + + // A concurrent commit tombstones a different leaf-derived file and lands first, so T1 must + // rebase over it during conflict resolution rather than conflict outright. + deltaLog.startTransaction().commit( + Seq(adds.last.removeWithTimestamp()), DeltaOperations.ManualUpdate) + + // T1 conflicts, rebases over the winner, and commits; the post-conflict-resolution invariant + // would throw here if the rebase dropped the RemoveFile's back reference. + txn1.commit(Seq(leafRemove), DeltaOperations.ManualUpdate) + + val rebasedRemove = actionsAfter(deltaLog, vBefore).collect { + case r: RemoveFile if r.path == leafRemove.path => r + } + assert(rebasedRemove.size == 1, + s"T1's tombstone for ${leafRemove.path} must land exactly once after the rebase.") + assert(rebasedRemove.head.backReference == adds.head.backReference, + s"the rebased tombstone must keep its source file's back reference " + + s"${adds.head.backReference}, was ${rebasedRemove.head.backReference}.") + } + } + + test("conflict-resolution rebase re-stamps a tombstone onto a winning commit's new tree") { + withTable("amt_commit_rebase_newtree") { + val adds = emitStampedAddFiles("amt_commit_rebase_newtree") + val deltaLog = deltaLogForName("amt_commit_rebase_newtree") + + // T1 reads the current snapshot and plans to tombstone a leaf-derived file. + val txn1 = deltaLog.startTransaction() + val target = adds.head + val readSnapshotRef = target.backReference + assert(readSnapshotRef.isDefined, + "precondition: the file T1 tombstones must be leaf-derived (carry a back reference).") + val leafRemove = target.removeWithTimestamp() + + // A concurrent full-rewrite OPTIMIZE CHECKPOINT wins T1's target version and installs a + // brand-new tree. + withSQLConf(DeltaSQLConf.AMT_ENTRIES_PER_LEAF.key -> "1") { + commitCheckpoint(deltaLog, incremental = false) + } + val winnerSnapshot = deltaLog.update() + val winnerRef = stampedBackRefs(winnerSnapshot)(target.path) + assert(winnerRef.isDefined && winnerRef != readSnapshotRef, + s"precondition: the full rewrite must move the target to a new back reference; " + + s"read=$readSnapshotRef winner=$winnerRef.") + + // T1 conflicts, rebases onto the winning tree + txn1.commit(Seq(leafRemove), DeltaOperations.ManualUpdate) + + val rebasedRemove = actionsAfter(deltaLog, winnerSnapshot.version).collect { + case r: RemoveFile if r.path == target.path => r + } + assert(rebasedRemove.size == 1, + s"T1's tombstone for ${target.path} must land exactly once after the rebase.") + assert(rebasedRemove.head.backReference == winnerRef, + s"the rebased tombstone must carry the winning tree's back reference $winnerRef, " + + s"was ${rebasedRemove.head.backReference}.") + } + } + + test("a conflict resolution that drops a back reference fails the post-commit invariant") { + withTable("amt_commit_rebase_drop") { + val adds = emitStampedAddFiles("amt_commit_rebase_drop") + assert(adds.size >= 2, "the scenario needs two distinct leaf-derived files.") + val deltaLog = deltaLogForName("amt_commit_rebase_drop") + + // T1's overridden rebase drops the tombstone's back reference; the pre-conflict check already + // passed on the intact actions, so only the post-conflict-resolution invariant can catch it. + val txn1 = new OptimisticTransaction( + deltaLog, deltaLog.getInitialCatalogTable, deltaLog.update()) { + override protected def resolveConflicts( + currentTransactionInfo: CurrentTransactionInfo, + firstWinningVersion: Long, + lastWinningVersion: Long, + conflictingCommitFiles: Seq[FileStatus], + commitIsolationLevel: IsolationLevel) + : (CurrentTransactionInfo, Seq[WinningCommitMetrics]) = { + val (resolved, metrics) = super.resolveConflicts(currentTransactionInfo, + firstWinningVersion, lastWinningVersion, conflictingCommitFiles, commitIsolationLevel) + (resolved.copy(actions = resolved.actions.map { + case r: RemoveFile if r.backReference.isDefined => r.copy(backReference = None) + case other => other + }), metrics) + } + } + val leafRemove = adds.head.removeWithTimestamp() + assert(leafRemove.backReference.isDefined, + "precondition: the file T1 tombstones must be leaf-derived (carry a back reference).") + + // A concurrent commit tombstones a different file and lands first, forcing T1 to rebase. + deltaLog.startTransaction().commit( + Seq(adds.last.removeWithTimestamp()), DeltaOperations.ManualUpdate) + + val ex = intercept[IllegalStateException] { + txn1.commit(Seq(leafRemove), DeltaOperations.ManualUpdate) + } + assert(ex.getMessage.contains("does not match the AMT")) + assert(ex.getMessage.contains(leafRemove.path)) + } + } + + test("InMemoryLogReplay preserves back references across the collapse") { + def add(path: String, br: Option[BackReference], modTime: Long): AddFile = + AddFile( + path = path, partitionValues = Map.empty, size = 1L, modificationTime = modTime, + dataChange = true, backReference = br) + def remove(path: String, br: Option[BackReference] = None): RemoveFile = + add(path, br, modTime = 1L).removeWithTimestamp(2L) + def newLogReplay: InMemoryLogReplay = new InMemoryLogReplay( + minFileRetentionTimestamp = None, + minSetTransactionRetentionTimestamp = None, + tableRoot = new Path("/fake/table/root"), + useDeletionVectorObjectIdentity = true, + retainFirstBackreference = true) + def postReplayAddFiles(replay: InMemoryLogReplay): Map[String, AddFile] = + replay.checkpoint.collect { case a: AddFile => a.path -> a }.toMap + def postReplayRemoveFiles(replay: InMemoryLogReplay): Map[String, RemoveFile] = + replay.checkpoint.collect { case r: RemoveFile => r.path -> r }.toMap + + val backRef = BackReference("leaf-0", 3) + + // Re-added AddFiles are preserved across the collapse. + val reAdd = newLogReplay + reAdd.append(0L, Iterator(add("f", br = Some(backRef), modTime = 1L))) + assert(postReplayAddFiles(reAdd)("f").backReference.contains(backRef), + "the only add must preserve its own back reference.") + reAdd.append(1L, Iterator(add("f", br = None, modTime = 2L))) + assert(postReplayAddFiles(reAdd)("f").backReference.contains(backRef), + "the collapsed re-add must preserve the first add's back reference.") + + // Cancelled RemoveFiles are preserved across the collapse. + val removeThenAdd = newLogReplay + removeThenAdd.append(0L, Iterator(remove("f", br = Some(backRef)))) + assert(postReplayRemoveFiles(removeThenAdd)("f").backReference.contains(backRef), + "the only remove must preserve its own back reference.") + removeThenAdd.append(1L, Iterator(add("f", br = None, modTime = 2L))) + assert(!postReplayRemoveFiles(removeThenAdd).contains("f"), + "the first remove must be cancelled by the later add.") + assert(postReplayAddFiles(removeThenAdd)("f").backReference.contains(backRef), + "the collapsed re-add must preserve the remove's back reference.") + + // Cancelled AddFiles are preserved across the collapse. + val addThenRemove = newLogReplay + addThenRemove.append(0L, Iterator(add("f", br = Some(backRef), modTime = 1L))) + addThenRemove.append(1L, Iterator(remove("f", br = None))) + assert(!postReplayAddFiles(addThenRemove).contains("f"), + "the first add must be cancelled by the later remove.") + assert(postReplayRemoveFiles(addThenRemove)("f").backReference.contains(backRef), + "the collapsed tombstone must preserve the add's back reference.") + + // File actions with no back reference are preserved across the collapse. + val unstamped = newLogReplay + unstamped.append(0L, Iterator( + add("a", br = None, modTime = 1L), + remove("b", br = None), + add("c", br = None, modTime = 3L))) + unstamped.append(1L, Iterator(remove("c", br = None))) + assert(postReplayAddFiles(unstamped)("a").backReference.isEmpty, + "the only add must preserve its empty back reference.") + assert(postReplayRemoveFiles(unstamped)("b").backReference.isEmpty, + "the only remove must preserve its empty back reference.") + assert(postReplayRemoveFiles(unstamped)("c").backReference.isEmpty, + "the collapsed tombstone must preserve its empty back reference.") + } + + test("minor-compaction preserves back-references") { + withTable("amt_minor_compaction_e2e") { + val name = "amt_minor_compaction_e2e" + val adds = emitStampedAddFiles(name) + val deltaLog = deltaLogForName(name) + assert(adds.size >= 3, "need at least three leaf-resident files for the scenarios.") + + // Drive several scenarios through one compaction window, each on its own leaf-resident file + // carrying its own distinct back reference, plus a net-new file: + // toAddThenAdd -- re-added with back-reference, the later re-added without + // toRemoveThenAdd -- removed with back-reference, then re-added without + // toAddThenRemove -- re-added with back-reference, then removed without + // toAddWithoutBr -- brand-new file, stays unstamped. + val toAddThenAdd = adds(0) + val toRemoveThenAdd = adds(1) + val toAddThenRemove = adds(2) + val toAddWithoutBr = AddFile( + path = "add-without-br", partitionValues = Map.empty, size = 1L, modificationTime = 1L, + dataChange = false, stats = """{"numRecords":1}""", backReference = None) + val brAddThenAdd = toAddThenAdd.backReference.getOrElse( + fail("toAddThenAdd must be leaf-resident.")) + val brRemoveThenAdd = toRemoveThenAdd.backReference.getOrElse( + fail("toRemoveThenAdd must be leaf-resident")) + val brAddThenRemove = toAddThenRemove.backReference.getOrElse( + fail("toAddThenRemove must be leaf-resident.")) + assert(Set(brAddThenAdd, brRemoveThenAdd, brAddThenRemove).size == 3, + "the files must carry distinct back references.") + + val startVersion = deltaLog.update().version + 1 + // Commit A: each file's first action, all carrying back references except for the last one. + commitActions(name, Seq( + toAddThenAdd, + toRemoveThenAdd.removeWithTimestamp(dataChange = false), + toAddThenRemove, + toAddWithoutBr)) + // Commit B: the later actions on the same files, all without back-references. + commitActions(name, Seq( + toAddThenAdd.copy(backReference = None), + toRemoveThenAdd.copy(backReference = None), + toAddThenRemove.removeWithTimestamp(dataChange = false).copy(backReference = None))) + val endVersion = deltaLog.update().version + + // Compact the two commits above. + minorCompactDeltaLog( + tablePath = deltaLog.dataPath.toString, + startVersion = startVersion, + endVersion = endVersion, + tableName = Some(name)) + + val compactedDelta = FileNames.compactedDeltaFile(deltaLog.logPath, startVersion, endVersion) + val compactedActions = deltaLog.store + .readAsIterator(compactedDelta, deltaLog.newDeltaHadoopConf()) + .map(Action.fromJson).toSeq + val compactedAdds = compactedActions.collect { case a: AddFile => a.path -> a }.toMap + val compactedRemoves = compactedActions.collect { case r: RemoveFile => r.path -> r }.toMap + + // Post-compaction file actions must preserve their own back reference, or the lack thereof. + assert(compactedAdds(toAddThenAdd.path).backReference.contains(brAddThenAdd), + "re-added file must keep its leaf back reference through the collapse.") + assert(compactedAdds(toRemoveThenAdd.path).backReference.contains(brRemoveThenAdd), + "remove-then-re-added file must recover its leaf back reference.") + assert(compactedRemoves(toAddThenRemove.path).backReference.contains(brAddThenRemove), + "add-then-removed file must recover its leaf back reference.") + assert(compactedAdds(toAddWithoutBr.path).backReference.isEmpty, + "add-without-br file must stay unstamped.") + + // No path leaks across the add / remove partition. + assert(!compactedAdds.contains(toAddThenRemove.path), + "a removed file must not also appear as a live add.") + assert( + !compactedRemoves.contains(toAddThenAdd.path) && + !compactedRemoves.contains(toRemoveThenAdd.path), + "a re-added file must not also appear as a tombstone.") + } + } + + // Snapshot selection is covered by AMTSnapshotDiscoverySuite and AMTLogSegmentTrimSuite. + testAcrossAMTCheckpointScenarios( + "minor compaction spanning a content root is allowed", + "amt_minor_compaction_spanning_root", + sqlConfs = Seq(DeltaSQLConf.DELTALOG_MINOR_COMPACTION_USE_FOR_READS.key -> "true"))( + inlineCheckpointTriggerActionsOrSQL = + Some(name => Right(s"INSERT INTO $name VALUES (1)"))) { context => + val deltaLog = context.postCheckpointSnapshot.deltaLog + val rootVersion = context.checkpoint.contentRoot.version + val manifestVersion = context.manifestCommitVersion + + sql(s"INSERT INTO ${context.tableName} VALUES (2)") + val endSnapshot = deltaLog.update() + + val startVersion = rootVersion - 1 + val endVersion = endSnapshot.version + assert(startVersion <= manifestVersion) + assert(manifestVersion < endVersion) + + val compactedDeltaPath = FileNames.compactedDeltaFile( + deltaLog.logPath, startVersion, endVersion) + val fs = compactedDeltaPath.getFileSystem(deltaLog.newDeltaHadoopConf()) + assert(!fs.exists(compactedDeltaPath)) + + minorCompactDeltaLog( + tablePath = deltaLog.dataPath.toString, + startVersion = startVersion, + endVersion = endVersion, + tableName = Some(context.tableName)) + assert(fs.exists(compactedDeltaPath), "Compaction must allow a content root within the window.") + } + /** * Builds an AMT source with stamped files, runs `clone(src, tgt)`, then asserts every AddFile * committed to the target carries no back reference. @@ -417,10 +798,10 @@ class AMTBackReferenceSuite extends AMTCheckpointTestBase with DeletionVectorsTe testAcrossAMTCheckpointScenarios( testName, "amt_clone_src", - sqlConfs = leafResidentConfs)( - setup = name => sql(s"INSERT INTO $name VALUES (1)"), + sqlConfs = leafPackingConfs)( + setup = name => appendRowsAsSeparateFiles(name, numFiles = leafPackedFiles - 1), inlineCheckpointTriggerActionsOrSQL = Some(name => Right( - s"INSERT INTO $name VALUES (2)"))) { context => + s"INSERT INTO $name VALUES (${leafPackedFiles - 1})"))) { context => val src = context.tableName // Derive the clone target from the scenario-unique source, so the scenarios do not share a // table directory (see the naming note in `testAcrossAMTCheckpointScenarios`). @@ -456,20 +837,19 @@ class AMTBackReferenceSuite extends AMTCheckpointTestBase with DeletionVectorsTe testAcrossAMTCheckpointScenarios( "RESTORE tombstones keep the current back ref; re-added files carry none", "amt_back_ref_restore_tombstones", - sqlConfs = leafResidentConfs)( - setup = name => sql(s"INSERT INTO $name VALUES (1)"), + sqlConfs = Seq(DeltaSQLConf.AMT_ENTRIES_PER_LEAF.key -> "1"))( + setup = name => appendRowsAsSeparateFiles(name, numFiles = leafPackedFiles - 1), inlineCheckpointTriggerActionsOrSQL = Some(name => Right( - s"INSERT INTO $name VALUES (2)"))) { context => + s"INSERT INTO $name VALUES (${leafPackedFiles - 1})"))) { context => val name = context.tableName - // restoreTarget holds two stamped files A, B. + // The version RESTORE returns to, where every seeded file is stamped from a leaf. val restoredToByPath = stampedBackRefs(context.postCheckpointSnapshot) val deltaLog = deltaLogForName(name) val restoreTarget = context.checkpoint.version - // The overwrite drops A, B and writes C; a second insert adds D. The suite-wide - // `leafResidentConfs` makes the explicit incremental checkpoint spill every net-new file into - // its own leaf, so the current live files C, D get stamped with this table's own back - // references. + // The overwrite replaces every seeded file with one new file, and a second insert adds another. + // One entry per leaf makes the explicit incremental checkpoint spill each net-new file into its + // own leaf, so both current live files get stamped with this table's own back references. sql(s"INSERT OVERWRITE $name VALUES (99)") sql(s"INSERT INTO $name VALUES (100)") commitCheckpoint(deltaLog, incremental = true) @@ -504,11 +884,37 @@ class AMTBackReferenceSuite extends AMTCheckpointTestBase with DeletionVectorsTe } assert(readded.size == restoredToByPath.size, s"RESTORE must re-add all ${restoredToByPath.size} restored-to files, saw ${readded.size}.") + /* + // RESTORE's needs to recompute the back references for the re-added files, so we cannot simply + // assert their emptiness directly. + // TODO: assert their emptiness accurately after RESTORE is supported. readded.foreach { a => assert(a.backReference.isEmpty, s"re-added AddFile ${a.path} must carry no back reference (stale pointer), " + s"but was ${a.backReference}.") } + */ + } + + + test("CONVERT TO DELTA adds files that carry no back reference") { + withTempDir { dir => + val path = dir.getCanonicalPath + // A plain parquet dataset converted in place becomes an ordinary (non-AMT) Delta table, so + // none of the AddFiles the conversion commits may carry a back reference. + spark.range(0, 5).selectExpr("CAST(id AS INT) AS id") + .write.mode("overwrite").parquet(path) + sql(s"CONVERT TO DELTA parquet.`$path`") + + val snapshot = DeltaLog.forTable(spark, new Path(path)).update() + assert(amtProvider(snapshot).isEmpty, "a converted parquet table must not be AMT-backed.") + val adds = snapshot.allFiles.collect() + assert(adds.nonEmpty, "CONVERT must add the parquet data files to the Delta log.") + adds.foreach { a => + assert(a.backReference.isEmpty, + s"converted AddFile ${a.path} must carry no back reference.") + } + } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTCheckpointPolicySuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTCheckpointPolicySuite.scala index d39fcf96940..8773a7e5d3f 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTCheckpointPolicySuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTCheckpointPolicySuite.scala @@ -17,7 +17,7 @@ package org.apache.spark.sql.delta.amt import com.databricks.spark.util.{Log4jUsageLogger, MetricDefinitions} -import org.apache.spark.sql.delta.{AdaptiveMetadataTableFeature, CommitStats, DeltaLog} +import org.apache.spark.sql.delta.{AdaptiveMetadataTableFeature, DeltaLog} import org.apache.spark.sql.delta.DeltaOperations import org.apache.spark.sql.delta.actions.Checkpoint import org.apache.spark.sql.delta.actions.TableFeatureProtocolUtils._ @@ -50,20 +50,19 @@ class AMTCheckpointPolicySuite extends AMTCheckpointTestBase { (0L to latest).flatMap(v => checkpointAt(deltaLog, v).map(cp => (v, cp))) } - /** The trigger name recorded in the AMT write metrics of the commit `f` produces at `version`. */ - private def amtTriggerNameAt(f: => Unit, version: Long): String = - amtWriteMetricsAt(f, version).trigger + /** The trigger name recorded in the AMT write metrics produced while running `f`. */ + private def amtTriggerNameAt(f: => Unit): String = + amtWriteMetricsAt(f).trigger - /** The AMT write metrics logged for the commit `f` produces at `version`. */ - private def amtWriteMetricsAt(f: => Unit, version: Long): SingleAMTWriteMetrics = { + /** The AMT write metrics produced while running `f`. */ + private def amtWriteMetricsAt(f: => Unit): SingleAMTWriteMetrics = { Log4jUsageLogger.track(f) .filter(e => e.metric == MetricDefinitions.EVENT_TAHOE.name && - e.tags.get("opType").contains("delta.commit.stats")) - .map(e => JsonUtils.fromJson[CommitStats](e.blob)) - .find(_.commitVersion == version) - .flatMap(_.amtWriteMetrics) - .flatMap(_.attempts.headOption) - .getOrElse(fail(s"No AMT write metrics logged for version $version.")) + e.tags.get("opType").contains(AMTUsageLogs.CONFLICT_RESOLUTION_ROUND)) + .map(e => JsonUtils.fromJson[AMTMetrics](e.blob)) + .flatMap(_.singleAMTWriteMetrics) + .headOption + .getOrElse(fail("No AMT write metrics logged.")) } /** @@ -160,7 +159,7 @@ class AMTCheckpointPolicySuite extends AMTCheckpointTestBase { // interval 2, default multiplier 5 -> v2 is NOT a 5x boundary, but it is the first AMT. createAMTTable(name, checkpointInterval = 2) sql(s"INSERT INTO $name VALUES (1)") - val v3Trigger = amtTriggerNameAt(sql(s"INSERT INTO $name VALUES (2)"), version = 3) + val v3Trigger = amtTriggerNameAt(sql(s"INSERT INTO $name VALUES (2)")) assert(v3Trigger == AMTTriggerMode.CheckpointIntervalFull.name, s"The first AMT must be a full rewrite; got $v3Trigger") @@ -281,7 +280,7 @@ class AMTCheckpointPolicySuite extends AMTCheckpointTestBase { "The first AMT is a full rewrite.") // Now a full AMT exists. The next large commit (v4) writes its AMT inline (incrementally). - val v4Metrics = amtWriteMetricsAt(sql(s"INSERT INTO $name VALUES (3)"), version = 4) + val v4Metrics = amtWriteMetricsAt(sql(s"INSERT INTO $name VALUES (3)")) assert(v4Metrics.trigger == AMTTriggerMode.InlineWithLargeCommitIncremental.name, s"Once a full AMT exists, a large commit inlines its AMT; got ${v4Metrics.trigger}") assert(v4Metrics.incremental == "true", diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTCheckpointTestBase.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTCheckpointTestBase.scala index 7f9952ced1e..ce4bd1218cd 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTCheckpointTestBase.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTCheckpointTestBase.scala @@ -18,19 +18,24 @@ package org.apache.spark.sql.delta.amt import java.io.File +import scala.collection.immutable.ListMap + // scalastyle:off import.ordering.noEmptyLine -import org.apache.spark.sql.delta.{AdaptiveMetadataTableFeature, DeltaLog, DeltaOperations, Snapshot} -import org.apache.spark.sql.delta.actions.{Action, AddFile, Checkpoint, RemoveFile} +import com.databricks.spark.util.{Log4jUsageLogger, MetricDefinitions} +import org.apache.spark.sql.delta.{AdaptiveMetadataTableFeature, CommitStats, DeltaLog, DeltaOperations, Snapshot} +import org.apache.spark.sql.delta.actions.{Action, AddFile, Checkpoint, ContentRoot, Metadata, Protocol, RemoveFile} import org.apache.spark.sql.delta.actions.TableFeatureProtocolUtils._ import org.apache.spark.sql.delta.coordinatedcommits.CatalogOwnedTestBaseSuite import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest -import org.apache.spark.sql.delta.util.FileNames +import org.apache.spark.sql.delta.util.{DeltaCommitFileProvider, FileNames, JsonUtils} +import org.apache.hadoop.fs.Path import org.apache.spark.SparkConf -import org.apache.spark.sql.QueryTest +import org.apache.spark.sql.{DataFrame, QueryTest} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.functions.col +import org.apache.spark.sql.types.{ArrayType, BinaryType, BooleanType, ByteType, DataType, DateType, DecimalType, DoubleType, FloatType, IntegerType, LongType, ShortType, StringType, StructField, StructType, TimestampNTZType, TimestampType} /** * Shared fixtures for AMT (`adaptiveMetadata-preview`) test suites: table creation, manifest-tree @@ -46,14 +51,21 @@ trait AMTCheckpointTestBase with DeltaSQLCommandTest { // Register the in-memory commit coordinator so catalog-managed AMT tables can be created locally. - // Backfill batch size 1 so every commit is backfilled to a standard NNN.json immediately (rather - // than staying as a UUID-named staged commit); the suites read commit actions via - // `deltaLog.getChanges`, which only sees backfilled deltas. - override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(1) + // With a small, non-1 batch size (3), commits are backfilled intermittently, creating off-by-one + // scenarios heuristically, maximizing the test coverage. + override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(3) override protected def sparkConf: SparkConf = super.sparkConf .set(DeltaSQLConf.DELTA_ALL_FILES_IN_CRC_ENABLED.key, "false") + /** + * Runs `body` with the format check that rejects reading AMT manifest parquet files directly + * disabled, so a test can read a root or leaf manifest off disk. + */ + protected def allowReadWithinDeltaLog[T](body: => T): T = { + body + } + /** Typed view of a snapshot's checkpoint provider when it is AMT-backed. */ protected def amtProvider(snapshot: Snapshot): Option[AMTCheckpointProvider] = snapshot.checkpointProvider match { @@ -61,6 +73,42 @@ trait AMTCheckpointTestBase case _ => None } + /** + * A fake AMT checkpoint provider built directly (not via `fromCheckpoint`, which would read the + * root manifest parquet). + */ + protected def fakeAMTProviderAt( + cpVersion: Long, + protocol: Protocol = Protocol.forTableFeature(AdaptiveMetadataTableFeature), + metadata: Metadata = Metadata()): AMTCheckpointProvider = { + val checkpoint = Checkpoint( + version = cpVersion, + contentRoot = ContentRoot(path = "fake-root.parquet", sizeInBytes = 0L, version = cpVersion), + protocol = protocol, + metaData = metadata, + domainMetadata = Nil, + txns = Nil, + sidecars = Nil) + new AMTCheckpointProvider( + manifestCommitVersion = cpVersion, + checkpointAction = checkpoint, + leaves = Nil, + tableRoot = new Path("file:/fake-table-root")) + } + + /** + * Runs `body` on the DATA (content_type=0) entry rows across `paths`, read straight off disk + * (pass a provider's manifest paths -- its root and/or live leaves). The read runs with the + * path-based Delta format check disabled, and `body` runs inside that scope, so its terminal + * action reads under the disabled check. + */ + protected def withManifestDataEntries[T](paths: Seq[String])(body: DataFrame => T): T = + allowReadWithinDeltaLog { + body( + spark.read.parquet(paths: _*) + .where(col("content_type") === AMTSingleAction.ContentType.Type.Data)) + } + /** The [[DeltaLog]] for a catalog-managed table accessed by name. */ protected def deltaLogForName(tableName: String): DeltaLog = DeltaLog.forTable(spark, new TableIdentifier(tableName)) @@ -72,11 +120,17 @@ trait AMTCheckpointTestBase protected def createAMTTable( tableName: String, checkpointInterval: Int = 2, - location: Option[String] = None): Unit = { + location: Option[String] = None, + tableSchema: String = "id INT", + partitionColumns: Seq[String] = Seq.empty): Unit = { val locationClause = location.map(l => s"LOCATION '$l'").getOrElse("") + val partitionClause = + if (partitionColumns.isEmpty) "" + else s"PARTITIONED BY (${partitionColumns.mkString(", ")})" sql( - s"""CREATE TABLE $tableName (id INT) USING DELTA + s"""CREATE TABLE $tableName ($tableSchema) USING DELTA |$locationClause + |$partitionClause |TBLPROPERTIES ( | '${propertyKey(AdaptiveMetadataTableFeature)}' = '$FEATURE_PROP_SUPPORTED', | 'delta.columnMapping.mode' = 'id', @@ -85,20 +139,184 @@ trait AMTCheckpointTestBase } /** - * Appends `numRows` rows (ids `startId` until `startId + numRows`) to `tableName` as `numRows` - * separate data files in a single commit. `startId` lets successive calls append disjoint id - * ranges. + * Appends `numFiles` separate data files to `tableName` in a single commit, each holding + * `rowsPerFile` rows (so `numFiles * rowsPerFile` rows total, ids `startId` until + * `startId + numFiles * rowsPerFile`). `startId` lets successive calls append disjoint id ranges. + * `columnExprs` is the projection over `range`'s `id`, one expression per table column; a + * partitioned table passes its partition columns here too. The rows are generated in a single + * partition and capped at `rowsPerFile` per file, so exactly `numFiles` files are written with + * deterministic names. */ protected def appendRowsAsSeparateFiles( - tableName: String, numRows: Int, startId: Int = 0): Unit = { + tableName: String, + numFiles: Int, + rowsPerFile: Int = 1, + startId: Int = 0, + columnExprs: Seq[String] = Seq("CAST(id AS INT)")): Unit = { + require(numFiles > 0 && rowsPerFile > 0, + s"numFiles ($numFiles) and rowsPerFile ($rowsPerFile) must be positive.") + val numRows = numFiles * rowsPerFile + require(numRows % rowsPerFile == 0, + s"numRows ($numRows) must be divisible by rowsPerFile ($rowsPerFile).") withSQLConf( - "spark.sql.files.maxRecordsPerFile" -> "1", + "spark.sql.files.maxRecordsPerFile" -> rowsPerFile.toString, DeltaSQLConf.DELTA_OPTIMIZE_WRITE_ENABLED.key -> "false") { sql( - s"INSERT INTO $tableName SELECT CAST(id AS INT) FROM range($startId, ${startId + numRows})") + s"""INSERT INTO $tableName + |SELECT ${columnExprs.mkString(", ")} + |FROM range($startId, ${startId + numRows}, 1, 1)""".stripMargin) } } + /** All live [[AddFile]]s of `snapshot`. */ + protected def liveAddFiles(snapshot: Snapshot): Seq[AddFile] = + snapshot.allFiles.collect().toSeq + + /** + * Test column fixture + */ + protected case class TestColumn( + name: String, + dataType: DataType, + valueExpr: String, + partitionable: Boolean) { + /** This column as a `StructField`, its name prefixed (e.g. `p_`, `c_`, `d_`). */ + def structField(prefix: String): StructField = StructField(prefix + name, dataType) + + /** This column as a ` ` SQL column definition, its name prefixed. */ + def columnDef(prefix: String): String = s"$prefix$name ${dataType.sql}" + } + + /** + * Every type the AMT fixtures exercise, keyed by column [[TestColumn.name]]. + */ + protected val allTypeColumns: ListMap[String, TestColumn] = { + val columns = Seq( + TestColumn("int", IntegerType, "CAST(id AS INT)", partitionable = true), + TestColumn("long", LongType, "CAST(id AS LONG)", partitionable = true), + TestColumn("short", ShortType, "CAST(id AS SHORT)", partitionable = true), + TestColumn("byte", ByteType, "CAST(id AS BYTE)", partitionable = true), + TestColumn("str", StringType, "CONCAT('row', CAST(id AS STRING))", partitionable = true), + TestColumn("date", DateType, "DATE '2026-07-25' + CAST(id AS INT)", partitionable = true), + TestColumn( + "ts", + TimestampType, + "TIMESTAMP '2026-07-25 01:02:03.456' + MAKE_INTERVAL(0, 0, 0, CAST(id AS INT))", + partitionable = true), + TestColumn( + "ts_ntz", + TimestampNTZType, + "CAST(TIMESTAMP_NTZ '2026-07-25 01:02:03.456' + " + + "MAKE_INTERVAL(0, 0, 0, CAST(id AS INT)) AS TIMESTAMP_NTZ)", + partitionable = true), + TestColumn("dec", DecimalType(9, 3), "CAST(id AS DECIMAL(9,3)) / 8", partitionable = true), + TestColumn("bool", BooleanType, "CAST(id % 2 AS BOOLEAN)", partitionable = true), + TestColumn("float", FloatType, "CAST(id AS FLOAT)", partitionable = true), + TestColumn("double", DoubleType, "CAST(id AS DOUBLE)", partitionable = true), + TestColumn( + "binary", + BinaryType, + "CAST(CONCAT('b', CAST(id AS STRING)) AS BINARY)", + partitionable = true), + TestColumn( + "arr", + ArrayType(StringType), + "array(CONCAT('a', CAST(id AS STRING)))", + partitionable = false), + TestColumn( + "nested", + StructType(Seq(StructField("inner", IntegerType))), + "named_struct('inner', CAST(id AS INT))", + partitionable = false)) + ListMap(columns.map(c => c.name -> c): _*) + } + + protected def partitionableTestColumns: Seq[TestColumn] = + allTypeColumns.values.filter(_.partitionable).toSeq + + protected def allTypeColumnDefinitions: Seq[String] = + allTypeColumns.values.map(_.columnDef("c_")).toSeq + + protected def allTypeColumnExprs: Seq[String] = + allTypeColumns.values.map(_.valueExpr).toSeq + + /** + * The value expressions, in column order, for a row of a [[withAllTypesTable]] table: the + * `d_`-prefixed all-type data columns followed by the `p_`-prefixed partition columns. Pass to + * [[appendRowsAsSeparateFiles]] to populate such a table with the default one-distinct-value-per- + * row layout. + */ + protected def allTypesTableColumnExprs: Seq[String] = + (allTypeColumns.values.toSeq ++ partitionableTestColumns).map(_.valueExpr) + + /** + * Creates an AMT table exercising every partition type (the `p_`-prefixed + * [[partitionableTestColumns]]) alongside the `d_`-prefixed data columns that content stats are + * collected on, and runs `body` with the table's [[DeltaLog]]. Populating the table is the + * caller's responsibility -- use [[appendRowsAsSeparateFiles]] with [[allTypesTableColumnExprs]] + * (or bespoke expressions), so each caller controls its own row count, packing, and null layout. + */ + protected def withAllTypesTable( + tableName: String, + maxEntriesPerLeaf: Int = entriesPerLeaf)(body: DeltaLog => Unit): Unit = { + // Every type, so the content-stats goldens cover the full set of data columns. + val dataColumns: Seq[TestColumn] = allTypeColumns.values.toSeq + withSQLConf(DeltaSQLConf.AMT_ENTRIES_PER_LEAF.key -> maxEntriesPerLeaf.toString) { + withTable(tableName) { + createAMTTable( + tableName, + checkpointInterval = Int.MaxValue, + tableSchema = + (dataColumns.map(_.columnDef("d_")) ++ partitionableTestColumns.map(_.columnDef("p_"))) + .mkString(", "), + partitionColumns = partitionableTestColumns.map("p_" + _.name)) + body(deltaLogForName(tableName)) + } + } + } + + /** + * The per-leaf cap tests use to get a deterministic, multi-entry leaf layout. + * + * A test that needs a leaf-resident entry cannot just write a file or two. For example, a back + * reference or a manifest deletion vector both need an entry that actually lives in a leaf. But a + * full rewrite spreads the live files over cap-sized partitions by hash of path and skips the + * empty ones, and if there is only a single leaf produced, that leaf is promoted to be a root, so + * a small table can end up with no leaf at all. + */ + protected val entriesPerLeaf: Int = 10 + + /** + * File count that packs into whole [[entriesPerLeaf]]-sized leaves. + */ + protected val leafPackedFiles: Int = 3 * entriesPerLeaf + + /** Session conf that puts [[entriesPerLeaf]] entries in each leaf. */ + protected def leafPackingConfs: Seq[(String, String)] = + Seq(DeltaSQLConf.AMT_ENTRIES_PER_LEAF.key -> entriesPerLeaf.toString) + + /** Leaves that `numFiles` live files pack into at [[entriesPerLeaf]] entries per leaf. */ + protected def expectedLeafCount(numFiles: Int): Int = + math.ceil(numFiles.toDouble / entriesPerLeaf).toInt + + /** + * Asserts `leaves` holds exactly the leaves `numFiles` live files pack into. Fails with the + * arithmetic spelled out, so a count mismatch reads as a packing problem rather than a bare + * number comparison. + */ + protected def assertLeafCount( + leaves: Seq[DataManifestEntry], numFiles: Int = leafPackedFiles): Unit = { + val expected = expectedLeafCount(numFiles) + assert(leaves.size == expected, + s"$numFiles files at $entriesPerLeaf per leaf must pack into $expected leaves; " + + s"got ${leaves.size}.") + } + + /** Asserts two leaf sequences are field-by-field equal. */ + protected def assertLeavesEqual( + actual: Seq[DataManifestEntry], expected: Seq[DataManifestEntry]): Unit = + AMTLeafComparisons.assertLeavesEqual(actual, expected) + /** A production-supported combination of AMT placement and materialization strategy. */ protected sealed abstract class AMTCheckpointScenario( val name: String, @@ -126,7 +344,13 @@ trait AMTCheckpointTestBase * * @param scenario the placement and materialization strategy used by the test * @param tableName the catalog-managed table created by the harness - * @param preCheckpointSnapshot the snapshot after `setup` and before the tested checkpoint + * @param postSetupSnapshot the snapshot after `setup` + * @param preCheckpointSnapshot the snapshot just before the checkpoint is materialized: for + * deferred scenarios, taken after the trigger's data commit but + * before the OPTIMIZE-CHECKPOINT commit; for inline scenarios, the + * checkpoint rides in the trigger commit, so there is no distinct + * pre-checkpoint version and this is the same as the pre-trigger + * snapshot. * @param manifestCommitVersion the commit carrying `checkpoint`; this equals * `checkpoint.version` for inline checkpoints and * `checkpoint.version + 1` for deferred checkpoints @@ -137,6 +361,7 @@ trait AMTCheckpointTestBase protected case class AMTCheckpointScenarioContext( scenario: AMTCheckpointScenario, tableName: String, + postSetupSnapshot: Snapshot, preCheckpointSnapshot: Snapshot, manifestCommitVersion: Long, checkpoint: Checkpoint, @@ -146,8 +371,8 @@ trait AMTCheckpointTestBase /** * Registers one test for each requested production-supported checkpoint scenario. * - * For each scenario, the harness creates an AMT table and a full bootstrap checkpoint, then runs - * `setup`. + * For each scenario, the harness runs `beforeTableCreation`, creates an AMT table and a full + * bootstrap checkpoint, then runs `setup`. * * - When `inlineCheckpointTriggerActionsOrSQL` is defined, every scenario executes that * table-name-aware business commit and the harness registers an @@ -168,7 +393,10 @@ trait AMTCheckpointTestBase deferredScenarios: Seq[AMTCheckpointScenario] = Seq( AMTCheckpointScenario.DeferredIncremental, AMTCheckpointScenario.DeferredFull), - sqlConfs: Seq[(String, String)] = Seq.empty)( + sqlConfs: Seq[(String, String)] = Seq.empty, + tableSchema: String = "id INT", + partitionColumns: Seq[String] = Seq.empty)( + beforeTableCreation: () => Unit = () => (), setup: String => Unit = _ => (), inlineCheckpointTriggerActionsOrSQL: Option[AMTCheckpointTrigger] = None)( body: AMTCheckpointScenarioContext => Unit): Unit = { @@ -186,7 +414,12 @@ trait AMTCheckpointTestBase val scenarioTable = s"${tableName}_${scenario.name.replace(' ', '_')}" withSQLConf(sqlConfs: _*) { withTable(scenarioTable) { - createAMTTable(scenarioTable, checkpointInterval = Int.MaxValue) + beforeTableCreation() + createAMTTable( + scenarioTable, + checkpointInterval = Int.MaxValue, + tableSchema = tableSchema, + partitionColumns = partitionColumns) // Incremental checkpoints require an existing full checkpoint. commitCheckpoint(deltaLogForName(scenarioTable), incremental = false) setup(scenarioTable) @@ -208,7 +441,7 @@ trait AMTCheckpointTestBase import AMTCheckpointScenario._ val deltaLog = deltaLogForName(tableName) - val preCheckpointSnapshot = deltaLog.update() + val postSetupSnapshot = deltaLog.update() def runCheckpointTrigger(): Unit = { inlineCheckpointTriggerActionsOrSQL.map(_(tableName)).foreach { @@ -219,19 +452,28 @@ trait AMTCheckpointTestBase } } - scenario match { + // Run the business trigger and materialize the checkpoint, capturing the snapshot just before + // the checkpoint is written. Inline scenarios write the checkpoint in the same commit that adds + // data, so there is no distinct pre-checkpoint version and this stays the pre-trigger snapshot. + // Deferred scenarios write it in a separate follow-up OPTIMIZE CHECKPOINT commit, so the + // snapshot taken after the trigger but before that commit is the table's state reconstructed + // independently of the AMT under test. + val preCheckpointSnapshot: Snapshot = scenario match { case InlineIncremental => withSQLConf( DeltaSQLConf.AMT_LARGE_COMMIT_ACTIONS_COUNT_THRESHOLD_FOR_INLINE_MANIFEST_COMMIT.key -> "1") { runCheckpointTrigger() } + postSetupSnapshot case _ => runCheckpointTrigger() + val beforeCheckpoint = deltaLog.update() commitCheckpoint(deltaLog, incremental = scenario.isIncremental) + beforeCheckpoint } - val checkpointedVersion = preCheckpointSnapshot.version + + val checkpointedVersion = postSetupSnapshot.version + (if (inlineCheckpointTriggerActionsOrSQL.isDefined) 1L else 0L) val manifestCommitVersion = checkpointedVersion + (if (scenario.isInline) 0L else 1L) val postCheckpointSnapshot = deltaLog.update() @@ -242,6 +484,7 @@ trait AMTCheckpointTestBase AMTCheckpointScenarioContext( scenario = scenario, tableName = tableName, + postSetupSnapshot = postSetupSnapshot, preCheckpointSnapshot = preCheckpointSnapshot, manifestCommitVersion = manifestCommitVersion, checkpoint = checkpoint, @@ -249,6 +492,10 @@ trait AMTCheckpointTestBase postCheckpointSnapshot = postCheckpointSnapshot) } + /** + * Emits an AMT checkpoint on `deltaLog` via the real commit path. `incremental = false` forces a + * full rewrite; `true` an incremental one. + */ protected def commitCheckpoint(deltaLog: DeltaLog, incremental: Boolean): Unit = { val triggerName = if (incremental) { AMTTriggerMode.CheckpointIntervalIncremental.name @@ -262,6 +509,55 @@ trait AMTCheckpointTestBase triggerName = triggerName)) } + /** + * Emits an incremental AMT checkpoint on `deltaLog` and returns the incremental write's shape + * metrics, read back out of the logged [[CommitStats]] (None if no incremental metrics were + * logged). Wraps the commit in usage tracking, whose buffer is process-wide, so this must not run + * concurrently with another tracked commit. + */ + protected def commitIncrementalCheckpointAndReturnMetrics( + deltaLog: DeltaLog): Option[IncrementalAMTWriteMetrics] = { + val attemptVersion = deltaLog.update().version + 1 + trackIncrementalAMTWriteMetrics(attemptVersion) { + commitCheckpoint(deltaLog, incremental = true) + } + } + + /** + * Runs `commit` and returns the [[IncrementalAMTWriteMetrics]] logged for the commit at + * `commitVersion`, or None if that commit wrote a full (non-incremental) AMT. Exposed for tests + * that commit user actions inline (their own checkpoint rides in the same commit) rather than + * through [[commitCheckpoint]]'s empty OPTIMIZE CHECKPOINT. + */ + protected def trackIncrementalAMTWriteMetrics( + commitVersion: Long)(commit: => Unit): Option[IncrementalAMTWriteMetrics] = { + Log4jUsageLogger.track { + commit + }.filter(e => e.metric == MetricDefinitions.EVENT_TAHOE.name && + e.tags.get("opType").contains("delta.commit.stats")) + .map(e => JsonUtils.fromJson[CommitStats](e.blob)) + .find(_.commitVersion == commitVersion) + .flatMap(_.amtCommitStats) + .flatMap(_.lastAMTWriteMetrics.incrementalWriteMetrics) + } + + /** + * Like [[trackIncrementalAMTWriteMetrics]] but returns one entry per attempt that made an + * incremental write, in attempt order -- a conflict retry materializes the tree more than once, + * so this exposes each attempt's shape. + */ + protected def trackIncrementalAMTWriteMetricsPerAttempt( + commit: => Unit): Seq[IncrementalAMTWriteMetrics] = { + val events = Log4jUsageLogger.track { + commit + } + events.filter(e => e.metric == MetricDefinitions.EVENT_TAHOE.name && + e.tags.get("opType").contains(AMTUsageLogs.CONFLICT_RESOLUTION_ROUND)) + .map(e => JsonUtils.fromJson[AMTMetrics](e.blob)) + .flatMap(_.singleAMTWriteMetrics) + .flatMap(_.incrementalWriteMetrics) + } + private def assertAMTCheckpointScenarioInvariants( context: AMTCheckpointScenarioContext): Unit = { val scenario = context.scenario @@ -271,7 +567,7 @@ trait AMTCheckpointTestBase val expectedLastFull = scenario match { case AMTCheckpointScenario.InlineIncremental | AMTCheckpointScenario.DeferredIncremental => - val bootstrap = amtProvider(context.preCheckpointSnapshot).getOrElse( + val bootstrap = amtProvider(context.postSetupSnapshot).getOrElse( fail("incremental scenario must bootstrap a full checkpoint")) bootstrap.checkpointAction.contentRoot.lastManifestCommitWithFullRewrite.get case AMTCheckpointScenario.DeferredFull => checkpoint.version @@ -279,6 +575,9 @@ trait AMTCheckpointTestBase assert(checkpoint.contentRoot.lastManifestCommitWithFullRewrite.contains(expectedLastFull), s"${scenario.name}: wrong last-full marker " + checkpoint.contentRoot.lastManifestCommitWithFullRewrite) + assert(checkpoint.contentRoot.version == checkpoint.version, + s"${scenario.name}: contentRoot.version ${checkpoint.contentRoot.version} must equal " + + s"checkpoint.version ${checkpoint.version}") assert(context.provider.checkpointVersion == checkpoint.version) assert(context.provider.checkpointAction == checkpoint) assert(context.postCheckpointSnapshot.version == context.manifestCommitVersion) @@ -300,30 +599,82 @@ trait AMTCheckpointTestBase } /** - * Asserts the manifest tree round-trips the live file set exactly: reconstructing from the - * checkpoint (root + leaves, minus MDV-masked entries and root tombstones) must yield precisely - * `snapshot.allFiles`, with no entry dropped or duplicated. - * - * Call this from tests that are about the tree capturing table state. It is deliberately NOT run - * for every scenario: it costs a full reconstruction scan per call, which is wasted on tests that - * assert something else (field ids, log-segment trimming, back references). + * A comparison key for an [[AddFile]] that ignores the fields which legitimately differ between a + * freshly committed AddFile and one an AMT reconstructs (via `DataEntry.toAddFile`), so two + * descriptions of the same live file compare equal: + * - `modificationTime` (not round-tripped by the AMT yet) is zeroed; + * - `backReference` and `amtPassthrough` are AMT's own auxiliary fields, not part of the + * AddFile's content, so they are zeroed rather than compared; + * - `stats` is a JSON string, compared as a parsed tree (field order is not significant); + * - the deletion vector is compared by the physical file it resolves to plus its offset/size/ + * cardinality: a committed DV is stored `storageType="r"` (the relative path) while the AMT + * re-encodes the same vector as `"u"` (the base85 UUID), and both resolve to the same file. */ - protected def assertReconstructsLiveFileSet(context: AMTCheckpointScenarioContext): Unit = { + private def canonicalAddFile(tableLocation: Path)(add: AddFile) = + add.copy( + modificationTime = 0L, + stats = null, + deletionVector = null, + backReference = None, + amtPassthrough = None) -> ( + Option(add.stats).map(JsonUtils.mapper.readTree), + Option(add.deletionVector).map { dv => + val physical = + if (dv.storageType == "i") dv.uniqueFileId + else dv.absolutePath(tableLocation).toString + (physical, dv.offset, dv.sizeInBytes, dv.cardinality, dv.maxRowIndex) + }) + + /** + * Asserts the live file set survives AMT checkpoint construction, comparing the state before the + * checkpoint against the state after. + */ + protected def assertLiveAddFilesRoundTrip(context: AMTCheckpointScenarioContext): Unit = { val snapshot = context.postCheckpointSnapshot - val committed = snapshot.allFiles.collect().map(_.path).toSet - val reconstructed = context.provider - .loadActionsForStateReconstruction(spark, snapshot.deltaLog) + val deltaLog = snapshot.deltaLog + allowReadWithinDeltaLog { + val reconstructedPaths = context.provider + .loadActionsForStateReconstruction(spark, deltaLog) .getOrElse(fail(s"${context.scenario.name}: provider must contribute file actions.")) .where(col("add").isNotNull) .select("add.path") .collect() .map(_.getString(0)) - assert(reconstructed.length == reconstructed.toSet.size, + assert(reconstructedPaths.length == reconstructedPaths.toSet.size, s"${context.scenario.name}: reconstruction must not duplicate entries; got " + - s"${reconstructed.toSeq.diff(reconstructed.distinct.toSeq)}") - assert(reconstructed.toSet == committed, - s"${context.scenario.name}: file set changed: committed=$committed " + - s"reconstructed=${reconstructed.toSet}") + s"${reconstructedPaths.toSeq.diff(reconstructedPaths.distinct.toSeq)}") + } + + // Inline checkpoints ride in the trigger commit, so there is no distinct pre-checkpoint + // snapshot to compare against; the before/after round-trip covers only the deferred scenarios, + // whose checkpoint is a separate no-data commit after the trigger. + if (!context.scenario.isInline) { + val canonical = canonicalAddFile(deltaLog.dataPath) _ + val expected = context.preCheckpointSnapshot.allFiles.collect().map(canonical).toSet + val reconstructed = snapshot.allFiles.collect().map(canonical).toSet + assert(reconstructed == expected, + s"${context.scenario.name}: live AddFiles changed across AMT checkpoint construction\n" + + s" before=$expected\n after=$reconstructed") + } + } + + /** Forces every write to inline its AMT incrementally (a low action-count threshold). */ + protected def withInline[T](body: => T): T = withInlineThreshold(1)(body) + + /** Runs `body` with the inline-manifest threshold at `n` actions. */ + protected def withInlineThreshold[T](n: Int)(body: => T): T = + withSQLConf( + DeltaSQLConf.AMT_LARGE_COMMIT_ACTIONS_COUNT_THRESHOLD_FOR_INLINE_MANIFEST_COMMIT.key + -> n.toString)(body) + + /** + * Runs the test with inline writes forced (a low action-count threshold). + * AMT checkpoints will be emitted in every commit after the first full OPTIMIZE CHECKPOINT. + */ + protected def testInline(testName: String)(body: => Unit): Unit = { + test(s"$testName (inline)") { + withInline { body } + } } /** True iff `name` looks like an AMT leaf parquet file. */ @@ -348,8 +699,33 @@ trait AMTCheckpointTestBase metadataFiles(path, isRootFileName) /** Returns the actions committed at exactly `version`. */ - protected def actionsAt(deltaLog: DeltaLog, version: Long): Seq[Action] = + protected def actionsAt(deltaLog: DeltaLog, version: Long): Seq[Action] = { + // TODO: Restore this call once `CoordinatedCommitsUtils.commitFilesIterator` discovers + // unbackfilled manifest commits. + /* deltaLog.getChanges(version).find(_._1 == version).map(_._2).getOrElse(Seq.empty) + */ + val snapshot = deltaLog.update() + val path = DeltaCommitFileProvider(snapshot).deltaFile(version) + deltaLog.store.read(path, deltaLog.newDeltaHadoopConf()).map(Action.fromJson) + } + + /** + * The live `AddFile` set reconstructed from the commit-log deltas alone (not the AMT tree). An + * AMT-backed table reconstructs its on-disk DVs as relative `r` descriptors through `allFiles`, + * so a test that needs the DV descriptors a commit actually recorded reads them from the log. + */ + protected def liveAddFilesFromLog(deltaLog: DeltaLog): Seq[AddFile] = { + val live = scala.collection.mutable.LinkedHashMap.empty[String, AddFile] + deltaLog.getChanges(0).foreach { case (_, actions) => + actions.foreach { + case a: AddFile => live(a.path) = a + case r: RemoveFile => live.remove(r.path) + case _ => + } + } + live.values.toSeq + } /** The [[Checkpoint]] committed at exactly `version`, if any. */ protected def checkpointAt(deltaLog: DeltaLog, version: Long): Option[Checkpoint] = { @@ -368,16 +744,83 @@ trait AMTCheckpointTestBase * does it count live files stored directly in the root. On an incremental tree a deleted file's * entry stays physically present in its carried-forward leaf (tombstoned via the leaf MDV), so * this can exceed the live file count. To assert the tree captures exactly the live file set, - * call [[assertReconstructsLiveFileSet]] instead. + * call [[assertLiveAddFilesRoundTrip]] instead. + */ + protected def leafLiveDataEntryCount(snapshot: Snapshot): Long = + leafLiveDataEntryCount(amtProvider(snapshot) + .getOrElse(fail("Snapshot has no AMTCheckpointProvider."))) + + /** + * Total DATA (content_type=0) entry rows across `provider`'s live leaves, 0 when the tree is + * leafless. Like the [[Snapshot]]-based overload but driven by a provider a test built directly + * (e.g. from a checkpoint), where no owning [[Snapshot]] exists. + */ + protected def leafLiveDataEntryCount(provider: AMTCheckpointProvider): Long = { + val leaves = provider.liveLeafManifestAbsolutePaths.map(_.toString) + if (leaves.isEmpty) 0L else withManifestDataEntries(leaves)(_.count()) + } + + + /** + * The number of live files the CURRENT snapshot's AMT reconstructs across the WHOLE tree -- root + * and leaves. Goes through the provider's own reconstruction, which drops MDV-masked leaf entries + * and `tracking=removed` root tombstones, so it equals `snapshot.allFiles.count()` on both full + * and incremental trees. Unlike [[leafLiveDataEntryCount]], it counts live files stored directly + * in the root too (as an incremental commit does below the spill threshold). + * + * Prefer [[assertLiveAddFilesRoundTrip]] when the test runs through + * [[testAcrossAMTCheckpointScenarios]]; this count is for scenario-specific tests that drive the + * checkpoint themselves and so have no [[AMTCheckpointScenarioContext]]. */ - protected def currentLeafDataEntries(snapshot: Snapshot): Long = { + protected def currentLiveDataEntries(snapshot: Snapshot): Long = { val provider = amtProvider(snapshot) .getOrElse(fail("Snapshot has no AMTCheckpointProvider.")) - provider.leafManifestAbsolutePaths.map { leafPath => - spark.read.parquet(leafPath.toString) - .where(col("content_type") === AMTSingleAction.ContentType.Type.Data) - .count() - }.sum + provider.loadActionsForStateReconstruction(spark, snapshot.deltaLog) + .getOrElse(fail("AMT provider must contribute leaf-derived file actions.")) + .where(col("add").isNotNull) + .count() } } + +/** + * Field-by-field equality for AMT leaf pointers, shared across suites (some of which do not extend + * [[AMTCheckpointTestBase]]). Compares by content rather than case-class `==` (which compares the + * `Array[Byte]` fields by reference) and rather than a json comparison (tests of serialization must + * not rely on serialization to check their result). + */ +object AMTLeafComparisons { + /** Value equality for `Option[Array[Byte]]` (case-class `==` compares arrays by reference). */ + private def sameBytes(a: Option[Array[Byte]], b: Option[Array[Byte]]): Boolean = + (a, b) match { + case (Some(x), Some(y)) => x.sameElements(y) + case (None, None) => true + case _ => false + } + + /** Asserts two leaf pointers are field-by-field equal, comparing byte-array fields by content. */ + def assertLeafEquals(actual: DataManifestEntry, expected: DataManifestEntry): Unit = { + // Blank the array-bearing members so a single `==` covers every other field structurally. + def blanked(e: DataManifestEntry): DataManifestEntry = e.copy( + tracking = e.tracking.copy(deleted_positions = None, replaced_positions = None), + manifest_info = e.manifest_info.copy(dv = None), + content_stats = None, + key_metadata = None) + assert(blanked(actual) == blanked(expected), + s"leaf non-array fields differ: $actual vs $expected") + assert(sameBytes(actual.tracking.deleted_positions, expected.tracking.deleted_positions), + "tracking.deleted_positions differ") + assert(sameBytes(actual.tracking.replaced_positions, expected.tracking.replaced_positions), + "tracking.replaced_positions differ") + assert(sameBytes(actual.manifest_info.dv, expected.manifest_info.dv), "manifest_info.dv differ") + assert(sameBytes(actual.key_metadata, expected.key_metadata), "key_metadata differ") + assert(actual.content_stats == expected.content_stats, "content_stats differ") + } + + /** Asserts two leaf sequences are field-by-field equal (see [[assertLeafEquals]]). */ + def assertLeavesEqual( + actual: Seq[DataManifestEntry], expected: Seq[DataManifestEntry]): Unit = { + assert(actual.size == expected.size, s"leaf count differs: ${actual.size} vs ${expected.size}") + actual.zip(expected).foreach { case (a, e) => assertLeafEquals(a, e) } + } +} diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTCheckpointWriteSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTCheckpointWriteSuite.scala index a4c1f0efe21..5abe60ab973 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTCheckpointWriteSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTCheckpointWriteSuite.scala @@ -18,100 +18,297 @@ package org.apache.spark.sql.delta.amt import java.io.File -// scalastyle:off import.ordering.noEmptyLine import com.databricks.spark.util.{Log4jUsageLogger, MetricDefinitions} -import org.apache.spark.sql.delta.{Checkpoints, CommitStats, CurrentTransactionInfo, DeltaOperations, Snapshot} -import org.apache.spark.sql.delta.actions.{AddFile, Checkpoint, ContentRoot, RemoveFile} +import org.apache.spark.sql.delta.{Checkpoints, CommitStats, DeltaOperations, LastCheckpointInfo, RowId} +import org.apache.spark.sql.delta.actions.{AddFile, Checkpoint, ContentRoot} +import org.apache.spark.sql.delta.coordinatedcommits.TrackingInMemoryCommitCoordinatorBuilder import org.apache.spark.sql.delta.sources.DeltaSQLConf -import org.apache.spark.sql.delta.util.{FileNames, JsonUtils} +import org.apache.spark.sql.delta.util.{DeltaCommitFileProvider, FileNames, JsonUtils} +import io.delta.storage.commit.CoordinatedCommitsUtils.getBackfilledDeltaFilePath import org.apache.hadoop.fs.Path import org.apache.spark.sql.Row +import org.apache.spark.sql.execution.datasources.parquet.ParquetUtils import org.apache.spark.sql.functions.col +import org.apache.spark.sql.types.StructType class AMTCheckpointWriteSuite extends AMTCheckpointTestBase { import testImplicits._ - testAcrossAMTCheckpointScenarios( - "checkpoint emission writes a manifest tree and carries the user action once", - "amt_emit")( - setup = name => sql(s"INSERT INTO $name VALUES (1)"), - inlineCheckpointTriggerActionsOrSQL = Some(_ => Left(( - Seq(AddFile("missing-file.parquet", Map.empty, 0L, 0L, dataChange = true) - .removeWithTimestamp(1L)), - DeltaOperations.ManualUpdate)))) { context => - val path = tablePath(context.tableName) - val actionsFromCheckpointedCommit = - actionsAt(context.postCheckpointSnapshot.deltaLog, context.checkpoint.version) - assert(actionsFromCheckpointedCommit.exists { - case _: AddFile | _: RemoveFile => true - case _ => false - }, "The described business commit must carry the user file action.") - - val rootName = new File(context.checkpoint.contentRoot.path).getName - assert(isRootFileName(rootName), - s"contentRoot must point at a root manifest file; got " + - context.checkpoint.contentRoot.path) - assert(rootFiles(path).exists(_.getName == rootName)) - - // The emitted tree must reconstruct exactly the table's live files. - assertReconstructsLiveFileSet(context) - } + /** + * The nested `(name, type, field id)` shape of a persisted struct, recursing into sub-structs so + * per-field Iceberg ids are covered. Comparing an on-disk struct's shape to the schema + * `AMTPartitionValues`/`AMTContentStats` declare verifies the writer stamped exactly the declared + * names, types and ids. + */ + private def fieldIdShape(schema: StructType): Seq[(String, String, Long)] = + schema.fields.toSeq.flatMap { field => + val id = field.metadata.getLong(ParquetUtils.FIELD_ID_METADATA_KEY) + field.dataType match { + case nested: StructType => (field.name, "struct", id) +: fieldIdShape(nested) + case dataType => Seq((field.name, dataType.sql, id)) + } + } - test("manifest pointers are stored relative to the table root") { - withTable("amt_relative_pointers") { - val name = "amt_relative_pointers" + test("interval boundary emits a follow-up OPTIMIZE CHECKPOINT commit carrying the Checkpoint") { + withTable("amt_inline_emit") { + val name = "amt_inline_emit" createAMTTable(name, checkpointInterval = 2) - withSQLConf(DeltaSQLConf.AMT_ENTRIES_PER_LEAF.key -> "1") { - sql(s"INSERT INTO $name VALUES (1)") // v1: one data file. - sql(s"INSERT INTO $name VALUES (2)") // v2: interval boundary -> triggers AMT emission. - } + sql(s"INSERT INTO $name VALUES (1)") // v1: not an interval boundary. + sql(s"INSERT INTO $name VALUES (2)") // v2: interval boundary -> schedule maintenance. val deltaLog = deltaLogForName(name) + val path = tablePath(name) val snapshot = deltaLog.update() - val provider = amtProvider(snapshot).getOrElse(fail("expected AMTCheckpointProvider")) - - // The Checkpoint's contentRoot pointer is stored relative to the table root, i.e. - // `metadata/root-.parquet`, not an absolute URI. - val rootPointer = provider.checkpointAction.contentRoot.path - assert(rootPointer == s"${FileNames.AMT_METADATA_DIR_NAME}/${new File(rootPointer).getName}", - s"contentRoot.path must be table-root-relative; got $rootPointer") - assert(!new Path(rootPointer).isAbsolute, - s"contentRoot.path must not be absolute; got $rootPointer") - assert(isRootFileName(new File(rootPointer).getName)) - - // Every leaf pointer stored in the root manifest is likewise table-root-relative. - val leafLocations = { - spark.read.parquet(new File(new File(tablePath(name), - FileNames.AMT_METADATA_DIR_NAME), new File(rootPointer).getName).toString) - .where(col("content_type") === AMTSingleAction.ContentType.Type.DataManifest) - .select("location").as[String].collect().toSeq - } - // The clustered writer decides leaf count by partitioning, so assert at least one leaf - // pointer rather than an exact count; every pointer must be table-root-relative. - assert(leafLocations.nonEmpty, s"Expected at least one leaf pointer, got $leafLocations") - leafLocations.foreach { loc => - assert(loc == s"${FileNames.AMT_METADATA_DIR_NAME}/${new File(loc).getName}", - s"leaf pointer must be table-root-relative; got $loc") - assert(isLeafFileName(new File(loc).getName)) + // The AMT is written by a follow-up OPTIMIZE CHECKPOINT commit at v3, not inline at v2. + assert(snapshot.version == 3, "A follow-up OPTIMIZE CHECKPOINT commit lands at v3.") + + // Note the leaf may be promoted to root -- in that case no file with root naming pattern + // will exist. + assert(rootFiles(path).size + leafFiles(path).size > 0, + "At least one root/leaf should be written") + + // The v2 commit carries only the user AddFile, no Checkpoint. + val v2Actions = actionsAt(deltaLog, 2) + assert(v2Actions.exists(_.isInstanceOf[AddFile]), "v2 carries the user AddFile.") + assert(v2Actions.collect { case c: Checkpoint => c }.isEmpty, + s"v2 must not carry a Checkpoint action; got: $v2Actions") + + // The v3 follow-up commit carries a single Checkpoint action and no user AddFile. + val v3Actions = actionsAt(deltaLog, 3) + val checkpoints = v3Actions.collect { case c: Checkpoint => c } + assert(checkpoints.size == 1, s"Expected one Checkpoint action at v3, got: $v3Actions") + assert(!v3Actions.exists(_.isInstanceOf[AddFile]), "v3 carries no user AddFile.") + // The Checkpoint describes state as of v2 (the version whose maintenance it fulfills). + assert(checkpoints.head.version == 2, + s"Checkpoint must describe state as of v2; got ${checkpoints.head.version}") + assert(checkpoints.head.contentRoot.version == 2, + s"contentRoot.version must be v2; got ${checkpoints.head.contentRoot.version}") + + // The Checkpoint's contentRoot points at an on-disk manifest file. A single promoted leaf + // keeps its leaf-* name, so accept either a root or leaf manifest. + val rootName = new File(checkpoints.head.contentRoot.path).getName + assert(isRootFileName(rootName) || isLeafFileName(rootName), + "contentRoot must follow the given naming pattern") + assert((rootFiles(path) ++ leafFiles(path)).exists(_.getName == rootName), + s"contentRoot must reference an on-disk manifest; got $rootName") + } + } + + testAcrossAMTCheckpointScenarios( + "checkpoint backfills preceding commits while its manifest commit remains staged", + "amt_checkpoint_backfill")( + beforeTableCreation = () => { + // Keep periodic coordinator backfill from publishing any of these commits. + clearBuilders() + registerBuilder(TrackingInMemoryCommitCoordinatorBuilder(batchSize = 100)) + }, + setup = name => { + (2 to 3).foreach(i => sql(s"INSERT INTO $name VALUES ($i)")) + val deltaLog = deltaLogForName(name) + val snapshot = deltaLog.update() + assert(snapshot.version == 3L) + val commitFileProvider = DeltaCommitFileProvider(snapshot) + val fs = deltaLog.logPath.getFileSystem(deltaLog.newDeltaHadoopConf()) + // The bootstrap manifest and both INSERTs must still need backfilling. + (1L to snapshot.version).foreach { version => + val commitFile = commitFileProvider.deltaFile(version) + assert(FileNames.isUnbackfilledDeltaFile(commitFile)) + assert(fs.exists(commitFile)) + assert(!fs.exists(getBackfilledDeltaFilePath(deltaLog.logPath, version)), + s"v$version must not be backfilled before the tested checkpoint") + } + }, + inlineCheckpointTriggerActionsOrSQL = Some(name => Right( + s"INSERT INTO $name VALUES (3)"))) { context => + val snapshot = context.postCheckpointSnapshot + val deltaLog = snapshot.deltaLog + val manifestCommitVersion = context.manifestCommitVersion + val commitFileProvider = DeltaCommitFileProvider(snapshot) + val fs = deltaLog.logPath.getFileSystem(deltaLog.newDeltaHadoopConf()) + + (0L until manifestCommitVersion).foreach { version => + assert(fs.exists(getBackfilledDeltaFilePath(deltaLog.logPath, version)), + s"v$version must be backfilled before manifest commit v$manifestCommitVersion") + } + val manifestCommitFile = commitFileProvider.deltaFile(manifestCommitVersion) + assert(FileNames.isUnbackfilledDeltaFile(manifestCommitFile)) + assert(fs.exists(manifestCommitFile)) + assert(!fs.exists(getBackfilledDeltaFilePath(deltaLog.logPath, manifestCommitVersion)), + s"Manifest commit v$manifestCommitVersion must remain unbackfilled") + } + + testAcrossAMTCheckpointScenarios( + "small full checkpoint promotes its single distributed manifest to the root", + "amt_root_only", + deferredScenarios = Seq(AMTCheckpointScenario.DeferredFull))( + setup = name => { + sql(s"INSERT INTO $name VALUES (1)") + sql(s"INSERT INTO $name VALUES (2)") + }) { context => + val rootDataEntries = withManifestDataEntries( + Seq(context.checkpoint.contentRoot.getAbsolutePath(context.provider.tableRoot).toString) + )(_.count()) + + assert(context.provider.leaves.isEmpty, "The root must contain no leaf pointers.") + assert(rootDataEntries == 2L, "Both live files must be reachable as DATA entries in the root.") + + // The promoted root alone must reconstruct exactly the table's live files. + assertLiveAddFilesRoundTrip(context) + } + + testAcrossAMTCheckpointScenarios( + "partition values use the typed Iceberg struct and round-trip", + "amt_typed_partition", + sqlConfs = leafPackingConfs, + tableSchema = + ("id INT" +: partitionableTestColumns.map(_.columnDef("p_"))).mkString(", "), + partitionColumns = partitionableTestColumns.map("p_" + _.name))( + setup = name => appendRowsAsSeparateFiles( + name, + numFiles = leafPackedFiles - 1, + columnExprs = "CAST(id AS INT)" +: partitionableTestColumns.map(_.valueExpr)), + inlineCheckpointTriggerActionsOrSQL = Some { name => + val cols = ("CAST(id AS INT)" +: partitionableTestColumns.map(_.valueExpr)).mkString(", ") + Right( + s"""INSERT INTO $name + |SELECT $cols + |FROM range(${leafPackedFiles - 1}, $leafPackedFiles)""".stripMargin) + }) { context => + val manifests = context.provider.topLevelFiles.map(_.getPath.toString) ++ + context.provider.liveLeafManifestAbsolutePaths.map(_.toString) + val partitionSchema = context.postCheckpointSnapshot.metadata.partitionSchema + val (partition, pIntValues) = allowReadWithinDeltaLog { + val manifestDf = spark.read.parquet(manifests: _*) + val dataValues = manifestDf + .where(col("content_type") === AMTSingleAction.ContentType.Type.Data) + .select("partition.p_int").collect().map(_.getInt(0)).toSet + (manifestDf.schema("partition"), dataValues) + } + assert(partition.nullable) + val expectedSchema = AMTPartitionValues.persistedSchema(partitionSchema) + assert( + fieldIdShape(partition.dataType.asInstanceOf[StructType]) == fieldIdShape(expectedSchema), + s"persisted partition schema did not match the expected typed shape\n" + + s" actual=${partition.dataType}\n expected=$expectedSchema") + assert(pIntValues == (0 until leafPackedFiles).toSet) + + // Every physical-name -> string partition entry must come back exactly as the log recorded it; + // a cast that disagrees with Delta's own serialization would corrupt these silently. + assertLiveAddFilesRoundTrip(context) + } + + testAcrossAMTCheckpointScenarios( + "an unpartitioned table writes no partition column at all", + "amt_unpartitioned", + sqlConfs = leafPackingConfs)( + setup = name => appendRowsAsSeparateFiles(name, numFiles = leafPackedFiles - 1), + inlineCheckpointTriggerActionsOrSQL = Some(name => Right( + s"INSERT INTO $name VALUES (${leafPackedFiles - 1})"))) { context => + // An unpartitioned table has no partition struct to persist, so `partition` is dropped from the + // schema rather than written as a null -- Iceberg field 102 is optional. Checking the parquet + // itself, since the reconstructed AddFile carries a `partition` either way (`forRead` adds one + // back as a null map). + val manifests = + (context.provider.topLevelFiles.map(_.getPath.toString) ++ + context.provider.liveLeafManifestAbsolutePaths.map(_.toString)) + assert(manifests.nonEmpty, "Expected at least a root manifest.") + manifests.foreach { manifest => + val columns = allowReadWithinDeltaLog { + spark.read.parquet(manifest).columns.toSeq } + assert(!columns.contains("partition"), + s"an unpartitioned manifest must have no `partition` column; $manifest has $columns") + } + } - // The pointers are stored relative on disk; the provider re-absolutizes them via - // `leafManifestAbsolutePaths`, and reconstruction driven off the manifest tree (root + - // leaves, both stored relative) surfaces exactly the two committed data files. - val leafLocs = provider.leaves.map(_.location) - assert(leafLocs.forall(loc => - loc == s"${FileNames.AMT_METADATA_DIR_NAME}/${new File(loc).getName}"), - s"leaf pointer locations must be table-root-relative; got $leafLocs") - assert(provider.leafManifestAbsolutePaths.forall(_.isAbsolute), - s"resolved leaf manifest paths must be absolute; got ${provider.leafManifestAbsolutePaths}") - val reconstructed = provider.loadActionsForStateReconstruction(spark, deltaLog) - .getOrElse(fail("AMT provider must contribute reconstructed actions.")) - assert(reconstructed.where("add is not null").count() == 2, - "Reconstruction from the relative-pointer manifest tree must surface both committed files.") - checkAnswer(spark.table(name), Seq(Row(1), Row(2))) + testAcrossAMTCheckpointScenarios( + "typed content stats round-trip through reconstruction", + "amt_typed_content_stats", + tableSchema = allTypeColumnDefinitions.mkString(", "))( + setup = name => + appendRowsAsSeparateFiles(name, numFiles = 2, columnExprs = allTypeColumnExprs), + inlineCheckpointTriggerActionsOrSQL = Some(name => Right( + s"INSERT INTO $name SELECT ${allTypeColumnExprs.mkString(", ")} FROM range(2, 3)"))) { + context => + val snapshot = context.postCheckpointSnapshot + + val manifest = context.provider.topLevelFiles.map(_.getPath.toString).head + val contentStats = allowReadWithinDeltaLog { + spark.read.parquet(manifest).schema("content_stats") + } + assert(contentStats.nullable) + val expectedSchema = AMTContentStats.persistedSchema(snapshot.metadata, snapshot.protocol) + assert( + fieldIdShape(contentStats.dataType.asInstanceOf[StructType]) == fieldIdShape(expectedSchema), + s"persisted content_stats schema did not match the expected typed shape\n" + + s" actual=${contentStats.dataType}\n expected=$expectedSchema") + + // The typed content stats must round-trip: the AddFiles reconstructed from the checkpoint must + // carry the same stats Delta originally wrote. + assertLiveAddFilesRoundTrip(context) + } + + testAcrossAMTCheckpointScenarios( + "manifest pointers are stored relative to the table root", + "amt_relative_pointers", + // The leaf packing keeps several leaf pointers in the root to check, rather than the single + // manifest a small table would promote. + sqlConfs = leafPackingConfs)( + setup = name => appendRowsAsSeparateFiles(name, numFiles = leafPackedFiles - 1), + inlineCheckpointTriggerActionsOrSQL = Some(name => Right( + s"INSERT INTO $name VALUES (${leafPackedFiles - 1})"))) { context => + val name = context.tableName + val provider = context.provider + + // The Checkpoint's contentRoot pointer is stored relative to the table root, i.e. + // `metadata/root-.parquet`, not an absolute URI. + val rootPointer = provider.checkpointAction.contentRoot.path + assert(rootPointer == s"${FileNames.AMT_METADATA_DIR_NAME}/${new File(rootPointer).getName}", + s"contentRoot.path must be table-root-relative; got $rootPointer") + assert(!new Path(rootPointer).isAbsolute, + s"contentRoot.path must not be absolute; got $rootPointer") + assert(isRootFileName(new File(rootPointer).getName)) + + // Every leaf pointer stored in the root manifest is likewise table-root-relative. + val leafLocations = allowReadWithinDeltaLog { + spark.read.parquet(new File(new File(tablePath(name), + FileNames.AMT_METADATA_DIR_NAME), new File(rootPointer).getName).toString) + .where(col("content_type") === AMTSingleAction.ContentType.Type.DataManifest) + .select("location").as[String].collect().toSeq } + // A full rewrite hash-distributes the files across ceil(numFiles / entriesPerLeaf) writer + // partitions and skips any empty ones, so the pointer count is a data-dependent upper bound + // for the full rewrite; the driver-side incremental spill packs into exactly that many. + if (context.scenario.isIncremental) { + assert(leafLocations.size == expectedLeafCount(leafPackedFiles), + s"$leafPackedFiles files at $entriesPerLeaf per leaf must yield " + + s"${expectedLeafCount(leafPackedFiles)} leaf pointers; got $leafLocations") + } else { + assert(leafLocations.size <= expectedLeafCount(leafPackedFiles), + s"$leafPackedFiles files at $entriesPerLeaf per leaf must yield at most " + + s"${expectedLeafCount(leafPackedFiles)} leaf pointers; got $leafLocations") + } + leafLocations.foreach { loc => + assert(loc == s"${FileNames.AMT_METADATA_DIR_NAME}/${new File(loc).getName}", + s"leaf pointer must be table-root-relative; got $loc") + assert(isLeafFileName(new File(loc).getName)) + } + + // The pointers are stored relative on disk; the provider re-absolutizes them via + // `liveLeafManifestAbsolutePaths`, and reconstruction driven off the manifest tree (root + + // leaves, both stored relative) surfaces exactly the committed data files. + val leafLocs = provider.leaves.map(_.location) + assert(leafLocs.forall(loc => + loc == s"${FileNames.AMT_METADATA_DIR_NAME}/${new File(loc).getName}"), + s"leaf pointer locations must be table-root-relative; got $leafLocs") + assert(provider.liveLeafManifestAbsolutePaths.forall(_.isAbsolute), + "resolved leaf manifest paths must be absolute; got " + + s"${provider.liveLeafManifestAbsolutePaths}") + assertLiveAddFilesRoundTrip(context) + // `setup` writes one file per id, and the trigger adds the last one. + checkAnswer(spark.table(name), (0 until leafPackedFiles).map(Row(_))) } test("manifest tree round-trips when the table root contains spaces") { @@ -145,7 +342,7 @@ class AMTCheckpointWriteSuite extends AMTCheckpointTestBase { s"contentRoot.path must be table-root-relative; got $rootPointer") // The provider re-absolutizes leaf pointers to paths that live under the spaced root and // are not percent-encoded. - provider.leafManifestAbsolutePaths.foreach { leafPath => + provider.liveLeafManifestAbsolutePaths.foreach { leafPath => assert(leafPath.isAbsolute && leafPath.toString.contains("amt table with spaces"), s"resolved leaf path must live under the spaced table root; got $leafPath") assert(!leafPath.toString.contains("%20"), @@ -182,10 +379,20 @@ class AMTCheckpointWriteSuite extends AMTCheckpointTestBase { // contain spaces, storing raw table-root-relative pointers, to prove the pointer round-trips. def writeManifest(fileName: String, rows: Seq[AMTSingleAction]): (String, Long) = { val file = new Path(metadataDir, fileName) - val df = spark.createDataset(rows)(enc).toDF() - Checkpoints.writeAtomicCheckpointParquetFile(spark, df, file, hadoopConf, useRename = false) - val relative = AMTUtils.relativizeManifestPathToTableRoot( - file.getFileSystem(hadoopConf), dataPath, file) + val metadata = base.metaData + val protocol = base.protocol + val withPartition = AMTPartitionValues.forWrite( + spark.createDataset(rows)(enc).toDF(), metadata.partitionSchema) + val df = AMTContentStats.forWrite(withPartition, metadata, protocol) + Checkpoints.writeAtomicCheckpointParquetFile( + spark, + df, + file, + hadoopConf, + useRename = false, + outputSchema = Some(AMTSingleAction.persistedSchema(metadata, protocol)), + format = new AMTParquetFileFormat) + val relative = AMTUtils.relativizeLocation(dataPath.toString, file.toString) assert(relative == s"${FileNames.AMT_METADATA_DIR_NAME}/$fileName" && !relative.contains("%20"), s"stored pointer must be raw and table-root-relative; got $relative") @@ -197,26 +404,32 @@ class AMTCheckpointWriteSuite extends AMTCheckpointTestBase { val (leafLoc, leafSize) = writeManifest( "leaf with space.parquet", Seq(AMTSingleAction.fromAddFile(dataAdd, addedTracking, dataPath))) + val parentTracking = addedTracking.copy( + sequence_number = Some(base.version), + file_sequence_number = Some(base.version), + first_row_id = Some(0L)) val leafPointer = DataManifestEntry( location = leafLoc, file_format = AMTSingleAction.FileFormatParquet, - tracking = addedTracking, + tracking = parentTracking, record_count = 1L, file_size_in_bytes = leafSize, manifest_info = emptyManifestInfo.copy(added_files_count = 1)) val (rootLoc, rootSize) = writeManifest("root with space.parquet", Seq(leafPointer.wrap)) - val checkpoint = base.copy(contentRoot = ContentRoot(path = rootLoc, sizeInBytes = rootSize)) + val checkpoint = base.copy( + contentRoot = ContentRoot(path = rootLoc, sizeInBytes = rootSize, version = base.version)) // The synthesized pointers really do carry spaces and are not URL-encoded. assert(rootLoc.contains("root with space.parquet") && !rootLoc.contains("%20")) assert(leafLoc.contains("leaf with space.parquet") && !leafLoc.contains("%20")) - val provider = AMTCheckpointProvider.fromCheckpoint(spark, deltaLog, checkpoint) + val provider = AMTCheckpointProvider.fromCheckpoint( + deltaLog, checkpoint, manifestCommitVersion = checkpoint.version) // The provider resolves the spaced pointers to absolute, raw paths under the table root. - assert(provider.leafManifestAbsolutePaths.forall(p => + assert(provider.liveLeafManifestAbsolutePaths.forall(p => p.isAbsolute && p.toString.contains("leaf with space.parquet") && !p.toString.contains("%20")), - s"resolved leaf paths must stay raw; got ${provider.leafManifestAbsolutePaths}") + s"resolved leaf paths must stay raw; got ${provider.liveLeafManifestAbsolutePaths}") // Reconstruction reads the DATA entry back through the spaced leaf path. val reconstructed = provider.loadActionsForStateReconstruction(spark, deltaLog) @@ -227,6 +440,81 @@ class AMTCheckpointWriteSuite extends AMTCheckpointTestBase { } } + private val eAcute: Char = 0xE9.toChar + + testAcrossAMTCheckpointScenarios( + "AddFile and DataEntry conversions use correct encoding of Delta and Iceberg specs", + "amt_data_entry_raw_location")( + setup = name => { + val deltaLog = deltaLogForName(name) + val inTableAbsolute = + deltaLog.dataPath.toUri.toString + s"/dir/part%20with%25percent(3)${eAcute}.parquet" + deltaLog.startTransaction().commit( + Seq( + // Relative. + AddFile( + path = s"dir/part%20with%25percent(1)${eAcute}.parquet", + partitionValues = Map.empty, + size = 128L, + modificationTime = 0L, + dataChange = true, + stats = s"""{"numRecords":1}"""), + // Out-of-root absolute. + AddFile( + path = s"file:/other-root/table/part%20with%25percent(2)${eAcute}.parquet", + partitionValues = Map.empty, + size = 128L, + modificationTime = 0L, + dataChange = true, + stats = s"""{"numRecords":1}"""), + // In table root absolute. + AddFile( + path = inTableAbsolute, + partitionValues = Map.empty, + size = 128L, + modificationTime = 0L, + dataChange = true, + stats = s"""{"numRecords":1}""")), + DeltaOperations.ManualUpdate) + }) { context => + // According to RFC-2396: + // %20: whitespace + // %25: percent sign + // (: unreserved in RFC-2396 (in contrast to RFC-3986), left literal + // ): same above + // e-acute(U+00E9): non-ASCII, left literal + val expectedRawLocations = Set( + s"dir/part with%percent(1)${eAcute}.parquet", + s"file:/other-root/table/part with%percent(2)${eAcute}.parquet", + s"dir/part with%percent(3)${eAcute}.parquet") + val expectedDeltaPaths = Set( + s"dir/part%20with%25percent(1)${eAcute}.parquet", + s"file:/other-root/table/part%20with%25percent(2)${eAcute}.parquet", + s"dir/part%20with%25percent(3)${eAcute}.parquet") + val rootPath = context.checkpoint.contentRoot.getAbsolutePath(context.provider.tableRoot) + val dataLocations = allowReadWithinDeltaLog { + spark.read.parquet(rootPath.toString) + .where(col("content_type") === AMTSingleAction.ContentType.Type.Data) + .select("location") + .as[String] + .collect() + .toSet + } + assert(dataLocations == expectedRawLocations, + s"AMT DATA locations must be raw; got $dataLocations.") + + val reconstructedPaths = context.provider + .loadActionsForStateReconstruction(spark, context.postCheckpointSnapshot.deltaLog) + .getOrElse(fail("AMT provider must contribute reconstructed actions.")) + .where("add is not null") + .select("add.path") + .as[String] + .collect() + .toSet + assert(reconstructedPaths == expectedDeltaPaths, + s"Reconstructed AddFile paths must stay Delta-encoded; got $reconstructedPaths.") + } + test("no emission on a vanilla (non-AMT) table") { withTable("amt_vanilla") { val name = "amt_vanilla" @@ -245,6 +533,59 @@ class AMTCheckpointWriteSuite extends AMTCheckpointTestBase { } } + /** Classic V1/V2 checkpoint parquet files under the table's `_delta_log`. */ + private def classicCheckpointFiles(tableName: String): Seq[File] = { + val logDir = new File(tablePath(tableName), "_delta_log") + if (!logDir.exists()) Seq.empty + else Option(logDir.listFiles()).toSeq.flatten + .filter(f => f.getName.contains(".checkpoint") && f.getName.endsWith(".parquet")) + } + + test("deltaLog.checkpoint emits an AMT instead of writing classic checkpoint files") { + withTable("amt_checkpoint_api") { + val name = "amt_checkpoint_api" + // Interval far away so no automatic emission fires; the explicit checkpoint() call below is + // the only thing that can emit. + createAMTTable(name, checkpointInterval = 100) + sql(s"INSERT INTO $name VALUES (1)") // v1. + sql(s"INSERT INTO $name VALUES (2)") // v2. + + val deltaLog = deltaLogForName(name) + assert(amtProvider(deltaLog.update()).isEmpty, "No AMT should exist before checkpoint().") + + // On an AMT table, DeltaLog.checkpoint rewrites the manifest tree via a follow-up OPTIMIZE + // CHECKPOINT commit rather than writing classic checkpoint files. + deltaLog.checkpoint(deltaLog.update()) + + val snapshot = deltaLog.update() + assert(snapshot.version == 3, "checkpoint() must emit a follow-up commit at v3.") + val v3Checkpoint = checkpointAt(deltaLog, 3).getOrElse(fail("Expected a Checkpoint at v3.")) + assert(v3Checkpoint.version == 2, + s"The Checkpoint must describe state as of v2; got ${v3Checkpoint.version}.") + assert(amtProvider(snapshot).isDefined, "The snapshot must be AMT-backed after checkpoint().") + assert(classicCheckpointFiles(name).isEmpty, + "checkpoint() must not write classic checkpoint files on an AMT table.") + } + } + + test("an AMT table gets no classic checkpoint across commits and commitLarge") { + withTable("amt_no_classic") { + val name = "amt_no_classic" + // Interval 2 so ordinary commits repeatedly cross the boundary and emit AMTs. + createAMTTable(name, checkpointInterval = 2) + // v1-v4, emitting an AMT at each interval boundary along the way. + (1 to 4).foreach(i => sql(s"INSERT INTO $name VALUES ($i)")) + sql(s"RESTORE TABLE $name TO VERSION AS OF 1") // commitLarge; emits its own full AMT. + + // The point of this test is the absence of classic checkpoint files: the AMT manifests are on + // disk and no classic checkpoint is ever written. + assert((rootFiles(tablePath(name)) ++ leafFiles(tablePath(name))).nonEmpty, + "AMT manifests must be written.") + assert(classicCheckpointFiles(name).isEmpty, + "An AMT table must never accumulate a classic checkpoint file.") + } + } + test("no emission on a non-interval commit") { withTable("amt_non_interval") { val name = "amt_non_interval" @@ -268,10 +609,105 @@ class AMTCheckpointWriteSuite extends AMTCheckpointTestBase { s"INSERT INTO $name VALUES (20)"))) { context => assert(context.postCheckpointSnapshot.allFiles.count() == 21, s"Expected 21 live files, got ${context.postCheckpointSnapshot.allFiles.count()}.") - assert(context.provider.leaves.size == 3, - s"21 files at entriesPerLeaf=7 must pack into 3 leaves; got ${context.provider.leaves.size}.") - assert(context.provider.leaves.forall(_.record_count <= 7), - s"Every leaf must respect entriesPerLeaf=7: ${context.provider.leaves}") + // A full rewrite hash-distributes the 21 files across ceil(21/7)=3 writer partitions and + // skips any that come up empty, so its leaf count is a data-dependent upper bound. The + // driver-side incremental spill is deterministic and still packs into exactly 3. + if (context.scenario.isIncremental) { + assert(context.provider.leaves.size == 3, + s"21 files at entriesPerLeaf=7 must pack into 3 leaves; " + + s"got ${context.provider.leaves.size}.") + } else { + assert(context.provider.leaves.size <= 3, + s"21 files at entriesPerLeaf=7 must pack into at most 3 leaves; " + + s"got ${context.provider.leaves.size}.") + } + } + + testAcrossAMTCheckpointScenarios( + "new leaf pointers carry manifest sequence numbers and the next unassigned row ID", + "amt_leaf_tracking", + sqlConfs = leafPackingConfs)( + setup = name => appendRowsAsSeparateFiles(name, numFiles = leafPackedFiles - 1), + inlineCheckpointTriggerActionsOrSQL = Some(name => Right( + s"INSERT INTO $name VALUES (${leafPackedFiles - 1})"))) { context => + val leaves = context.provider.leaves + assertLeafCount(leaves) + val expectedFirstRowId = + AMTWriteHelper.firstRowIdAfter(RowId.extractHighWatermark(context.postCheckpointSnapshot)) + leaves.foreach { leaf => + assert(leaf.tracking.sequence_number.contains(context.checkpoint.version)) + assert(leaf.tracking.file_sequence_number.contains(context.checkpoint.version)) + assert(leaf.tracking.first_row_id.contains(expectedFirstRowId)) + val childFirstRowIds = withManifestDataEntries( + Seq(leaf.toFileStatus(context.provider.tableRoot).getPath.toString)) { entries => + entries.select("tracking.first_row_id").collect().map { row => + if (row.isNullAt(0)) None else Some(row.getLong(0)) + } + } + assert(childFirstRowIds.forall(_.isDefined), + s"Every entry in ${leaf.location} must carry a first_row_id: $childFirstRowIds") + } + } + + testAcrossAMTCheckpointScenarios( + "full rewrite records each distributed leaf's entry count", + "amt_leaf_counts", + deferredScenarios = Seq(AMTCheckpointScenario.DeferredFull), + sqlConfs = leafPackingConfs)( + setup = name => { + // Half the files carry 2 rows and half carry 4, so a leaf's existing_rows_count reflects + // the summed physical rows of its files rather than just its entry count. + val filesPerRowGroup = leafPackedFiles / 2 + appendRowsAsSeparateFiles(name, numFiles = filesPerRowGroup, rowsPerFile = 2) + appendRowsAsSeparateFiles(name, numFiles = filesPerRowGroup, rowsPerFile = 4, + startId = filesPerRowGroup * 2) + }) { context => + val leaves = context.provider.leaves + // A full rewrite hash-distributes the live files across ceil(numFiles / entriesPerLeaf) + // writer partitions and skips any empty ones, so the produced leaf count is an upper bound. + assert(leaves.size <= expectedLeafCount(leafPackedFiles), + s"$leafPackedFiles files at $entriesPerLeaf per leaf must pack into at most " + + s"${expectedLeafCount(leafPackedFiles)} leaves; got ${leaves.size}.") + val totalLiveFiles = context.postCheckpointSnapshot.allFiles.count() + val filesPerRowGroup = leafPackedFiles / 2 + val expectedTotalRows = filesPerRowGroup * 2 + filesPerRowGroup * 4 + leaves.foreach { leaf => + val mi = leaf.manifest_info + // The distributed writer flushes files that already lived in the table, so they are EXISTING + // (not ADDED). + assert(leaf.record_count > 0L, + s"A full-rewrite leaf must report a non-zero record_count; got $leaf.") + assert(mi.existing_files_count.toLong == leaf.record_count, + s"existing_files_count must equal record_count; got $mi vs ${leaf.record_count}.") + // A leaf holds a data-dependent mix of 2- and 4-row files (every file has at least 2 rows), + // so per-leaf existing_rows_count is at least twice its entry count; the exact total is + // asserted in aggregate below. + assert(mi.existing_rows_count >= 2 * leaf.record_count, + s"existing_rows_count must be at least 2x record_count; got $mi vs ${leaf.record_count}.") + assert(mi.added_rows_count == 0L && mi.deleted_rows_count == 0L && + mi.replaced_rows_count == 0L && mi.modified_rows_count == 0L, + s"A fresh full-rewrite leaf counts only existing rows; got $mi.") + assert(mi.added_files_count == 0 && mi.deleted_files_count == 0 && + mi.replaced_files_count == 0 && mi.modified_files_count == 0, + s"A fresh full-rewrite leaf counts only existing files; got $mi.") + val dataSequenceNumbers = withManifestDataEntries( + Seq(leaf.toFileStatus(context.provider.tableRoot).getPath.toString)) { entries => + entries.select("tracking.sequence_number").collect().map(_.getLong(0)) + } + assert(dataSequenceNumbers.nonEmpty, + s"A full-rewrite leaf must contain at least one data sequence number; got $leaf.") + assert(mi.min_sequence_number == dataSequenceNumbers.min, + s"min_sequence_number must match the leaf's live entries; got $mi vs " + + s"${dataSequenceNumbers.toSeq}.") + } + // Conservation: the per-leaf counts account for every live file and row (a multi-leaf tree + // keeps no root-resident data entries). + assert(leaves.map(_.record_count).sum == totalLiveFiles, + s"Leaf record_counts must sum to $totalLiveFiles; got ${leaves.map(_.record_count)}.") + assert(leaves.map(_.manifest_info.existing_rows_count).sum == expectedTotalRows, + s"Leaf existing_rows_counts must sum to $expectedTotalRows; got " + + s"${leaves.map(_.manifest_info.existing_rows_count)}.") + assertLiveAddFilesRoundTrip(context) } /** Parses the `delta.commit.stats` [[CommitStats]] logged for `version`, or fails. */ @@ -284,30 +720,40 @@ class AMTCheckpointWriteSuite extends AMTCheckpointTestBase { .getOrElse(fail(s"No commit stats logged for version $version.")) } - test("the follow-up OPTIMIZE CHECKPOINT commit stats carry AMT write metrics") { + test("the follow-up OPTIMIZE CHECKPOINT emits round and commit AMT metrics") { withTable("amt_commit_stats") { val name = "amt_commit_stats" createAMTTable(name, checkpointInterval = 2) sql(s"INSERT INTO $name VALUES (1)") // v1: below the interval, no maintenance. // v2 hits the interval boundary; the AMT is written by the follow-up commit at v3, so the - // AMT write metrics are recorded on v3's stats, not v2's. - val allStats = Log4jUsageLogger.track { + // AMT write metrics are emitted immediately by the v3 writer and summarized in its commit + // stats after that attempt commits successfully. + val events = Log4jUsageLogger.track { sql(s"INSERT INTO $name VALUES (2)") - }.filter(e => e.metric == MetricDefinitions.EVENT_TAHOE.name && + } + val allStats = events.filter(e => e.metric == MetricDefinitions.EVENT_TAHOE.name && e.tags.get("opType").contains("delta.commit.stats")) .map(e => JsonUtils.fromJson[CommitStats](e.blob)) val v2Stats = allStats.find(_.commitVersion == 2).getOrElse(fail("No stats for v2.")) - assert(v2Stats.amtWriteMetrics.isEmpty, "v2 defers the AMT; its stats carry no AMT metrics.") + assert(v2Stats.amtCommitStats.isEmpty, "v2 defers the AMT; its stats carry no AMT metrics.") val v3Stats = allStats.find(_.commitVersion == 3).getOrElse(fail("No stats for v3.")) - val metrics = v3Stats.amtWriteMetrics - .getOrElse(fail("The follow-up commit's stats should carry AMT write metrics.")) - assert(metrics.attempts.size == 1, s"Expected one AMT write attempt, got ${metrics.attempts}") + val metrics = events.filter(e => e.metric == MetricDefinitions.EVENT_TAHOE.name && + e.tags.get("opType").contains(AMTUsageLogs.CONFLICT_RESOLUTION_ROUND)) + .map(e => JsonUtils.fromJson[AMTMetrics](e.blob)) + .find(_.singleAMTWriteMetrics.nonEmpty) + .getOrElse(fail("The follow-up commit should emit AMT write metrics.")) + assert(metrics.txnId.nonEmpty) + assert(metrics.roundId == 0) + assert(metrics.conflictResolutionMetrics.isEmpty) // The first AMT has no prior tree to build on, so it is always a full rewrite. - assert(metrics.attempts.head.trigger == AMTTriggerMode.CheckpointIntervalFull.name) - assert(metrics.attempts.head.materializeDurationMs >= 0L) + val writeMetrics = metrics.singleAMTWriteMetrics.get + assert(writeMetrics.trigger == AMTTriggerMode.CheckpointIntervalFull.name) + assert(writeMetrics.materializeDurationMs >= 0L) + assert(v3Stats.amtCommitStats.map(_.lastAMTWriteMetrics).contains(writeMetrics), + "the successful follow-up commit must retain its last AMT write metrics.") } } @@ -318,11 +764,87 @@ class AMTCheckpointWriteSuite extends AMTCheckpointTestBase { val commitStats = commitStatsAt(sql(s"INSERT INTO $name VALUES (1)"), version = 1) - assert(commitStats.amtWriteMetrics.isEmpty, + assert(commitStats.amtCommitStats.isEmpty, "Commit stats must not carry AMT write metrics when no AMT is emitted.") } } + testAcrossAMTCheckpointScenarios( + "an emitted AMT populates _last_checkpoint", + "amt_last_checkpoint")( + setup = name => sql(s"INSERT INTO $name VALUES (1)"), + inlineCheckpointTriggerActionsOrSQL = Some(name => Right( + s"INSERT INTO $name VALUES (2)"))) { context => + val deltaLog = context.postCheckpointSnapshot.deltaLog + val info = deltaLog.readLastCheckpointFile().getOrElse { + fail("An AMT emission must write _last_checkpoint.") + } + + // `version` names the tree's content-root version, and `parts` the leaf count. + assert(info.version == context.checkpoint.contentRoot.version, + s"Expected content root v${context.checkpoint.contentRoot.version}, got v${info.version}.") + assert(info.parts.contains(context.provider.leaves.size), + s"Expected ${context.provider.leaves.size} leaves, got ${info.parts}.") + assert(info.sizeInBytes.exists(_ > 0L), "The AMT size in bytes must be recorded.") + // `size` (action count) and `numOfAddFiles` are intentionally unavailable for AMT. + assert(info.size == -1, s"AMT size must be -1, got ${info.size}.") + assert(info.numOfAddFiles.isEmpty, + s"AMT numOfAddFiles must be empty, got ${info.numOfAddFiles}.") + + assert(info.checkpointType == Some(LastCheckpointInfo.CheckpointType.AMT)) + val amt = info.amtCheckpoint.getOrElse { + fail("_last_checkpoint must carry amtCheckpoint.") + } + assert(amt.checkpoint == Some(context.checkpoint)) + assertLeavesEqual(amt.leaves.get, context.provider.leaves) + assert(amt.manifestCommitVersion == context.manifestCommitVersion, + s"Expected manifest at v${context.manifestCommitVersion}, got v${amt.manifestCommitVersion}.") + } + + test("_last_checkpoint is not updated when no AMT is emitted") { + withTable("amt_last_checkpoint_unchanged") { + val name = "amt_last_checkpoint_unchanged" + // Interval 3. Emission triggers only when the version distance from the last AMT is an exact + // positive multiple of the interval, so the timeline is: + // v1: below the first boundary -> no emission + // v2: below the first boundary -> no emission + // v3: first boundary -> deferred AMT recorded at v4, describing v3 + // v4: OPTIMIZE CHECKPOINT -> emits an AMT + // v5: one past the last AMT -> no emission (the next boundary is v6) + createAMTTable(name, checkpointInterval = 3) + val deltaLog = deltaLogForName(name) + + // -- Before any emission: no pointer at all. -- + sql(s"INSERT INTO $name VALUES (1)") + sql(s"INSERT INTO $name VALUES (2)") + assert(deltaLog.update().version == 2, "Neither v1 nor v2 should schedule a follow-up AMT.") + assert(checkpointAt(deltaLog, 1).isEmpty, "v1 emits no AMT.") + assert(checkpointAt(deltaLog, 2).isEmpty, "v2 emits no AMT.") + assert(deltaLog.readLastCheckpointFile().isEmpty, + "Commits that emit no AMT must not write _last_checkpoint.") + + // -- The boundary commit emits, and the follow-up commit writes the pointer. -- + sql(s"INSERT INTO $name VALUES (3)") + assert(deltaLog.update().version == 4, "The v3 boundary defers its AMT to v4.") + val afterEmission = deltaLog.readLastCheckpointFile().getOrElse { + fail("The deferred AMT must write _last_checkpoint.") + } + assert(afterEmission.version == 3, "The tree describes state as of the v3 boundary.") + assert(afterEmission.amtCheckpoint.map(_.manifestCommitVersion) == Some(4L), + "The manifest commit version should be v4.") + + // -- A later commit that emits nothing must leave the existing pointer untouched. -- + sql(s"INSERT INTO $name VALUES (4)") + assert(deltaLog.update().version == 5, "v5 should not schedule a follow-up AMT.") + assert(checkpointAt(deltaLog, 5).isEmpty, "v5 emits no AMT.") + val afterNonAMTCommit = deltaLog.readLastCheckpointFile().getOrElse { + fail("_last_checkpoint must survive commits that emit no AMT.") + } + assert(afterNonAMTCommit == afterEmission, + "A non-AMT commit must not modify _last_checkpoint.") + } + } + /** An ADDED tracking envelope with no lineage/sequence numbers, matching the AMT writer. */ private def addedTracking: Tracking = Tracking( status = Tracking.Status.Added, @@ -340,10 +862,12 @@ class AMTCheckpointWriteSuite extends AMTCheckpointTestBase { existing_files_count = 0, deleted_files_count = 0, replaced_files_count = 0, + modified_files_count = 0, added_rows_count = 0L, existing_rows_count = 0L, deleted_rows_count = 0L, replaced_rows_count = 0L, + modified_rows_count = 0L, min_sequence_number = 0L, dv = None, dv_cardinality = None) diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTCommitLargeSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTCommitLargeSuite.scala new file mode 100644 index 00000000000..636d0bfae60 --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTCommitLargeSuite.scala @@ -0,0 +1,163 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.amt + +import org.apache.spark.sql.delta.actions.{AddFile, RemoveFile} + +import org.apache.spark.sql.Row + +class AMTCommitLargeSuite extends AMTCheckpointTestBase { + + /** + * One `commitLarge` operation under test. + * + * @param operationName operation name, used in test names and assertion messages. + * @param targetTable the AMT table the operation commits to. + * @param tableNames every table the scenario creates, for `withTable` cleanup. + * @param commitVersion the version the operation itself commits at; the follow-up AMT lands at + * `commitVersion + 1`. + * @param expectedRows rows the target holds once the operation completes. + * @param setup creates the tables and commits everything up to, but not including, the + * operation. + * @param commitLargeOperation runs the operation, which commits at `commitVersion`. + */ + private case class CommitLargeOp( + operationName: String, + targetTable: String, + tableNames: Seq[String], + commitVersion: Long, + expectedRows: Seq[Row], + setup: () => Unit, + commitLargeOperation: () => Unit) + + /** + * RESTORE on an AMT table: three INSERTs (v1-v3), then RESTORE commits at v4. `onBoundary` picks + * the checkpoint interval that lands the RESTORE ON an interval boundary (the version it commits + * at) or well OFF every boundary; emission is unconditional either way. + */ + private def restoreOp(onBoundary: Boolean): CommitLargeOp = { + val table = "amt_cl_restore" + val checkpointInterval = if (onBoundary) 4 else 100 + CommitLargeOp( + operationName = "RESTORE", + targetTable = table, + tableNames = Seq(table), + commitVersion = 4, + expectedRows = Seq(Row(1)), // RESTORE TO VERSION 1 keeps only the first INSERT's file. + setup = () => { + createAMTTable(table, checkpointInterval = checkpointInterval) + (1 to 3).foreach(i => sql(s"INSERT INTO $table VALUES ($i)")) // v1, v2, v3. + }, + commitLargeOperation = () => sql(s"RESTORE TABLE $table TO VERSION AS OF 1")) // v4. + } + + + /** + * Asserts `op` emitted exactly one FULL AMT via a follow-up OPTIMIZE CHECKPOINT commit at + * `commitVersion + 1`, and that the resulting snapshot reads back through the manifest tree. + */ + private def checkFullAMTEmitted(op: CommitLargeOp): Unit = { + // A freshly-resolved DeltaLog cold-reads the emitted AMT: reconciliation installs the manifest + // checkpoint provider from the CRC, so no warm handle from the emitting commit is needed. + val deltaLog = deltaLogForName(op.targetTable) + val commitVersion = op.commitVersion + val followUpVersion = commitVersion + 1 + val snapshot = deltaLog.update() + assert(snapshot.version == followUpVersion, + s"A follow-up OPTIMIZE CHECKPOINT commit must land at v$followUpVersion.") + assert(checkpointAt(deltaLog, commitVersion).isEmpty, + s"The ${op.operationName} commit itself must carry no Checkpoint action.") + val checkpoint = checkpointAt(deltaLog, followUpVersion) + .getOrElse(fail(s"Expected a Checkpoint at v$followUpVersion.")) + assert(checkpoint.version == commitVersion, + s"The Checkpoint must describe state as of v$commitVersion; got ${checkpoint.version}.") + // A commitLarge AMT is always a full rewrite, and a full rewrite records its own described + // version as the last-full-rewrite marker. + assert(checkpoint.contentRoot.isIncremental.contains(false), + "A commitLarge AMT must be a full rewrite.") + assert(checkpoint.contentRoot.lastManifestCommitWithFullRewrite.contains(commitVersion), + "A full rewrite records its own described version as the last-full-rewrite marker.") + assert(!actionsAt(deltaLog, followUpVersion).exists(_.isInstanceOf[AddFile]), + "The follow-up commit carries no user AddFile.") + + // The manifest tree is on disk and the data reads back correctly (all cold-read-safe). A small + // tree may be a single leaf promoted to the root (no separate root- file), so assert the tree + // exists via the root/leaf union rather than requiring a distinct root manifest. + assert((rootFiles(tablePath(op.targetTable)) ++ leafFiles(tablePath(op.targetTable))).nonEmpty, + "A manifest tree (root and/or promoted leaf) must be written.") + checkAnswer(spark.read.table(op.targetTable), op.expectedRows) + + // The post-operation snapshot is AMT-backed and its tree reconstructs the live file set. Use + // currentLiveDataEntries (whole tree) since DATA entries can live directly in the root when the + // tree is small enough to skip separate leaves. Reconstruct the pre-checkpoint live file set by + // replaying the commit log up to commitVersion (deltas only, independent of the AMT tree) so + // this is a real cross-check rather than a tautological tree-vs-tree count. + val liveFilePaths = scala.collection.mutable.Set.empty[String] + deltaLog.getChanges(0).takeWhile(_._1 <= commitVersion).foreach { case (_, actions) => + actions.foreach { + case a: AddFile => liveFilePaths += a.path + case r: RemoveFile => liveFilePaths -= r.path + case _ => + } + } + val preCheckpointFileCount = liveFilePaths.size.toLong + assert(amtProvider(snapshot).isDefined, "The post-operation snapshot must be AMT-backed.") + assert(currentLiveDataEntries(snapshot) == preCheckpointFileCount, + "AMT live DATA entries must equal the pre-checkpoint live file count.") + } + + for { + // Each operation is run once ON its checkpoint interval boundary and once well OFF every + // boundary; each op derives its own interval from `onBoundary` (see its factory). Emission is + // unconditional either way. + buildOp <- Seq[Boolean => CommitLargeOp]( + restoreOp + ) + (boundaryLabel, onBoundary) <- Seq( + "on the checkpoint interval boundary" -> true, + "off the checkpoint interval boundary" -> false) + } { + val op = buildOp(onBoundary) + test(s"${op.operationName} emits a full AMT $boundaryLabel") { + withTable(op.tableNames: _*) { + op.setup() + // The table carries no AMT before the commitLarge operation. + assert(amtProvider(deltaLogForName(op.targetTable).update()).isEmpty, + s"No AMT should exist before the ${op.operationName}.") + + op.commitLargeOperation() + + checkFullAMTEmitted(op) + } + } + } + + + test("a RESTORE emits a full rewrite even when a prior AMT already exists") { + val op = restoreOp(onBoundary = true) + withTable(op.tableNames: _*) { + op.setup() + op.commitLargeOperation() // v4 -> follow-up AMT at v5. + checkFullAMTEmitted(op) + + // A second RESTORE has a prior tree it *could* build on incrementally, but commitLarge always + // emits a full rewrite, so this one is full too (with an advanced marker). + sql(s"RESTORE TABLE ${op.targetTable} TO VERSION AS OF 2") // v6 -> follow-up AMT at v7. + checkFullAMTEmitted(op.copy(commitVersion = 6, expectedRows = Seq(Row(1), Row(2)))) + } + } +} diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTConflictResolutionSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTConflictResolutionSuite.scala new file mode 100644 index 00000000000..0163f6eb5f3 --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTConflictResolutionSuite.scala @@ -0,0 +1,1080 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.amt + +import scala.concurrent.Future +import scala.concurrent.duration.Duration + +import com.databricks.spark.util.{Log4jUsageLogger, MetricDefinitions} +import org.apache.spark.sql.delta.{ConcurrentAMTCheckpointLandedException, ConcurrentWriteException, DeltaLog, FullAMTWriteFailedWithConflict} +import org.apache.spark.sql.delta.concurrency.{PhaseLockingTestMixin, TransactionExecutionTestMixin} +import org.apache.spark.sql.delta.sources.DeltaSQLConf +import org.apache.spark.sql.delta.util.DeltaTestBarrier +import org.apache.spark.sql.delta.util.JsonUtils + +import org.apache.spark.SparkException +import org.apache.spark.sql.Row +import org.apache.spark.sql.SparkSession +import org.apache.spark.util.ThreadUtils + +/** + * Conflict-resolution behavior for AMT-backed tables. + */ +class AMTConflictResolutionSuite + extends AMTCheckpointTestBase + with PhaseLockingTestMixin + with TransactionExecutionTestMixin { + + import AMTConflictResolutionRoundMetrics._ + import BackRefRebaseMetrics._ + + // The phase-locking tests pause a transaction mid-commit at a TestBarrier, then commit a + // concurrent winner from the test thread. The commit lock (enabled by default here) is held for + // the whole commit attempt, so the paused transaction would block that winner and deadlock the + // interleaving; disable it for the whole suite. + override protected def sparkConf = + super.sparkConf + .set(DeltaSQLConf.DELTA_COMMIT_LOCK_ENABLED.key, "false") + // The phase-locking tests arm a DeltaTestBarrier, which requires this config. + .set(DeltaSQLConf.TEST_BARRIER_ENABLED.key, "true") + + // A large interval keeps the checkpoint-interval maintenance hook from firing its own checkpoint + // mid-test; every AMT write here is one the test drives explicitly. + private val noAutoCheckpointInterval = 1000 + + private def trackConflictResolutionRounds( + f: => Unit): Seq[AMTConflictResolutionRoundMetrics] = { + val metrics = Log4jUsageLogger.track(f) + .filter(e => e.metric == MetricDefinitions.EVENT_TAHOE.name && + e.tags.get("opType").contains(AMTUsageLogs.CONFLICT_RESOLUTION_ROUND)) + .map(e => JsonUtils.fromJson[AMTMetrics](e.blob)) + .filter(_.conflictResolutionMetrics.nonEmpty) + .sortBy(_.roundId) + assert(metrics.forall(m => m.txnId.nonEmpty && m.roundId > 0), + s"conflict rounds must identify their transaction and positive round ID: $metrics") + metrics.flatMap(_.conflictResolutionMetrics) + } + + // Asserts the common conflict-round grid metrics for a run that produced exactly one conflict + // round, and returns that round so a caller can assert anything not covered here. + private def assertSingleAMTConflictRoundMetrics( + rounds: Seq[AMTConflictResolutionRoundMetrics], + expectedLosingCommitType: String, + expectedTreeOutcome: String, + expectedWinnerTreeSatisfiesRequirement: Option[Boolean], + expectedLosingTreeType: Option[String] = None, + expectedException: Option[Class[_ <: Throwable]] = None, + numLogOnlyWinners: Int = 0, + numIncrementalCheckpointWinners: Int = 0, + numFullCheckpointWinners: Int = 0, + allLogWinnersPreserveLosingTree: Option[Boolean] = None) + : AMTConflictResolutionRoundMetrics = { + assert(rounds.size == 1, s"expected exactly one AMT conflict round, got $rounds") + val round = rounds.head + assert(round.losingCommitType == expectedLosingCommitType) + assert(round.losingTreeType == expectedLosingTreeType) + assert(round.treeOutcome == expectedTreeOutcome) + assert(round.winnerTreeSatisfiesRequirement == + expectedWinnerTreeSatisfiesRequirement) + assert(round.exceptionThrown == expectedException.map(_.getSimpleName)) + assert(round.winningCommits.numLogOnly == numLogOnlyWinners) + assert(round.winningCommits.numInlineIncremental == 0) + assert(round.winningCommits.numIncrementalCheckpoints == + numIncrementalCheckpointWinners) + assert(round.winningCommits.numFullCheckpoints == numFullCheckpointWinners) + assert((round.winningCommits.numFullCheckpoints > 0) == (numFullCheckpointWinners > 0)) + assert(round.winningCommits.allLogWinnersPreserveLosingTree == + allLogWinnersPreserveLosingTree) + round + } + + /** A transaction body that commits an incremental OPTIMIZE CHECKPOINT (writes a new AMT tree). */ + private def optimizeCheckpointTxn(deltaLog: DeltaLog): () => Array[Row] = () => { + commitCheckpoint(deltaLog, incremental = true) + Array.empty + } + + /** A transaction body that commits a full-rewrite OPTIMIZE CHECKPOINT (a brand-new AMT tree). */ + private def fullCheckpointTxn(deltaLog: DeltaLog): () => Array[Row] = () => { + commitCheckpoint(deltaLog, incremental = false) + Array.empty + } + + /** A transaction body that appends `id` as a plain (log-only, no tree) business commit. */ + private def appendTxn(tableName: String, id: Int): () => Array[Row] = () => { + spark.sql(s"INSERT INTO $tableName VALUES ($id)").collect() + } + + /** A transaction body that deletes `id` -- a non-blind commit. */ + private def deleteTxn(tableName: String, id: Int): () => Array[Row] = () => { + spark.sql(s"DELETE FROM $tableName WHERE id = $id").collect() + } + + /** + * Creates an AMT table with a full checkpoint base and a few seeded files, then returns its + * [[DeltaLog]]. + */ + private def setupAMTTable(tableName: String): DeltaLog = { + createAMTTable(tableName, checkpointInterval = noAutoCheckpointInterval) + val deltaLog = deltaLogForName(tableName) + // Do a full checkpoint -- it is to make sure that incremental checkpoints can be done in tests. + commitCheckpoint(deltaLog, incremental = false) + appendRowsAsSeparateFiles(tableName, numFiles = 3, startId = 1) + deltaLog + } + + /** + * Awaits `future`, asserting that a losing full maintenance checkpoint signaled a full-AMT + * regenerate via [[FullAMTWriteFailedWithConflict]] (distinct from a deferred + * [[ConcurrentWriteException]]). Used when the checkpoint is driven directly rather than through + * `CheckpointHook`, so the signal surfaces to the caller instead of being retried. + */ + private def assertRetrySignal(future: Future[Array[Row]]): Unit = { + val ex = intercept[SparkException] { + ThreadUtils.awaitResult(future, Duration.Inf) + } + val causes = Iterator + .iterate[Throwable](ex)(t => if (t.getCause eq t) null else t.getCause) + .takeWhile(_ != null) + .take(50) + .toList + assert(causes.exists(_.isInstanceOf[FullAMTWriteFailedWithConflict]), + s"expected a FullAMTWriteFailedWithConflict in the cause chain, got " + + causes.map(_.getClass.getName).mkString(" -> ")) + assert(!causes.exists(_.isInstanceOf[ConcurrentWriteException]), + "the regenerate signal must be distinct from a deferred ConcurrentWriteException.") + } + + /** + * Awaits `future`, asserting that a losing maintenance checkpoint signaled a clean skip via + * [[ConcurrentAMTCheckpointLandedException]] (distinct from a deferred + * [[ConcurrentWriteException]]). Used when the checkpoint is driven directly rather than through + * the maintenance path, so the skip signal surfaces to the caller instead of being suppressed. + */ + private def assertCleanSkipSignal(future: Future[Array[Row]]): Unit = { + val ex = intercept[SparkException] { + ThreadUtils.awaitResult(future, Duration.Inf) + } + val causes = Iterator + .iterate[Throwable](ex)(t => if (t.getCause eq t) null else t.getCause) + .takeWhile(_ != null) + .take(50) + .toList + assert(causes.exists(_.isInstanceOf[ConcurrentAMTCheckpointLandedException]), + s"expected a ConcurrentAMTCheckpointLandedException in the cause chain, got " + + causes.map(_.getClass.getName).mkString(" -> ")) + assert(!causes.exists(_.isInstanceOf[ConcurrentWriteException]), + "the clean-skip signal must be distinct from a deferred ConcurrentWriteException.") + } + + // A losing OPTIMIZE checkpoint -- whether its first attempt wrote a full or an incremental + // tree -- that loses its target version to a single base-preserving log-only winner recommits + // its already-written tree as-is at the next version instead of folding the winner in or redoing + // the rewrite. The winner survives and the reused tree keeps its full/incremental flag. + for ((label, losingCheckpointTxn, reusedTreeIsIncremental) <- Seq( + ("full", fullCheckpointTxn _, false), + ("incremental", optimizeCheckpointTxn _, true))) { + test(s"one Log Commit vs a losing $label OPTIMIZE checkpoint - reuses base") { + withTable(s"amt_conflict_${label}_ckpt_one_log_reuse") { + val name = s"amt_conflict_${label}_ckpt_one_log_reuse" + val deltaLog = setupAMTTable(name) + val liveIdsBefore = spark.sql(s"SELECT id FROM $name").collect().map(_.getInt(0)).toSet + + // A is a losing OPTIMIZE checkpoint describing the read snapshot; B is a plain append that + // wins A's target version and only adds brand-new files (defaultRowCommitVersion past A's + // read version), so A's tree stays exact and is recommitted as-is. + val rounds = trackConflictResolutionRounds { + val (futureA, futureB) = runTxnsWithOrder__A_Start__B__A_End( + losingCheckpointTxn(deltaLog), + appendTxn(name, id = 100)) + ThreadUtils.awaitResult(futureB, Duration.Inf) + ThreadUtils.awaitResult(futureA, Duration.Inf) + } + assertSingleAMTConflictRoundMetrics( + rounds, + expectedLosingCommitType = + if (reusedTreeIsIncremental) INCREMENTAL_CHECKPOINT else FULL_CHECKPOINT, + expectedLosingTreeType = + Some(if (reusedTreeIsIncremental) INCREMENTAL_TREE else FULL_TREE), + expectedTreeOutcome = REUSED_LOSING_TREE, + expectedWinnerTreeSatisfiesRequirement = None, + numLogOnlyWinners = 1, + allLogWinnersPreserveLosingTree = Some(true)) + + // B survives and A reused its base: the live set reconstructs through A's reused tree plus + // B's log delta. The reused checkpoint is A's first-attempt tree recommitted as-is, so its + // incremental flag matches the first attempt (full or incremental). + val liveIdsAfter = spark.sql(s"SELECT id FROM $name").collect().map(_.getInt(0)).toSet + assert(liveIdsAfter == liveIdsBefore + 100, + s"B's append must survive A's base reuse; before=$liveIdsBefore after=$liveIdsAfter") + val latest = deltaLog.update() + val checkpoint = checkpointAt(deltaLog, latest.version).getOrElse( + fail("A's rebased commit must carry an AMT checkpoint.")) + assert(checkpoint.contentRoot.isIncremental.contains(reusedTreeIsIncremental), + s"a base-preserving winner must let A recommit its $label base as-is.") + assert(amtProvider(latest).isDefined, "the table must remain AMT-backed after the reuse.") + } + } + } + + // A losing OPTIMIZE checkpoint -- whether its first attempt wrote a full or an incremental + // tree -- that loses its target version to two base-preserving log-only winners recommits its + // already-written tree as-is across the whole conflicted range: both winners survive and a + // second base-preserving log winner does not drop it to a ConcurrentWriteException. + for ((label, losingCheckpointTxn, reusedTreeIsIncremental) <- Seq( + ("full", fullCheckpointTxn _, false), + ("incremental", optimizeCheckpointTxn _, true))) { + test(s"two Log Commits vs a losing $label OPTIMIZE checkpoint - reuses base") { + withTable(s"amt_conflict_${label}_ckpt_two_log_reuse") { + val name = s"amt_conflict_${label}_ckpt_two_log_reuse" + val deltaLog = setupAMTTable(name) + val liveIdsBefore = spark.sql(s"SELECT id FROM $name").collect().map(_.getInt(0)).toSet + + // A is a losing OPTIMIZE checkpoint describing the read snapshot; B and C are plain appends + // that both win versions ahead of A. Both winners only add brand-new files (their + // defaultRowCommitVersion postdates A's read version), so A's tree stays exact and it is + // recommitted as-is across both winners. + val rounds = trackConflictResolutionRounds { + val (futureA, futureB, futureC) = runTxnsWithOrder__A_Start__B__C__A_End( + losingCheckpointTxn(deltaLog), + appendTxn(name, id = 100), + appendTxn(name, id = 200)) + ThreadUtils.awaitResult(futureB, Duration.Inf) + ThreadUtils.awaitResult(futureC, Duration.Inf) + ThreadUtils.awaitResult(futureA, Duration.Inf) + } + assertSingleAMTConflictRoundMetrics( + rounds, + expectedLosingCommitType = + if (reusedTreeIsIncremental) INCREMENTAL_CHECKPOINT else FULL_CHECKPOINT, + expectedLosingTreeType = + Some(if (reusedTreeIsIncremental) INCREMENTAL_TREE else FULL_TREE), + expectedTreeOutcome = REUSED_LOSING_TREE, + expectedWinnerTreeSatisfiesRequirement = None, + numLogOnlyWinners = 2, + allLogWinnersPreserveLosingTree = Some(true)) + + // All commits survive: the live set reconstructs through A's reused tree plus B's and C's + // log deltas. The reused checkpoint is A's first-attempt tree recommitted as-is, so its + // incremental flag matches the first attempt (full or incremental). + val liveIdsAfter = spark.sql(s"SELECT id FROM $name").collect().map(_.getInt(0)).toSet + assert(liveIdsAfter == liveIdsBefore ++ Set(100, 200), + s"all commits must survive A's base reuse; before=$liveIdsBefore after=$liveIdsAfter") + val latest = deltaLog.update() + val checkpoint = checkpointAt(deltaLog, latest.version).getOrElse( + fail("A's rebased commit must carry an AMT checkpoint.")) + assert(checkpoint.contentRoot.isIncremental.contains(reusedTreeIsIncremental), + s"a base-preserving winner must let A recommit its $label base as-is.") + assert(amtProvider(latest).isDefined, "the table must remain AMT-backed after the reuse.") + } + } + } + + test("Winning Commit [Log Commit] vs Losing commit [inline-incremental commit] - Success") { + withTable("amt_conflict_inline_rebase") { + val name = "amt_conflict_inline_rebase" + val deltaLog = setupAMTTable(name) + val liveIdsBefore = spark.sql(s"SELECT id FROM $name").collect().map(_.getInt(0)).toSet + + // Threshold 6: A's 10-file insert inlines its AMT tree, while B's single-file append (a + // couple of actions) stays log-only. So A is a tree writer losing to a log-only winner -- it + // must rebase and rebuild its tree with B folded into the incremental window rather than + // hard-failing. Capture A's per-attempt AMT write metrics to inspect the window growth. + val perAttempt = trackIncrementalAMTWriteMetricsPerAttempt { + withInlineThreshold(6) { + val (futureA, futureB) = runTxnsWithOrder__A_Start__B__A_End( + () => { appendRowsAsSeparateFiles(name, numFiles = 10, startId = 100); Array.empty }, + appendTxn(name, id = 200)) + ThreadUtils.awaitResult(futureB, Duration.Inf) + ThreadUtils.awaitResult(futureA, Duration.Inf) + } + } + + // Reading the live set back goes through A's rebased inline tree, so a correct id set proves + // the tree folded the winning append. + val liveIdsAfter = spark.sql(s"SELECT id FROM $name").collect().map(_.getInt(0)).toSet + assert(liveIdsAfter == liveIdsBefore ++ (100 to 109).toSet ++ Set(200), + s"both commits must survive the rebase; before=$liveIdsBefore after=$liveIdsAfter") + // A's commit rode an inline AMT checkpoint, confirming it took the tree-writing rebase path. + val latest = deltaLog.update() + assert(checkpointAt(deltaLog, latest.version).isDefined, + "the rebased inline commit must carry an AMT checkpoint at its version.") + assert(amtProvider(latest).isDefined, "the table must remain AMT-backed after the rebase.") + + // A materialized its tree exactly twice -- once on its first attempt and once on the rebase + // -- and the rebase's incremental window folds in exactly one extra commit, the winning + // append, so its numIntermediateCommits is precisely one more than the first attempt's. + val numIntermediateCommits = perAttempt.map(_.numIntermediateCommits) + assert(numIntermediateCommits.size == 2, + s"A must materialize exactly twice (first attempt + rebase); got $numIntermediateCommits") + assert(numIntermediateCommits(1) == numIntermediateCommits(0) + 1, + "the rebase must fold exactly one more intermediate commit (the winning append) than the " + + s"first attempt; got $numIntermediateCommits") + } + } + + test("Winning Commit [Incremental checkpoint commit] vs Losing commit " + + "[inline-incremental commit] - Success") { + withTable("amt_conflict_inline_vs_tree_winner") { + val name = "amt_conflict_inline_vs_tree_winner" + val deltaLog = setupAMTTable(name) + val liveIdsBefore = spark.sql(s"SELECT id FROM $name").collect().map(_.getInt(0)).toSet + + // A's 10-file insert inlines a tree; B installs a new tree (OPTIMIZE CHECKPOINT) and wins A's + // target version. A must re-seat its incremental write onto B's tree and rebuild its own tree + // on top (re-deriving back references if B's leaf set moved), rather than hard-failing. + withInlineThreshold(6) { + val (futureA, futureB) = runTxnsWithOrder__A_Start__B__A_End( + () => { appendRowsAsSeparateFiles(name, numFiles = 10, startId = 100); Array.empty }, + optimizeCheckpointTxn(deltaLog)) + ThreadUtils.awaitResult(futureB, Duration.Inf) + ThreadUtils.awaitResult(futureA, Duration.Inf) + } + + val liveIdsAfter = spark.sql(s"SELECT id FROM $name").collect().map(_.getInt(0)).toSet + assert(liveIdsAfter == liveIdsBefore ++ (100 to 109).toSet, + s"A's files must survive the rebase onto the winner's tree; before=$liveIdsBefore " + + s"after=$liveIdsAfter") + // A's rebased commit rode an inline AMT checkpoint built on the winner's tree. + val latest = deltaLog.update() + assert(checkpointAt(deltaLog, latest.version).isDefined, + "the rebased inline commit must carry an AMT checkpoint at its version.") + assert(amtProvider(latest).isDefined, "the table must remain AMT-backed after the rebase.") + } + } + + test("Winning Commit [log commit] vs Losing commit [log commit] - SUCCESS") { + withTable("amt_conflict_log_vs_log") { + val name = "amt_conflict_log_vs_log" + val deltaLog = setupAMTTable(name) + val liveIdsBefore = spark.sql(s"SELECT id FROM $name").collect().map(_.getInt(0)).toSet + + // Both A and B are plain appends of distinct rows: neither writes a tree and blind appends do + // not logically conflict, so A must rebase past B and commit. + val (futureA, futureB) = runTxnsWithOrder__A_Start__B__A_End( + appendTxn(name, id = 100), + appendTxn(name, id = 200)) + + // Neither future may fail: B wins its version and A rebases onto it. + ThreadUtils.awaitResult(futureB, Duration.Inf) + ThreadUtils.awaitResult(futureA, Duration.Inf) + + // Neither commit writes a tree, so the base AMT is unchanged. Reading the live set back + // still goes through it, and a correct id set proves A rebased its log actions past B. + val liveIdsAfter = spark.sql(s"SELECT id FROM $name").collect().map(_.getInt(0)).toSet + assert(liveIdsAfter == liveIdsBefore ++ Set(100, 200), + s"both appends must survive the rebase; before=$liveIdsBefore after=$liveIdsAfter") + assert(amtProvider(deltaLog.update()).isDefined, + "the table must remain AMT-backed after the rebase.") + } + } + + test("Winning Commit [Incremental checkpoint commit] vs Losing commit [log commit] - " + + "Success") { + withTable("amt_conflict_log_vs_tree_winner") { + val name = "amt_conflict_log_vs_tree_winner" + val deltaLog = setupAMTTable(name) + val liveIdsBefore = spark.sql(s"SELECT id FROM $name").collect().map(_.getInt(0)).toSet + + // A is a non-blind DELETE (it removes id 1's file); B installs a new tree (OPTIMIZE + // CHECKPOINT) and wins A's target version. A must rebase past the new tree -- re-deriving its + // RemoveFile's back reference against it when the tree's leaf set moved -- rather than + // hard-failing. + val rounds = trackConflictResolutionRounds { + val (futureA, futureB) = runTxnsWithOrder__A_Start__B__A_End( + deleteTxn(name, id = 1), + optimizeCheckpointTxn(deltaLog)) + ThreadUtils.awaitResult(futureB, Duration.Inf) + ThreadUtils.awaitResult(futureA, Duration.Inf) + } + val round = assertSingleAMTConflictRoundMetrics( + rounds, + expectedLosingCommitType = LOG_ONLY, + expectedTreeOutcome = NO_TREE_REBASE, + expectedWinnerTreeSatisfiesRequirement = None, + numIncrementalCheckpointWinners = 1) + + val liveIdsAfter = spark.sql(s"SELECT id FROM $name").collect().map(_.getInt(0)).toSet + assert(liveIdsAfter == liveIdsBefore - 1, + s"A's delete must survive the rebase onto the winner's tree; before=$liveIdsBefore " + + s"after=$liveIdsAfter") + assert(amtProvider(deltaLog.update()).isDefined, + "the table must remain AMT-backed after the rebase.") + // The rebase really ran: one round re-derived A's RemoveFile back reference against B's tree. + assert(round.backRefRebaseMetrics.numActionsRegeneratingBackref.exists(_ >= 1), + s"A must record one back-ref rebase that re-derived its RemoveFile; got $round") + } + } + + test("Winning Commit [Full checkpoint commit] vs Losing commit [log commit] - Success") { + withTable("amt_conflict_log_vs_full_tree_winner") { + val name = "amt_conflict_log_vs_full_tree_winner" + val deltaLog = setupAMTTable(name) + val liveIdsBefore = spark.sql(s"SELECT id FROM $name").collect().map(_.getInt(0)).toSet + + // A is a non-blind DELETE (it removes id 1's file); B installs a brand-new FULL-rewrite tree + // (OPTIMIZE CHECKPOINT, incremental = false) and wins A's target version. A full rewrite + // moves every leaf position, so A's RemoveFile back reference is re-derived from scratch + // against B's tree before A commits, rather than hard-failing. + val rounds = trackConflictResolutionRounds { + val (futureA, futureB) = runTxnsWithOrder__A_Start__B__A_End( + deleteTxn(name, id = 1), + fullCheckpointTxn(deltaLog)) + ThreadUtils.awaitResult(futureB, Duration.Inf) + ThreadUtils.awaitResult(futureA, Duration.Inf) + } + val round = assertSingleAMTConflictRoundMetrics( + rounds, + expectedLosingCommitType = LOG_ONLY, + expectedTreeOutcome = NO_TREE_REBASE, + expectedWinnerTreeSatisfiesRequirement = None, + numFullCheckpointWinners = 1) + + val liveIdsAfter = spark.sql(s"SELECT id FROM $name").collect().map(_.getInt(0)).toSet + assert(liveIdsAfter == liveIdsBefore - 1, + s"A's delete must survive the rebase onto the full-rewrite winner's tree; " + + s"before=$liveIdsBefore after=$liveIdsAfter") + assert(amtProvider(deltaLog.update()).isDefined, + "the table must remain AMT-backed after the rebase.") + // The rebase really ran: the full rewrite forced A's RemoveFile back reference to be + // re-derived against B's tree. + assert(round.backRefRebaseMetrics.numActionsRegeneratingBackref.exists(_ >= 1), + s"A must record one back-ref rebase that re-derived its RemoveFile; got $round") + } + } + + test("Winning Commit [Full checkpoint commit] vs Losing commit " + + "[inline-incremental commit] - Success") { + withTable("amt_conflict_inline_vs_full_tree_winner") { + val name = "amt_conflict_inline_vs_full_tree_winner" + val deltaLog = setupAMTTable(name) + val liveIdsBefore = spark.sql(s"SELECT id FROM $name").collect().map(_.getInt(0)).toSet + + // A's 10-file insert inlines a tree; B installs a brand-new FULL-rewrite tree and wins A's + // target version. A must re-seat its incremental write onto B's full tree and rebuild its + // own tree on top, re-deriving all its back references, rather than hard-failing. + withInlineThreshold(6) { + val (futureA, futureB) = runTxnsWithOrder__A_Start__B__A_End( + () => { appendRowsAsSeparateFiles(name, numFiles = 10, startId = 100); Array.empty }, + fullCheckpointTxn(deltaLog)) + ThreadUtils.awaitResult(futureB, Duration.Inf) + ThreadUtils.awaitResult(futureA, Duration.Inf) + } + + val liveIdsAfter = spark.sql(s"SELECT id FROM $name").collect().map(_.getInt(0)).toSet + assert(liveIdsAfter == liveIdsBefore ++ (100 to 109).toSet, + s"A's files must survive the rebase onto the full-rewrite winner's tree; " + + s"before=$liveIdsBefore after=$liveIdsAfter") + val latest = deltaLog.update() + assert(checkpointAt(deltaLog, latest.version).isDefined, + "the rebased inline commit must carry an AMT checkpoint at its version.") + assert(amtProvider(latest).isDefined, "the table must remain AMT-backed after the rebase.") + } + } + + test("Winning Commit [Full checkpoint commit] vs Losing commit " + + "[incremental checkpoint commit] - SKIP") { + withTable("amt_conflict_incr_ckpt_vs_full_winner") { + val name = "amt_conflict_incr_ckpt_vs_full_winner" + val deltaLog = setupAMTTable(name) + val liveIdsBefore = spark.sql(s"SELECT id FROM $name").collect().map(_.getInt(0)).toSet + + // A is an incremental OPTIMIZE checkpoint; B installs a brand-new full AMT tree and wins A's + // target version. B's tree supersedes A's, so the losing incremental checkpoint is redundant + // and cleanly skips (ConcurrentAMTCheckpointLandedException) rather than hard-failing. A is a + // direct checkpoint commit, so the skip signal surfaces to the caller. + val rounds = trackConflictResolutionRounds { + val (futureA, futureB) = runTxnsWithOrder__A_Start__B__A_End( + optimizeCheckpointTxn(deltaLog), + fullCheckpointTxn(deltaLog)) + ThreadUtils.awaitResult(futureB, Duration.Inf) + assertCleanSkipSignal(futureA) + } + assertSingleAMTConflictRoundMetrics( + rounds, + expectedLosingCommitType = INCREMENTAL_CHECKPOINT, + expectedLosingTreeType = Some(INCREMENTAL_TREE), + expectedTreeOutcome = SKIP_WINNER_SATISFIES_REQUIREMENT, + expectedWinnerTreeSatisfiesRequirement = Some(true), + expectedException = Some(classOf[ConcurrentAMTCheckpointLandedException]), + numFullCheckpointWinners = 1) + + val liveIdsAfter = spark.sql(s"SELECT id FROM $name").collect().map(_.getInt(0)).toSet + assert(liveIdsAfter == liveIdsBefore, + s"the skipped checkpoint must not change the live set; before=$liveIdsBefore " + + s"after=$liveIdsAfter") + assert(amtProvider(deltaLog.update()).isDefined, + "the table must remain AMT-backed after the winner's checkpoint.") + } + } + + test("Winning Commit [Incremental checkpoint commit] vs Losing commit " + + "[incremental checkpoint commit] - SKIP") { + withTable("amt_conflict_incr_ckpt_skips") { + val name = "amt_conflict_incr_ckpt_skips" + val deltaLog = setupAMTTable(name) + val liveIdsBefore = spark.sql(s"SELECT id FROM $name").collect().map(_.getInt(0)).toSet + + // A and B are both incremental OPTIMIZE checkpoints. B wins A's target version and installs a + // new tree, so A's checkpoint is redundant: the writer signals a clean skip + // (ConcurrentAMTCheckpointLandedException) rather than deferring. A is a direct checkpoint + // commit, so that skip signal surfaces to the caller. + val rounds = trackConflictResolutionRounds { + val (futureA, futureB) = runTxnsWithOrder__A_Start__B__A_End( + optimizeCheckpointTxn(deltaLog), + optimizeCheckpointTxn(deltaLog)) + ThreadUtils.awaitResult(futureB, Duration.Inf) + assertCleanSkipSignal(futureA) + } + assertSingleAMTConflictRoundMetrics( + rounds, + expectedLosingCommitType = INCREMENTAL_CHECKPOINT, + expectedLosingTreeType = Some(INCREMENTAL_TREE), + expectedTreeOutcome = SKIP_WINNER_SATISFIES_REQUIREMENT, + expectedWinnerTreeSatisfiesRequirement = Some(true), + expectedException = Some(classOf[ConcurrentAMTCheckpointLandedException]), + numIncrementalCheckpointWinners = 1) + + val liveIdsAfter = spark.sql(s"SELECT id FROM $name").collect().map(_.getInt(0)).toSet + assert(liveIdsAfter == liveIdsBefore, + s"the skipped checkpoint must not change the live set; before=$liveIdsBefore " + + s"after=$liveIdsAfter") + assert(amtProvider(deltaLog.update()).isDefined, + "the table must remain AMT-backed after the winner's checkpoint.") + } + } + + test("Winning Commit [Full checkpoint commit] vs Losing commit " + + "[full checkpoint commit] - SKIP") { + withTable("amt_conflict_full_ckpt_skips") { + val name = "amt_conflict_full_ckpt_skips" + val deltaLog = setupAMTTable(name) + + // A and B are both full OPTIMIZE checkpoints. B wins A's target version and installs a + // full-rewrite tree that already provides an up-to-date AMT, so A's full checkpoint is + // redundant and cleanly skips (as opposed to the full-vs-incremental-winner case, which + // defers). A is a direct checkpoint commit, so the skip signal surfaces to the caller. + val rounds = trackConflictResolutionRounds { + val (futureA, futureB) = runTxnsWithOrder__A_Start__B__A_End( + fullCheckpointTxn(deltaLog), + fullCheckpointTxn(deltaLog)) + ThreadUtils.awaitResult(futureB, Duration.Inf) + assertCleanSkipSignal(futureA) + } + assertSingleAMTConflictRoundMetrics( + rounds, + expectedLosingCommitType = FULL_CHECKPOINT, + expectedLosingTreeType = Some(FULL_TREE), + expectedTreeOutcome = SKIP_WINNER_SATISFIES_REQUIREMENT, + expectedWinnerTreeSatisfiesRequirement = Some(true), + expectedException = Some(classOf[ConcurrentAMTCheckpointLandedException]), + numFullCheckpointWinners = 1) + assert(amtProvider(deltaLog.update()).isDefined, + "the table must remain AMT-backed after the winner's checkpoint.") + } + } + + test("Winning Commit [Incremental checkpoint commit] vs Losing commit " + + "[full checkpoint commit] - regenerate") { + withTable("amt_conflict_full_ckpt_regenerates_vs_incr_winner") { + val name = "amt_conflict_full_ckpt_regenerates_vs_incr_winner" + val deltaLog = setupAMTTable(name) + + // A is a full OPTIMIZE checkpoint; B installs an incremental tree and wins A's target + // version. B's incremental tree neither satisfies A's requested full AMT nor serves as a full + // base to fold onto, so A signals a full-AMT regenerate (FullAMTWriteFailedWithConflict) for + // the maintenance path to refresh and retry -- distinct from a clean skip. A is a direct + // checkpoint commit, so the signal surfaces to the caller. + val rounds = trackConflictResolutionRounds { + val (futureA, futureB) = runTxnsWithOrder__A_Start__B__A_End( + fullCheckpointTxn(deltaLog), + optimizeCheckpointTxn(deltaLog)) + ThreadUtils.awaitResult(futureB, Duration.Inf) + assertRetrySignal(futureA) + } + assertSingleAMTConflictRoundMetrics( + rounds, + expectedLosingCommitType = FULL_CHECKPOINT, + expectedLosingTreeType = Some(FULL_TREE), + expectedTreeOutcome = RETRY_VIA_NEW_TXN, + expectedWinnerTreeSatisfiesRequirement = Some(false), + expectedException = Some(classOf[FullAMTWriteFailedWithConflict]), + numIncrementalCheckpointWinners = 1) + } + } + + test("Winning Commits [Full checkpoint then Incremental checkpoint] vs Losing commit " + + "[full checkpoint commit] - SKIP") { + withTable("amt_conflict_full_ckpt_skips_full_then_incr_winners") { + val name = "amt_conflict_full_ckpt_skips_full_then_incr_winners" + val deltaLog = setupAMTTable(name) + val liveIdsBefore = spark.sql(s"SELECT id FROM $name").collect().map(_.getInt(0)).toSet + + // A is a full OPTIMIZE checkpoint. Two winners land ahead of it in one conflicted range: B + // installs a full-rewrite tree, then C folds an incremental tree on top of B. The latest + // installed tree is therefore incremental, but a full rewrite (B) already happened in A's + // conflicted range, so A's periodic full-rewrite intent is satisfied and it cleanly skips + // (ConcurrentAMTCheckpointLandedException) rather than regenerating. Deciding this off only + // the latest installed tree (C's incremental) would wrongly signal a full-AMT regenerate. + val rounds = trackConflictResolutionRounds { + val (futureA, futureB, futureC) = runTxnsWithOrder__A_Start__B__C__A_End( + fullCheckpointTxn(deltaLog), + fullCheckpointTxn(deltaLog), + optimizeCheckpointTxn(deltaLog)) + ThreadUtils.awaitResult(futureB, Duration.Inf) + ThreadUtils.awaitResult(futureC, Duration.Inf) + assertCleanSkipSignal(futureA) + } + assertSingleAMTConflictRoundMetrics( + rounds, + expectedLosingCommitType = FULL_CHECKPOINT, + expectedLosingTreeType = Some(FULL_TREE), + expectedTreeOutcome = SKIP_WINNER_SATISFIES_REQUIREMENT, + expectedWinnerTreeSatisfiesRequirement = Some(true), + expectedException = Some(classOf[ConcurrentAMTCheckpointLandedException]), + numIncrementalCheckpointWinners = 1, + numFullCheckpointWinners = 1) + + val liveIdsAfter = spark.sql(s"SELECT id FROM $name").collect().map(_.getInt(0)).toSet + assert(liveIdsAfter == liveIdsBefore, + s"the skipped checkpoint must not change the live set; before=$liveIdsBefore " + + s"after=$liveIdsAfter") + assert(amtProvider(deltaLog.update()).isDefined, + "the table must remain AMT-backed after the winners' checkpoints.") + } + } + + test("Winning Commits [Incremental checkpoint then Full checkpoint] vs Losing commit " + + "[full checkpoint commit] - SKIP") { + withTable("amt_conflict_full_ckpt_skips_incr_then_full_winners") { + val name = "amt_conflict_full_ckpt_skips_incr_then_full_winners" + val deltaLog = setupAMTTable(name) + val liveIdsBefore = spark.sql(s"SELECT id FROM $name").collect().map(_.getInt(0)).toSet + + // The mirror of the previous case: A is a full OPTIMIZE checkpoint and two winners land ahead + // of it in one conflicted range, but now B folds an incremental tree first and C then writes + // a full-rewrite tree last. A full rewrite (C) still happened in A's conflicted range, so A's + // periodic full-rewrite intent is satisfied and it cleanly skips regardless of the earlier + // incremental winner. Here the full winner is also the latest installed tree, so the skip + // decision holds whether the full rewrite is the earliest or the latest winner in the range. + val rounds = trackConflictResolutionRounds { + val (futureA, futureB, futureC) = runTxnsWithOrder__A_Start__B__C__A_End( + fullCheckpointTxn(deltaLog), + optimizeCheckpointTxn(deltaLog), + fullCheckpointTxn(deltaLog)) + ThreadUtils.awaitResult(futureB, Duration.Inf) + ThreadUtils.awaitResult(futureC, Duration.Inf) + assertCleanSkipSignal(futureA) + } + assertSingleAMTConflictRoundMetrics( + rounds, + expectedLosingCommitType = FULL_CHECKPOINT, + expectedLosingTreeType = Some(FULL_TREE), + expectedTreeOutcome = SKIP_WINNER_SATISFIES_REQUIREMENT, + expectedWinnerTreeSatisfiesRequirement = Some(true), + expectedException = Some(classOf[ConcurrentAMTCheckpointLandedException]), + numIncrementalCheckpointWinners = 1, + numFullCheckpointWinners = 1) + + val liveIdsAfter = spark.sql(s"SELECT id FROM $name").collect().map(_.getInt(0)).toSet + assert(liveIdsAfter == liveIdsBefore, + s"the skipped checkpoint must not change the live set; before=$liveIdsBefore " + + s"after=$liveIdsAfter") + assert(amtProvider(deltaLog.update()).isDefined, + "the table must remain AMT-backed after the winners' checkpoints.") + } + } + + test("a full OPTIMIZE checkpoint signals a regenerate past a non-base-preserving winner") { + withTable("amt_conflict_full_ckpt_regenerate") { + val name = "amt_conflict_full_ckpt_regenerate" + // setupAMTTable seeds ids 1..3 as separate files before A reads, so their + // defaultRowCommitVersion predates A's read version (the full base A rewrote). + val deltaLog = setupAMTTable(name) + + // A is a full OPTIMIZE checkpoint. B deletes id = 1, rewriting a file that predates A's base + // (its defaultRowCommitVersion < the base version), so B is NOT base-preserving: A cannot + // recommit the base as-is and signals a full-AMT regenerate (FullAMTWriteFailedWithConflict). + // A is driven as a direct checkpoint commit here, so that signal surfaces to the caller + // (folding the winner into the reused base is Part 4.5; CheckpointHook's refresh-and-retry is + // covered below). + val rounds = trackConflictResolutionRounds { + val (futureA, futureB) = runTxnsWithOrder__A_Start__B__A_End( + fullCheckpointTxn(deltaLog), + deleteTxn(name, id = 1)) + ThreadUtils.awaitResult(futureB, Duration.Inf) + assertRetrySignal(futureA) + } + assertSingleAMTConflictRoundMetrics( + rounds, + expectedLosingCommitType = FULL_CHECKPOINT, + expectedLosingTreeType = Some(FULL_TREE), + expectedTreeOutcome = RETRY_VIA_NEW_TXN, + expectedWinnerTreeSatisfiesRequirement = None, + expectedException = Some(classOf[FullAMTWriteFailedWithConflict]), + numLogOnlyWinners = 1, + allLogWinnersPreserveLosingTree = Some(false)) + assert(amtProvider(deltaLog.update()).isDefined, + "the table must remain AMT-backed after the winner's commit.") + } + } + + + private def runTwoTreeRoundsReuseRegenerate(loserInlinesTree: Boolean): Unit = { + val tableSuffix = if (loserInlinesTree) "inline" else "log" + withTable(s"amt_conflict_two_tree_rounds_$tableSuffix") { + val name = s"amt_conflict_two_tree_rounds_$tableSuffix" + // Seed a MIXED tree so a rebase can both reuse and regenerate back references: a full + // checkpoint packs whole leaves, then an incremental checkpoint promotes two files into the + // root (root-resident, no back reference). A trailing log insert is left pending so round 1's + // incremental winner has a new file to fold. + createAMTTable(name, checkpointInterval = noAutoCheckpointInterval) + val deltaLog = deltaLogForName(name) + val firstRootId = leafPackedFiles + 1 + val trailingId = firstRootId + 2 + withSQLConf(leafPackingConfs: _*) { + appendRowsAsSeparateFiles( + name, + numFiles = leafPackedFiles, + startId = 1) + commitCheckpoint(deltaLog, incremental = false) + val fullCheckpointProvider = amtProvider(deltaLog.update()).get + assertLeafCount(fullCheckpointProvider.leaves) + appendRowsAsSeparateFiles( + name, + numFiles = 2, + startId = firstRootId) + commitCheckpoint(deltaLog, incremental = true) + val incrementalCheckpointProvider = amtProvider(deltaLog.update()).get + val fullLeafLocations = fullCheckpointProvider.leaves.map(_.location).toSet + val incrementalLeafLocations = incrementalCheckpointProvider.leaves.map(_.location).toSet + assert(incrementalLeafLocations == fullLeafLocations, + s"expected the incremental checkpoint to preserve the full checkpoint leaves; " + + s"fullLeaves=${fullCheckpointProvider.leaves} " + + s"incrementalLeaves=${incrementalCheckpointProvider.leaves}") + appendRowsAsSeparateFiles( + name, + numFiles = 1, + startId = trailingId) + } + val liveIdsBefore = spark.sql(s"SELECT id FROM $name").collect().map(_.getInt(0)).toSet + + // A (a non-blind DELETE) removes id 2 (leaf-resident) and one root-resident file, then loses + // two tree-installing rounds. Round 1's winner B is an incremental checkpoint that leaves the + // leaves untouched, so A reuses id 2's still-valid leaf back reference and regenerates only + // the root-resident file's one. Round 2's winner C is a FULL rewrite that moves every leaf, + // so A regenerates BOTH. A TestBarrier pauses A right after round 1 so C can win A's rebased + // target version and force the second round. + val deleteLoser = + () => spark.sql(s"DELETE FROM $name WHERE id IN (2, $firstRootId)").collect() + val label = AMTWriterManager.REBASE_BACK_REFERENCES_TEST_BARRIER + // A low inline threshold forces A's DELETE to inline its own tree instead of staying + // log-only, so a log and an inline-AMT losing commit both exercise the same back-ref rebase. + def maybeInline[T](body: => T): T = + if (loserInlinesTree) withInlineThreshold(1)(body) else body + val rounds = maybeInline { + trackConflictResolutionRounds { + val Seq(futureA) = + runFunctionsWithOrderingFromObserver(Seq(deleteLoser)) { + case observerA :: Nil => + // A reads the table, then is held before it commits. + unblockUntilPreCommit(observerA) + waitForPrecommit(observerA) + // B installs a new tree (incremental OPTIMIZE checkpoint) and wins A's version. + commitCheckpoint(deltaLog, incremental = true) + // Arm the barrier so A pauses right after folding B, before its retry write. + val barrier = + DeltaTestBarrier.createBarrier(spark.sessionState.conf, label) + try { + unblockCommit(observerA) + barrier.waitUntilBlocked() + // C installs a FULL-rewrite tree and wins A's rebased target version. + commitCheckpoint(deltaLog, incremental = false) + } finally { + DeltaTestBarrier.destroyBarrier(label) + } + waitForCommit(observerA) + } + ThreadUtils.awaitResult(futureA, Duration.Inf) + } + } + + // A survived after rebasing past both tree winners; only its two target ids are gone. + val liveIdsAfter = spark.sql(s"SELECT id FROM $name").collect().map(_.getInt(0)).toSet + assert(liveIdsAfter == liveIdsBefore -- Set(2, firstRootId), + s"A's delete must survive both tree rebases; before=$liveIdsBefore after=$liveIdsAfter") + assert(rounds.size == 2, + s"expected two AMT conflict rounds across two tree-installing winners, got $rounds") + val Seq(round1, round2) = rounds + // Round 1's incremental winner left the leaves untouched: reuse id 2's leaf back reference, + // regenerate only the root-resident file's one. + assert(!round1.backRefRebaseMetrics.skipped) + assert(round1.backRefRebaseMetrics.skipReason.isEmpty) + assert(round1.backRefRebaseMetrics.numActionsReusingBackref.contains(1) && + round1.backRefRebaseMetrics.numActionsRegeneratingBackref.contains(1), + s"unexpected round-1 back-reference split; round=$round1") + // Round 2's full rewrite moved every leaf, so both back references are regenerated. + assert(!round2.backRefRebaseMetrics.skipped) + assert(round2.backRefRebaseMetrics.skipReason.isEmpty) + assert(round2.backRefRebaseMetrics.numActionsReusingBackref.contains(0) && + round2.backRefRebaseMetrics.numActionsRegeneratingBackref.contains(2), + s"unexpected round-2 back-reference split; round=$round2") + } + } + + test("Losing commit rebasing through two tree-installing conflict rounds reuses and " + + "regenerates back references (log loser)") { + runTwoTreeRoundsReuseRegenerate(loserInlinesTree = false) + } + + test("Losing commit rebasing through two tree-installing conflict rounds reuses and " + + "regenerates back references (inline-AMT loser)") { + runTwoTreeRoundsReuseRegenerate(loserInlinesTree = true) + } + // A losing OPTIMIZE checkpoint -- full or incremental -- whose winner rewrote a pre-base file + // (non-base-preserving) cannot reuse its base, so it rebuilds the tree rather than swallowing the + // conflict or surfacing an error. A losing incremental checkpoint recreates itself in the same + // transaction (its base is unchanged and its fold window is small). A losing full checkpoint + // signals FullAMTWriteFailedWithConflict, and deltaLog.checkpoint refreshes (deltaLog.update()) + // and retries, regenerating the full AMT against the post-winner snapshot. Either way the rebuilt + // tree keeps its full/incremental characteristic. + for ((label, triggerMode, regeneratedTreeIsIncremental) <- Seq( + ("full", AMTTriggerMode.CheckpointIntervalFull, false), + ("incremental", AMTTriggerMode.CheckpointIntervalIncremental, true))) { + test(s"deltaLog.checkpoint rebuilds a $label checkpoint that lost to a " + + "non-base-preserving winner") { + withTable(s"amt_ckpt_hook_regenerates_$label") { + val name = s"amt_ckpt_hook_regenerates_$label" + val deltaLog = setupAMTTable(name) + // The maintenance checkpoint reads this snapshot; a winner commits past it first. + val staleSnapshot = deltaLog.update() + + // A concurrent winner deletes id = 1 -- a log commit rewriting a file predating the base. + spark.sql(s"DELETE FROM $name WHERE id = 1").collect() + val versionBeforeRetry = deltaLog.update().version + + val rounds = trackConflictResolutionRounds { + deltaLog.checkpoint( + staleSnapshot, catalogTableOpt = None, amtTriggerModeOpt = Some(triggerMode)) + } + assertSingleAMTConflictRoundMetrics( + rounds, + expectedLosingCommitType = + if (regeneratedTreeIsIncremental) INCREMENTAL_CHECKPOINT else FULL_CHECKPOINT, + expectedLosingTreeType = + Some(if (regeneratedTreeIsIncremental) INCREMENTAL_TREE else FULL_TREE), + // The incremental loser rebuilds its tree in-transaction + // (REBUILT_OPTIMIZE_CHECKPOINT_INCREMENTAL_TREE, no exception); the full loser signals a + // regenerate for deltaLog.checkpoint to retry. + expectedTreeOutcome = + if (regeneratedTreeIsIncremental) REBUILT_OPTIMIZE_CHECKPOINT_INCREMENTAL_TREE + else RETRY_VIA_NEW_TXN, + expectedWinnerTreeSatisfiesRequirement = None, + expectedException = + if (regeneratedTreeIsIncremental) None + else Some(classOf[FullAMTWriteFailedWithConflict]), + numLogOnlyWinners = 1, + allLogWinnersPreserveLosingTree = Some(false)) + + // The rebuild committed a fresh checkpoint past the winner (not a silent no-op), the delete + // survived, and the table stays AMT-backed. + val latest = deltaLog.update() + assert(latest.version > versionBeforeRetry, + s"the rebuild must commit a new checkpoint past the winner; before=$versionBeforeRetry " + + s"after=${latest.version}") + val checkpoint = checkpointAt(deltaLog, latest.version).getOrElse( + fail("the rebuilt checkpoint must carry an AMT checkpoint at its version.")) + assert(checkpoint.contentRoot.isIncremental.contains(regeneratedTreeIsIncremental), + s"the rebuilt checkpoint must be a fresh $label tree.") + val liveIdsAfter = spark.sql(s"SELECT id FROM $name").collect().map(_.getInt(0)).toSet + assert(!liveIdsAfter.contains(1) && liveIdsAfter.contains(2) && liveIdsAfter.contains(3), + s"the delete of id=1 must survive the regenerated checkpoint; after=$liveIdsAfter") + assert(amtProvider(latest).isDefined, + "the table must remain AMT-backed after the regenerated checkpoint.") + } + } + } + + test("deltaLog.checkpoint skips a maintenance checkpoint that lost to a tree-installing winner") { + withTable("amt_ckpt_hook_skips_vs_tree_winner") { + val name = "amt_ckpt_hook_skips_vs_tree_winner" + val deltaLog = setupAMTTable(name) + // The maintenance checkpoint reads this snapshot; a tree-installing winner commits past it. + val staleSnapshot = deltaLog.update() + + // A concurrent winner installs a brand-new full AMT tree, superseding what the losing + // checkpoint would write. + commitCheckpoint(deltaLog, incremental = false) + val versionAfterWinner = deltaLog.update().version + val liveIdsBefore = spark.sql(s"SELECT id FROM $name").collect().map(_.getInt(0)).toSet + + // The losing checkpoint reads the stale snapshot, finds the winner's tree already serves as + // an up-to-date AMT, and skips as a graceful no-op: emitAMTCheckpoint suppresses the + // ConcurrentAMTCheckpointLandedException rather than surfacing it or committing a new tree. + val rounds = trackConflictResolutionRounds { + deltaLog.checkpoint( + staleSnapshot, catalogTableOpt = None, + amtTriggerModeOpt = Some(AMTTriggerMode.CheckpointIntervalFull)) + } + assertSingleAMTConflictRoundMetrics( + rounds, + expectedLosingCommitType = FULL_CHECKPOINT, + expectedLosingTreeType = Some(FULL_TREE), + expectedTreeOutcome = SKIP_WINNER_SATISFIES_REQUIREMENT, + expectedWinnerTreeSatisfiesRequirement = Some(true), + expectedException = Some(classOf[ConcurrentAMTCheckpointLandedException]), + numFullCheckpointWinners = 1) + + val latest = deltaLog.update() + assert(latest.version == versionAfterWinner, + s"the skipped checkpoint must not commit a new version; winner=$versionAfterWinner " + + s"after=${latest.version}") + val liveIdsAfter = spark.sql(s"SELECT id FROM $name").collect().map(_.getInt(0)).toSet + assert(liveIdsAfter == liveIdsBefore, + s"the skipped checkpoint must not change the live set; before=$liveIdsBefore " + + s"after=$liveIdsAfter") + assert(amtProvider(latest).isDefined, + "the table must remain AMT-backed via the winner's checkpoint after the skip.") + } + } + + test("deltaLog.checkpoint retries a full checkpoint that lost to an incremental winner") { + withTable("amt_ckpt_hook_retries_vs_incr_winner") { + val name = "amt_ckpt_hook_retries_vs_incr_winner" + val deltaLog = setupAMTTable(name) + // The maintenance checkpoint reads this snapshot; an incremental-tree winner commits past it. + val staleSnapshot = deltaLog.update() + + // A concurrent winner installs an incremental AMT tree. It does not satisfy the requested + // full AMT and cannot serve as a full base to fold onto, so the full checkpoint must + // regenerate against the refreshed snapshot rather than skip. + commitCheckpoint(deltaLog, incremental = true) + val versionBeforeRetry = deltaLog.update().version + + val rounds = trackConflictResolutionRounds { + deltaLog.checkpoint( + staleSnapshot, catalogTableOpt = None, + amtTriggerModeOpt = Some(AMTTriggerMode.CheckpointIntervalFull)) + } + assertSingleAMTConflictRoundMetrics( + rounds, + expectedLosingCommitType = FULL_CHECKPOINT, + expectedLosingTreeType = Some(FULL_TREE), + expectedTreeOutcome = RETRY_VIA_NEW_TXN, + expectedWinnerTreeSatisfiesRequirement = Some(false), + expectedException = Some(classOf[FullAMTWriteFailedWithConflict]), + numIncrementalCheckpointWinners = 1) + + // The retry committed a fresh full checkpoint past the winner (not a skip / no-op). + val latest = deltaLog.update() + assert(latest.version > versionBeforeRetry, + s"the retry must commit a new full checkpoint past the incremental winner; " + + s"before=$versionBeforeRetry after=${latest.version}") + val checkpoint = checkpointAt(deltaLog, latest.version).getOrElse( + fail("the retried checkpoint must carry an AMT checkpoint at its version.")) + assert(checkpoint.contentRoot.isIncremental.contains(false), + "the regenerated checkpoint must be a fresh full tree.") + assert(amtProvider(latest).isDefined, + "the table must remain AMT-backed after the regenerated checkpoint.") + } + } + + test("deltaLog.checkpoint stops regenerating a full checkpoint after the configured " + + "retry bound") { + withTable("amt_ckpt_hook_retry_bound") { + val name = "amt_ckpt_hook_retry_bound" + val deltaLog = setupAMTTable(name) // seeds ids 1, 2, 3 + val maxRetries = 3 + withSQLConf( + DeltaSQLConf.AMT_CONFLICT_CHECKING_MAX_FULL_REGENERATE_RETRIES.key -> + maxRetries.toString) { + val staleSnapshot = deltaLog.update() + // A non-base-preserving winner lands before the maintenance checkpoint commits, so its + // first attempt signals FullAMTWriteFailedWithConflict. + spark.sql(s"DELETE FROM $name WHERE id = 1").collect() + + // The barrier pauses the checkpoint on each regenerate retry (after it refreshes its + // snapshot). The test lands one more non-base-preserving winner per retry so every attempt + // keeps conflicting; after `maxRetries` attempts the loop gives up and the last + // FullAMTWriteFailedWithConflict propagates. + val label = AMTWriterManager.FULL_AMT_REGENERATE_RETRY_TEST_BARRIER + val barrier = DeltaTestBarrier.createBarrier(spark.sessionState.conf, label) + @volatile var caught: Throwable = null + val checkpointThread = new Thread(() => { + SparkSession.setActiveSession(spark) + try { + deltaLog.checkpoint( + staleSnapshot, + catalogTableOpt = None, + amtTriggerModeOpt = Some(AMTTriggerMode.CheckpointIntervalFull)) + } catch { + case t: Throwable => caught = t + } + }) + checkpointThread.setDaemon(true) + val rounds = trackConflictResolutionRounds { + try { + checkpointThread.start() + // Drive maxRetries - 1 retries: each hits the barrier, lands a fresh conflicting + // winner, and is released. The final attempt then conflicts with no retries left and + // propagates. + (2 to maxRetries).foreach { deleteId => + barrier.waitUntilBlocked() + spark.sql(s"DELETE FROM $name WHERE id = $deleteId").collect() + barrier.releaseOne() + // Let the released retry leave the barrier before waiting on the next hit, so the + // next waitUntilBlocked() can't observe this now-stale block. + val deadlineMs = System.currentTimeMillis() + 120000L + while (barrier.getNumBlockedThreads > 0) { + assert(System.currentTimeMillis() < deadlineMs, + "timed out waiting for the retry to leave the barrier.") + Thread.sleep(50) + } + } + } finally { + checkpointThread.join(120000L) + DeltaTestBarrier.destroyBarrier(label) + } + } + assert(rounds.size == maxRetries, + s"expected one metric per failed checkpoint attempt, got $rounds") + rounds.foreach { round => + assertSingleAMTConflictRoundMetrics( + Seq(round), + expectedLosingCommitType = FULL_CHECKPOINT, + expectedLosingTreeType = Some(FULL_TREE), + expectedTreeOutcome = RETRY_VIA_NEW_TXN, + expectedWinnerTreeSatisfiesRequirement = None, + expectedException = Some(classOf[FullAMTWriteFailedWithConflict]), + numLogOnlyWinners = 1, + allLogWinnersPreserveLosingTree = Some(false)) + } + assert(!checkpointThread.isAlive, + "the retry loop must terminate once the bound is exhausted.") + val causes = caught match { + case null => Nil + case t => Iterator + .iterate[Throwable](t)(c => if (c.getCause eq c) null else c.getCause) + .takeWhile(_ != null).take(50).toList + } + assert(causes.exists(_.isInstanceOf[FullAMTWriteFailedWithConflict]), + s"the last conflict past the retry bound must propagate a " + + s"FullAMTWriteFailedWithConflict; got $caught") + } + } + } +} diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTContentStatsSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTContentStatsSuite.scala new file mode 100644 index 00000000000..8b336372073 --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTContentStatsSuite.scala @@ -0,0 +1,505 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.amt + +import org.apache.spark.sql.delta.{CatalogOwnedTableFeature, ColumnMappingTableFeature, DeletionVectorsTableFeature, DeltaColumnMapping, DeltaConfigs, DomainMetadataTableFeature, RowTrackingFeature} +import org.apache.spark.sql.delta.actions.{AddFile, Metadata, Protocol} +import org.apache.spark.sql.delta.util.JsonUtils +import org.apache.hadoop.fs.Path + +import org.apache.spark.sql.DataFrame +import org.apache.spark.sql.execution.datasources.parquet.ParquetUtils +import org.apache.spark.sql.functions.{col, lit} +import org.apache.spark.sql.types.{BooleanType, ByteType, DataType, DateType, DecimalType, DoubleType, FloatType, IntegerType, LongType, ShortType, StringType, StructField, StructType, TimestampNTZType, TimestampType, VariantType} + +/** + * Tests for [[AMTContentStats]] -- the conversion between Delta's stats JSON string and the typed + * Iceberg V4 `content_stats` struct. `forWrite` and `forRead` are exercised end to end against a + * real all-types table (see [[withAllTypesTable]]); the edge cases -- unsupported bound types, a + * table that collects no statistics, and the `tightBounds` reconstruction -- run on synthesized + * `(Metadata, Protocol)` fixtures (see [[metadataWithColumnIds]]). + */ +class AMTContentStatsSuite extends AMTCheckpointTestBase { + + import testImplicits._ + + /** + * The `content_stats` sub-struct field name for logical column `name`. Sub-structs are named by + * the logical name with the (unique) field id attached (see `StatsLeaf.fieldName`) -- e.g. + * `d_int_1` -- so a test refers to a column by its logical name and this resolves the id suffix. + * The logical names used here are unique, so exactly one sub-struct matches. + */ + private def contentStatsField(persisted: DataFrame, name: String): String = { + val matches = persisted.schema("content_stats").dataType.asInstanceOf[StructType] + .fieldNames.filter(_.matches(java.util.regex.Pattern.quote(name) + "_\\d+")) + assert(matches.length == 1, + s"expected exactly one content_stats sub-struct for $name, got ${matches.toSeq}") + matches.head + } + + /** The logical column names of a persisted `content_stats` struct, id suffix stripped. */ + private def contentStatsLogicalNames(persisted: DataFrame): Seq[String] = + persisted.schema("content_stats").dataType.asInstanceOf[StructType] + .fieldNames.toSeq.map(_.replaceAll("_\\d+$", "")) + + /** + * Asserts the `content_stats` sub-struct `forWrite` produced for column `name` has exactly the + * `expected` stat fields: each maps to its Iceberg type and the string form of its value. + */ + private def assertStatFields( + persistedDf: DataFrame, + columnName: String, + expectedSubFieldToValueMapping: Map[String, (DataType, String)]): Unit = { + val fieldName = contentStatsField(persistedDf, columnName) + val sub = persistedDf.schema("content_stats").dataType.asInstanceOf[StructType](fieldName) + .dataType.asInstanceOf[StructType] + val row = persistedDf.select(s"content_stats.$fieldName.*").head() + val actual = sub.fields.zipWithIndex.map { case (field, i) => + field.name -> (field.dataType, String.valueOf(row.get(i))) + }.toMap + assert(actual == expectedSubFieldToValueMapping, + s"content_stats.$columnName\n expected=$expectedSubFieldToValueMapping\n actual=$actual") + } + + test("forWrite persists content_stats as the typed struct the delta log's stats fill") { + withAllTypesTable("amt_content_stats_write") { deltaLog => + appendRowsAsSeparateFiles( + "amt_content_stats_write", numFiles = 1, columnExprs = allTypesTableColumnExprs) + val snapshot = deltaLog.update() + val metadata = snapshot.metadata + val protocol = snapshot.protocol + val add = liveAddFiles(snapshot).head + val entry = DataEntry.fromAddFile( + add, AMTWriteHelper.addedTrackingForDataEntry(), deltaLog.dataPath) + val input = Seq(entry.wrap).toDS().toDF() + val persisted = AMTContentStats.forWrite(input, metadata, protocol) + + assert(contentStatsLogicalNames(persisted) == Seq( + "d_int", "d_long", "d_short", "d_byte", "d_str", "d_date", "d_ts", "d_ts_ntz", "d_dec", + "d_bool", "d_float", "d_double", "d_binary", "d_arr", "d_nested_inner")) + + def bounded(name: String, dataType: DataType, value: String, tight: Boolean): Unit = + assertStatFields(persisted, name, Map( + "lower_bound" -> (dataType, value), + "upper_bound" -> (dataType, value), + "tight_bounds" -> (BooleanType, tight.toString), + "null_value_count" -> (LongType, "0"))) + def boundless(name: String): Unit = + assertStatFields(persisted, name, Map("null_value_count" -> (LongType, "0"))) + + bounded("d_int", IntegerType, "0", tight = true) + bounded("d_long", LongType, "0", tight = true) + bounded("d_short", ShortType, "0", tight = true) + bounded("d_byte", ByteType, "0", tight = true) + bounded("d_str", StringType, "row0", tight = false) + bounded("d_date", DateType, "2026-07-25", tight = true) + bounded("d_ts", TimestampType, "2026-07-25 01:02:03.456", tight = false) + bounded("d_ts_ntz", TimestampNTZType, "2026-07-25T01:02:03.456", tight = false) + bounded("d_dec", DecimalType(9, 3), "0.000", tight = true) + bounded("d_float", FloatType, "0.0", tight = true) + bounded("d_double", DoubleType, "0.0", tight = true) + bounded("d_nested_inner", IntegerType, "0", tight = true) + boundless("d_bool") + boundless("d_binary") + boundless("d_arr") + } + } + + test("forRead reconstructs the stats JSON the delta log holds, for every type") { + withAllTypesTable("amt_content_stats_roundtrip") { deltaLog => + appendRowsAsSeparateFiles( + "amt_content_stats_roundtrip", numFiles = leafPackedFiles, + columnExprs = allTypesTableColumnExprs) + commitCheckpoint(deltaLog, incremental = false) + val snapshot = deltaLog.update() + val provider = amtProvider(snapshot).getOrElse(fail("expected AMTCheckpointProvider")) + val metadata = snapshot.metadata + val protocol = snapshot.protocol + val leaves = provider.liveLeafManifestAbsolutePaths.map(_.toString) + assert(leaves.nonEmpty, "Expected at least one leaf manifest.") + + // Read the stats straight from the commit json rather than the snapshot: an AMT snapshot + // reconstructs its AddFiles through `forRead`, so comparing against it would compare it with + // itself. Compare as parsed JSON trees so field order and formatting do not matter. + val loggedSchema = StructType(Seq(StructField("add", StructType(Seq( + StructField("stats", StringType)))))) + val logged = spark.read.schema(loggedSchema) + .json(new Path(deltaLog.logPath, "*.json").toString) + .where(col("add").isNotNull) + .select(col("add.stats")) + .as[String].collect() + .map(JsonUtils.mapper.readTree).toSet + assert(logged.size == leafPackedFiles, + s"Expected one logged stats per file, got ${logged.size}.") + + // forRead reconstructs the Delta stats JSON from the leaves' DATA entries; compare as parsed + // JSON trees so field order and formatting do not matter. + val reconstructed = withManifestDataEntries(leaves) { entries => + AMTContentStats.forRead(entries, metadata, protocol) + .select(col("content_stats")) + .as[String].collect() + .map(JsonUtils.mapper.readTree).toSet + } + // Precompute the comparison and message: ScalaTest's `assert` macro fails to expand on Scala + // 2.12 when its arguments interpolate `Set[JsonNode]` values, so hand it plain Boolean/String + // values instead ("macro has not been expanded"). + val reconstructedMatchesLogged = reconstructed == logged + val mismatchClue = + s"forRead did not reproduce the logged stats.\n logged=$logged\n" + + s" reconstructed=$reconstructed" + assert(reconstructedMatchesLogged, mismatchClue) + } + } + + /** + * A stand-in table root; synthesized entries carry no DV, so it is only a + * path placeholder. + */ + private val tableRoot = new Path("file:/tmp/amt-test-table") + + /** + * A [[Protocol]] representative of a real AMT table: it carries every + * feature `AdaptiveMetadataTableFeature` requires. In particular it is + * DV-capable, so Delta's derived stats schema always carries `tightBounds`. + */ + private def amtProtocol: Protocol = + Protocol(3, 7) + .withFeature(CatalogOwnedTableFeature) + .withFeature(RowTrackingFeature) + .withFeature(DomainMetadataTableFeature) + .withFeature(DeletionVectorsTableFeature) + .withFeature(ColumnMappingTableFeature) + + /** A [[Metadata]] for `schema` in `id` column-mapping mode, with column ids + * and physical names assigned the way a real AMT table has them (AMT + * requires that mode, and the content-stats field ids are derived from those + * column ids). Stats are keyed by whatever physical name Delta assigns, so + * it does not matter that the assignment is a fresh UUID per column. + */ + private def metadataWithColumnIds( + schema: StructType, + partitionColumns: Seq[String] = Seq.empty, + configuration: Map[String, String] = Map.empty + ): Metadata = { + val base = Metadata( + id = "amt-content-stats-test", + schemaString = schema.json, + partitionColumns = partitionColumns, + configuration = configuration + + (DeltaConfigs.COLUMN_MAPPING_MODE.key -> "id") + ) + DeltaColumnMapping.assignColumnIdAndPhysicalName( + newMetadata = base, + oldMetadata = Metadata(), + isChangingModeOnExistingTable = false, + isOverwritingSchema = false + ) + } + + /** + * A Delta stats JSON string covering every data column of `metadata` (keyed by physical name, as + * Delta writes it): a placeholder min/max, a zero null count, and the given file-level + * `tightBounds`. Boundless types (variant, geometry/geography) get a null count but no min/max. + */ + private def getSampleStatsJson(metadata: Metadata, tightBounds: Boolean): String = { + def bounds(dataType: DataType): Option[(String, String)] = dataType match { + case _: IntegerType | _: LongType => Some(("1", "9")) + case _: StringType => Some(("\"apple\"", "\"pear\"")) + case _: TimestampType => + Some(("\"2020-01-01T00:00:00.000Z\"", "\"2020-12-31T00:00:00.000Z\"")) + case _: VariantType => None + case other => fail(s"getSampleStatsJson has no placeholder for $other") + } + val columns = metadata.dataSchema + def obj(value: StructField => Option[String]): String = + columns.flatMap(f => + value(f).map(v => s""""${DeltaColumnMapping.getPhysicalName(f)}":$v""")).mkString(",") + s"""{"numRecords":10, + |"minValues":{${obj(f => bounds(f.dataType).map(_._1))}}, + |"maxValues":{${obj(f => bounds(f.dataType).map(_._2))}}, + |"nullCount":{${obj(_ => Some("0"))}}, + |"tightBounds":$tightBounds}""".stripMargin + } + + /** + * Builds the [[AMTSingleAction]] input for one data file carrying `statsJson`, then runs + * [[AMTContentStats.forWrite]] to get the persisted, typed `content_stats`. + */ + private def writeStats(statsJson: String, metadata: Metadata, protocol: Protocol): DataFrame = { + val addFile = AddFile( + path = "part-00000.parquet", + partitionValues = Map("p" -> "1"), + size = 1024L, + modificationTime = 100L, + dataChange = true, + stats = statsJson) + val entry = DataEntry.fromAddFile( + addFile, AMTWriteHelper.addedTrackingForDataEntry(), tableRoot) + val input = Seq(entry.wrap).toDS().toDF() + AMTContentStats.forWrite(input, metadata, protocol) + } + + /** + * Runs [[AMTContentStats.forRead]] on a persisted DataFrame and returns the single entry's + * reconstructed Delta stats JSON. + */ + private def readStats(persisted: DataFrame, metadata: Metadata, protocol: Protocol): String = + AMTContentStats.forRead(persisted, metadata, protocol) + .as[AMTSingleAction].head() + .unwrap.asInstanceOf[DataEntry].toAddFile(tableRoot).stats + + test("variant columns persist a null count but no bounds") { + // Delta collects min/max for VARIANT, but AMTContentStats has no Iceberg V4 bound + // representation for it yet, so `isSupportedBoundType` rejects it. It is still counted for + // nulls, so it surfaces with a `null_value_count` only -- like a plain array column. + val metadata = metadataWithColumnIds( + new StructType() + .add("id", LongType) // supported bound type: full typed bounds + .add("v", VariantType)) // unsupported bound type + val protocol = amtProtocol + val persisted = + writeStats(getSampleStatsJson(metadata, tightBounds = true), metadata, protocol) + assertStatFields(persisted, "id", Map( + "lower_bound" -> (LongType, "1"), + "upper_bound" -> (LongType, "9"), + "tight_bounds" -> (BooleanType, "true"), + "null_value_count" -> (LongType, "0"))) + assertStatFields(persisted, "v", Map("null_value_count" -> (LongType, "0"))) + } + + test("content stats adapter round-trips a table that collects no statistics") { + // numIndexedCols = 0 means Delta collects only numRecords, so there are no per-column stats and + // `content_stats` is dropped from the schema (an empty struct is not Parquet-writable). + val metadata = metadataWithColumnIds( + schema = new StructType().add("id", LongType), + configuration = Map(DeltaConfigs.DATA_SKIPPING_NUM_INDEXED_COLS.key -> "0")) + val protocol = amtProtocol + + assert( + !AMTSingleAction.persistedSchema(metadata, protocol).fieldNames.contains( + "content_stats"), + "a table with no per-column statistics omits content_stats from the persisted schema") + + val persisted = writeStats("""{"numRecords":7}""", metadata, protocol) + // record_count alone reconstructs the stats Delta had. + val restoredJson = readStats(persisted, metadata, protocol) + assert(JsonUtils.mapper.readTree(restoredJson) == + JsonUtils.mapper.readTree("""{"numRecords":7}""")) + } + + test("content stats cover only the indexed columns, not every column") { + // Delta collects stats only for the first `dataSkippingNumIndexedCols` columns, so + // `statCollectionPhysicalSchema` -- and therefore `content_stats` -- covers only that prefix. + // Three columns with a limit of 2: only `a` and `b` get a sub-struct; `c` is left out. + val metadata = metadataWithColumnIds( + schema = new StructType().add("a", LongType).add("b", LongType).add("c", LongType), + configuration = Map(DeltaConfigs.DATA_SKIPPING_NUM_INDEXED_COLS.key -> "2")) + val protocol = amtProtocol + val persisted = + writeStats(getSampleStatsJson(metadata, tightBounds = true), metadata, protocol) + + assert(contentStatsLogicalNames(persisted) == Seq("a", "b"), + s"only the first 2 indexed columns get content_stats; got " + + s"${contentStatsLogicalNames(persisted)}") + assertStatFields(persisted, "a", Map( + "lower_bound" -> (LongType, "1"), + "upper_bound" -> (LongType, "9"), + "tight_bounds" -> (BooleanType, "true"), + "null_value_count" -> (LongType, "0"))) + } + + test("reading back with fewer indexed columns than were written keeps only the read subset") { + // Written with 3 indexed columns, then read back after the stats-columns config was lowered to + // 2 (same table, physical names/ids unchanged). `forRead` derives its leaves from the read + // metadata, so it projects only a and b; c's typed sub-struct is still on disk but drops out of + // the reconstructed Delta stats. A superset of persisted stats is never an error, just unused. + val schema = new StructType().add("a", LongType).add("b", LongType).add("c", LongType) + val writeMetadata = metadataWithColumnIds( + schema, configuration = Map(DeltaConfigs.DATA_SKIPPING_NUM_INDEXED_COLS.key -> "3")) + val readMetadata = writeMetadata.copy( + configuration = writeMetadata.configuration + + (DeltaConfigs.DATA_SKIPPING_NUM_INDEXED_COLS.key -> "2")) + val protocol = amtProtocol + + val persisted = + writeStats(getSampleStatsJson(writeMetadata, tightBounds = true), writeMetadata, protocol) + assert(contentStatsLogicalNames(persisted) == Seq("a", "b", "c"), + s"written with 3 indexed columns, content_stats should carry a, b, c; got " + + s"${contentStatsLogicalNames(persisted)}") + + val physical = readMetadata.dataSchema + .map(f => f.name -> DeltaColumnMapping.getPhysicalName(f)).toMap + val minValues = JsonUtils.mapper.readTree(readStats(persisted, readMetadata, protocol)) + .get("minValues") + assert( + minValues.has(physical("a")) && minValues.has(physical("b")) && !minValues.has(physical("c")), + s"reading back with 2 indexed columns keeps only a and b, dropping c; got $minValues") + } + + test("the dataSkippingStatsColumns allowlist selects which columns get content_stats") { + // The `delta.dataSkippingStatsColumns` table property is an explicit allowlist that overrides + // the first-N-columns default (`dataSkippingNumIndexedCols`). Allowlisting `b,c` (skipping the + // leading `a`) proves the selection follows the property, not a column prefix. + val metadata = metadataWithColumnIds( + schema = new StructType().add("a", LongType).add("b", LongType).add("c", LongType), + configuration = Map(DeltaConfigs.DATA_SKIPPING_STATS_COLUMNS.key -> "b,c")) + val protocol = amtProtocol + val persisted = + writeStats(getSampleStatsJson(metadata, tightBounds = true), metadata, protocol) + + assert(contentStatsLogicalNames(persisted) == Seq("b", "c"), + s"only the allowlisted columns get content_stats; got " + + s"${contentStatsLogicalNames(persisted)}") + assertStatFields(persisted, "b", Map( + "lower_bound" -> (LongType, "1"), + "upper_bound" -> (LongType, "9"), + "tight_bounds" -> (BooleanType, "true"), + "null_value_count" -> (LongType, "0"))) + assertStatFields(persisted, "c", Map( + "lower_bound" -> (LongType, "1"), + "upper_bound" -> (LongType, "9"), + "tight_bounds" -> (BooleanType, "true"), + "null_value_count" -> (LongType, "0"))) + } + + test("content stats round-trip a schema whose flattened leaf names collide") { + // A top-level column `a_b` and a nested leaf `a`.`b` both flatten to the logical name "a_b". + // Naming each content_stats sub-struct by that flattened name makes the two collide: the + // persisted struct gets two "a_b" fields and `forRead`'s by-name lookup is ambiguous. Keying + // the sub-structs by the (unique) column id keeps them distinct. This asserts each column's + // stats reconstruct into its own physical-name slot -- it fails while the sub-structs are named + // by the flattened logical name and passes once they are keyed by column id. + val schema = new StructType() + .add("a_b", LongType) + .add("a", new StructType().add("b", LongType)) + val metadata = metadataWithColumnIds(schema) + val protocol = amtProtocol + + val physTop = DeltaColumnMapping.getPhysicalName(metadata.dataSchema("a_b")) + val structA = metadata.dataSchema("a") + val physA = DeltaColumnMapping.getPhysicalName(structA) + val physB = DeltaColumnMapping.getPhysicalName(structA.dataType.asInstanceOf[StructType]("b")) + + // Distinct bounds per column so a mixed-up mapping is detectable: a_b -> [1,9], a.b -> [2,8]. + val statsJson = + s"""{"numRecords":10, + |"minValues":{"$physTop":1,"$physA":{"$physB":2}}, + |"maxValues":{"$physTop":9,"$physA":{"$physB":8}}, + |"nullCount":{"$physTop":0,"$physA":{"$physB":0}}, + |"tightBounds":true}""".stripMargin + + val persisted = writeStats(statsJson, metadata, protocol) + val reconstructed = JsonUtils.mapper.readTree(readStats(persisted, metadata, protocol)) + val minValues = reconstructed.get("minValues") + val maxValues = reconstructed.get("maxValues") + + assert(minValues.get(physTop).asLong == 1 && maxValues.get(physTop).asLong == 9, + s"top-level a_b stats must reconstruct into its own slot; got $reconstructed") + assert(minValues.get(physA).get(physB).asLong == 2 && + maxValues.get(physA).get(physB).asLong == 8, + s"nested a.b stats must reconstruct into its own slot; got $reconstructed") + } + + /** The persisted Iceberg `tight_bounds` of the sub-struct for logical column `column`. */ + private def persistedTightBounds(persisted: DataFrame, column: String): Boolean = { + val sub = col("content_stats").getField(contentStatsField(persisted, column)) + persisted.select(sub.getField("tight_bounds")).as[Boolean].head() + } + + /** The file-level `tightBounds` recorded in a Delta stats JSON, or None if absent. */ + private def tightBoundsOf(statsJson: String): Option[Boolean] = + Option(JsonUtils.mapper.readTree(statsJson).get("tightBounds")).map(_.asBoolean) + + test("all columns have truncated type: Delta tightBounds is reconstructed conservatively false") { + // The only bounded columns are string/timestamp. Each is forced to tight_bounds=false on write, + // and on read there is no untruncated column to recover the file flag from, so it falls back + // to false -- a safe precision loss even though the source file was tight. + val metadata = metadataWithColumnIds( + new StructType().add("s", StringType).add("ts", TimestampType)) + Seq(true, false).foreach { tight => + val persisted = + writeStats(getSampleStatsJson(metadata, tightBounds = tight), metadata, amtProtocol) + assert(!persistedTightBounds(persisted, "s"), "a string column is never tight") + assert(!persistedTightBounds(persisted, "ts"), "a timestamp column is never tight") + assert(tightBoundsOf(readStats(persisted, metadata, amtProtocol)).contains(false), + "with no untruncated column to vouch for it, the file flag reconstructs to false") + } + } + + test("no truncated column types: the Delta tightBounds round-trips exactly") { + // Every column is untruncated, so each carries the file flag verbatim and the + // reconstructed flag matches -- tight and wide both round-trip unchanged. + val metadata = metadataWithColumnIds( + new StructType().add("id", LongType).add("n", IntegerType)) + Seq(true, false).foreach { tight => + val persisted = + writeStats(getSampleStatsJson(metadata, tightBounds = tight), metadata, amtProtocol) + assert(persistedTightBounds(persisted, "id") == tight) + assert(persistedTightBounds(persisted, "n") == tight) + assert(tightBoundsOf(readStats(persisted, metadata, amtProtocol)).contains(tight), + s"an all-untruncated table must round-trip tightBounds=$tight unchanged") + } + } + + test("mixed column types: the Delta tightBounds round-trips via the untruncated column") { + // A mix of an untruncated (long) and a truncated (string) column. The untruncated column + // carries the file flag, the string is never tight, and the reconstructed flag comes back from + // the untruncated column -- so both tight and wide round-trip unchanged. + val metadata = metadataWithColumnIds( + new StructType().add("id", LongType).add("name", StringType)) + Seq(true, false).foreach { tight => + val persisted = + writeStats(getSampleStatsJson(metadata, tightBounds = tight), metadata, amtProtocol) + assert(persistedTightBounds(persisted, "id") == tight, + "the untruncated column carries the file flag") + assert(!persistedTightBounds(persisted, "name"), "a string column is never tight") + assert(tightBoundsOf(readStats(persisted, metadata, amtProtocol)).contains(tight), + s"a mixed table must round-trip tightBounds=$tight via the untruncated column") + } + } + + test("reading back to Delta ignores a string column's Iceberg tight_bounds") { + val metadata = metadataWithColumnIds(new StructType().add("name", StringType)) + // `forWrite` never marks a string tight, so this happens only when an external writer sets a + // string column with tight_bounds=true. + val persisted = + writeStats(getSampleStatsJson(metadata, tightBounds = true), metadata, amtProtocol) + val tampered = persisted.withColumn("content_stats", + col("content_stats").withField(s"${contentStatsField(persisted, "name")}.tight_bounds", + lit(true))) + assert(persistedTightBounds(tampered, "name"), "the tampered Iceberg flag reads back as true") + assert(tightBoundsOf(readStats(tampered, metadata, amtProtocol)).contains(false), + "Delta ignores a truncated column's tightness, so the reconstructed flag stays false") + } + + test("reading back to Delta ANDs untruncated columns, so a mix reconstructs tightBounds false") { + // Two untruncated columns. A tight file makes both tight_bounds=true, but if a manifest carries + // a mix (one column not tight), `forRead` ANDs the untruncated columns, so the reconstructed + // Delta tightBounds is false even though the other column is still tight. + val metadata = metadataWithColumnIds( + new StructType().add("id", LongType).add("id2", LongType)) + val persisted = + writeStats(getSampleStatsJson(metadata, tightBounds = true), metadata, amtProtocol) + val tampered = persisted.withColumn("content_stats", + col("content_stats").withField(s"${contentStatsField(persisted, "id2")}.tight_bounds", + lit(false))) + assert(persistedTightBounds(tampered, "id"), "the untouched column stays tight") + assert(!persistedTightBounds(tampered, "id2"), "the tampered column is not tight") + assert(tightBoundsOf(readStats(tampered, metadata, amtProtocol)).contains(false), + "a mix of tight and non-tight untruncated columns reconstructs to false") + } +} diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTFieldSpecSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTFieldSpecSuite.scala new file mode 100644 index 00000000000..f4ba3f5331e --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTFieldSpecSuite.scala @@ -0,0 +1,537 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.amt + +import java.io.File + +import scala.collection.JavaConverters._ + +import org.apache.spark.sql.delta.DeltaColumnMapping +import org.apache.spark.sql.delta.DeltaLog +import org.apache.spark.sql.delta.actions.Metadata +import org.apache.spark.sql.delta.util.JsonUtils +import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.fs.Path +import org.apache.parquet.format.converter.ParquetMetadataConverter +import org.apache.parquet.hadoop.ParquetFileReader +import org.apache.parquet.schema.{GroupType, LogicalTypeAnnotation, MessageType, PrimitiveType, Type} + +import org.apache.spark.SparkRuntimeException +import org.apache.spark.sql.DataFrame +import org.apache.spark.sql.execution.datasources.parquet.ParquetUtils +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.types.{StructField, StructType} + +/** + * Pins the Iceberg V4 field ids an AMT checkpoint puts on disk: + * `AMTSingleAction.persistedSchema` must reproduce the authoritative name-to-id map, and every leaf + * and root footer written by the incremental and full write paths must carry those ids on its + * parquet schema. + */ +class AMTFieldSpecSuite extends AMTCheckpointTestBase { + + private val tsExpr = allTypeColumns("ts").valueExpr + private val tsNtzExpr = allTypeColumns("ts_ntz").valueExpr + + /** + * Every field of a Parquet footer keyed by its dotted path. Intermediate LIST/MAP groups + * (`list`, `key_value`) are omitted from the path so it matches Spark/Iceberg names + * (e.g. `split_offsets.element`). + */ + private def typesByPath(group: GroupType): Map[String, Type] = { + def collect(current: GroupType, prefix: Seq[String]): Map[String, Type] = + current.getFields.asScala.foldLeft(Map.empty[String, Type]) { (acc, field) => + val skipInPath = field.getName == "list" || field.getName == "key_value" + val path = if (skipInPath) prefix else prefix :+ field.getName + val withOwn = if (skipInPath) acc else acc + (path.mkString(".") -> field) + if (field.isPrimitive) withOwn else withOwn ++ collect(field.asGroupType(), path) + } + + collect(group, Seq.empty) + } + + /** The Parquet footer schema of `file`. */ + private def footerSchema(hadoopConf: Configuration, file: File): MessageType = + ParquetFileReader.readFooter( + hadoopConf, new Path(file.getAbsolutePath), ParquetMetadataConverter.NO_FILTER) + .getFileMetaData.getSchema + + /** + * The [[StructType]] of manifest column `column`, read bare off the root manifest (the schema is + * uniform across the tree, so the always-present root suffices). + */ + private def onDiskColumnSchema(provider: AMTCheckpointProvider, column: String): StructType = + allowReadWithinDeltaLog { + spark.read.parquet(provider.topLevelFiles.map(_.getPath.toString): _*) + .schema(column).dataType.asInstanceOf[StructType] + } + + /** + * Reconstructs the live AddFiles through the production read path, but under a checkpoint whose + * metadata is `metadata` (so the read derives its requested, field-id-stamped schema from + * `metadata` rather than the checkpoint's own). Returns the rows with a non-null `add`. + */ + private def reconstructAddFiles( + provider: AMTCheckpointProvider, deltaLog: DeltaLog, metadata: Metadata): DataFrame = + new AMTCheckpointProvider( + manifestCommitVersion = provider.manifestCommitVersion, + checkpointAction = provider.checkpointAction.copy(metaData = metadata), + leaves = provider.leaves, + tableRoot = provider.tableRoot) + .loadActionsForStateReconstruction(spark, deltaLog) + .getOrElse(fail("expected reconstructed actions")) + .where("add is not null") + + private def assertFieldIds( + hadoopConf: Configuration, + file: File, + label: String, + expectedIdByPath: Map[String, Int]): Unit = { + val actualIdByPath = typesByPath(footerSchema(hadoopConf, file)).flatMap { case (path, field) => + Option(field.getId).map(id => path -> id.intValue()) + } + assert( + actualIdByPath.values.toSet.size == actualIdByPath.size, + s"$label contains duplicate field ids: $actualIdByPath") + assert( + actualIdByPath == expectedIdByPath, + s"$label field ids did not match the schema the writer derived\n" + + s" missing=${expectedIdByPath.toSet.diff(actualIdByPath.toSet)}\n" + + s" unexpected=${actualIdByPath.toSet.diff(expectedIdByPath.toSet)}") + } + + /** + * Collects the field-id assignments actually stamped on `AMTSingleAction.persistedSchema` as + * dotted paths, mirroring [[typesByPath]]'s convention: a scalar/struct id comes from the + * field's own `parquet.field.id`; list-element and map key/value ids come from the parent's + * `parquet.field.nested.ids` (keyed `.element` / `values.key` / `values.value`). + */ + private def stampedFieldIdByName(schema: StructType): Map[String, Int] = { + // Read the nested-id sub-metadata (an opaque `Metadata` whose entries are keyed by the + // relative element/key/value path) via its public JSON form, since the key set is not known + // to the reader and `Metadata`'s underlying map is not publicly accessible. + def nestedIds(field: StructField): Map[String, Int] = { + val key = DeltaColumnMapping.PARQUET_FIELD_NESTED_IDS_METADATA_KEY + if (!field.metadata.contains(key)) Map.empty + else { + val json = field.metadata.getMetadata(key).json + JsonUtils.fromJson[Map[String, Long]](json).map { case (k, id) => k -> id.toInt } + } + } + def collect(fields: Seq[StructField], prefix: Seq[String]): Map[String, Int] = + fields.foldLeft(Map.empty[String, Int]) { (acc, field) => + val path = prefix :+ field.name + val ownId = + if (field.metadata.contains(ParquetUtils.FIELD_ID_METADATA_KEY)) { + Map(path.mkString(".") -> + field.metadata.getLong(ParquetUtils.FIELD_ID_METADATA_KEY).toInt) + } else Map.empty[String, Int] + // Nested (list-element / map key-value) ids are stamped as `.` relative to + // the field name, so join them under the current prefix. + val nested = nestedIds(field).map { case (rel, id) => + (prefix :+ rel).mkString(".") -> id + } + val children = field.dataType match { + case s: StructType => collect(s.fields.toSeq, path) + case _ => Map.empty[String, Int] + } + acc ++ ownId ++ nested ++ children + } + collect(schema.fields.toSeq, Seq.empty) + } + + test("persistedSchema stamps an id on every mapped field of AMTSingleAction") { + withAllTypesTable("amt_fieldid_schema") { deltaLog => + val snapshot = deltaLog.update() + val metadata = snapshot.metadata + val protocol = snapshot.protocol + val stamped = stampedFieldIdByName(AMTSingleAction.persistedSchema(metadata, protocol)) + + // Content-stats sub-structs are based at 10_000 + 200 * columnId, with the per-statistic + // offsets from the Iceberg V4 column-stats proposal. The sub-struct carries the base id + // (stats are resolved by id) and is named by the column's logical name with its field id + // attached (see StatsLeaf.fieldName), so the name stays unique. An array is not + // skipping-eligible, so it gets a null count and no bounds; a struct contributes its leaf. + def statsIds(columnId: Int, name: String, withBounds: Boolean): Seq[(String, Int)] = { + val base = 10000 + 200 * columnId + val field = s"${name}_$columnId" // sub-struct field name: logical name + field id + // Iceberg V4 offsets: lower_bound=1, upper_bound=2, tight_bounds=3, null_value_count=5. + Seq( + s"content_stats.$field" -> base, + s"content_stats.$field.null_value_count" -> (base + 5)) ++ + (if (!withBounds) Nil + else Seq( + s"content_stats.$field.lower_bound" -> (base + 1), + s"content_stats.$field.upper_bound" -> (base + 2), + s"content_stats.$field.tight_bounds" -> (base + 3))) + } + // Partition fields take Iceberg ids from 1000 in partition-spec order, whatever their type. + val expected = AMTSingleAction.allFieldIdByName ++ Map("partition" -> 102) ++ + partitionableTestColumns.map(_.structField("p_")).zipWithIndex.map { + case (field, ordinal) => s"partition.${field.name}" -> (1000 + ordinal) + } ++ + // Data columns are one per type in `allTypeColumns` order, so column ids run d_int=1 .. + // d_double=12, d_binary=13, d_arr=14, d_nested=15 (inner=16). + Seq( + (1, "d_int"), (2, "d_long"), (3, "d_short"), (4, "d_byte"), (5, "d_str"), + (6, "d_date"), (7, "d_ts"), (8, "d_ts_ntz"), (9, "d_dec"), + (11, "d_float"), (12, "d_double"), (16, "d_nested_inner")) + .flatMap { case (id, name) => statsIds(id, name, withBounds = true) } ++ + // Boolean, array, and binary are not skipping-eligible (no min/max): null count only. + Seq((10, "d_bool"), (13, "d_binary"), (14, "d_arr")) + .flatMap { case (id, name) => statsIds(id, name, withBounds = false) } + assert( + stamped == expected, + "persistedSchema field ids did not match the expected map\n" + + s" missing=${expected.toSet.diff(stamped.toSet)}\n" + + s" unexpected=${stamped.toSet.diff(expected.toSet)}") + + // Guard against a field added to a nested id-bearing struct (tracking / deletion_vector / + // manifest_info) without a corresponding id: every scalar of those structs must be stamped. + val schema = AMTSingleAction.persistedSchema(metadata, protocol) + Seq("tracking", "deletion_vector", "manifest_info").foreach { parent => + val struct = schema(parent).dataType.asInstanceOf[StructType] + struct.fields.foreach { f => + assert(f.metadata.contains(ParquetUtils.FIELD_ID_METADATA_KEY), + s"nested field '$parent.${f.name}' has no stamped Iceberg field id.") + } + } + } + } + + testAcrossAMTCheckpointScenarios( + "writes stamp field ids on every leaf and root field", + "amt_fieldid_write", + sqlConfs = leafPackingConfs)( + setup = name => appendRowsAsSeparateFiles(name, numFiles = leafPackedFiles - 1), + inlineCheckpointTriggerActionsOrSQL = Some(name => Right( + s"INSERT INTO $name VALUES (${leafPackedFiles - 1})"))) { context => + val hadoopConf = context.postCheckpointSnapshot.deltaLog.newDeltaHadoopConf() + // `content_stats` is shaped per table, so the footer's ids can't match a static map. Compare + // against the schema the writer actually derived from the metadata/protocol the checkpoint + // recorded -- which is exactly what a reader projects with. + val checkpointAction = context.provider.checkpointAction + val expected = stampedFieldIdByName(AMTSingleAction.persistedSchema( + checkpointAction.metaData, checkpointAction.protocol)) + // `leaf.location` and `contentRoot.path` are stored table-root-relative, so go through the + // provider, which resolves them against the table root. + val leaves = context.provider.liveLeafManifestAbsolutePaths.map(path => new File(path.toUri)) + val root = new File(context.provider.topLevelFiles.head.getPath.toUri) + assert(leaves.nonEmpty, "Expected at least one leaf manifest.") + leaves.foreach(f => assertFieldIds(hadoopConf, f, s"leaf ${f.getName}", expected)) + assertFieldIds(hadoopConf, root, s"root ${root.getName}", expected) + } + + test("a full checkpoint rewrite refreshes content_stats names after a column rename") { + withTable("amt_rename_rewrite") { + // A huge checkpoint interval keeps the rename commit from auto-checkpointing, so the only + // rewrites are the two explicit ones below. + createAMTTable( + "amt_rename_rewrite", tableSchema = "c LONG", checkpointInterval = Int.MaxValue) + appendRowsAsSeparateFiles( + "amt_rename_rewrite", numFiles = 2, columnExprs = Seq("CAST(id AS LONG)")) + val deltaLog = deltaLogForName("amt_rename_rewrite") + + // The logical name of the single `content_stats` sub-struct on the latest checkpoint's + // manifests. Sub-structs are named `_` (the id keeps the name unique; + // see StatsLeaf.fieldName), so strip the id suffix to compare the logical part that a rename + // refreshes. + def contentStatsNames(): Seq[String] = { + val snapshot = deltaLog.update() + val provider = amtProvider(snapshot).getOrElse(fail("expected AMTCheckpointProvider")) + onDiskColumnSchema(provider, "content_stats").fieldNames.toSeq + .map(_.replaceAll("_\\d+$", "")) + } + + // The pre-rename checkpoint names the sub-struct by the original logical name. + commitCheckpoint(deltaLog, incremental = false) + assert(contentStatsNames() == Seq("c")) + + // A metadata-only rename does not rewrite the existing manifests, so the checkpoint on disk + // still names the sub-struct by the old logical name until it is rewritten. + sql("ALTER TABLE amt_rename_rewrite RENAME COLUMN c TO c_renamed") + assert(contentStatsNames() == Seq("c")) + + // Forcing a full rewrite re-derives content_stats from the current metadata, so the + // sub-struct picks up the new logical name (same field id). + commitCheckpoint(deltaLog, incremental = false) + assert(contentStatsNames() == Seq("c_renamed")) + } + } + + testAcrossAMTCheckpointScenarios( + "content stats survive a column rename (field-id-based read)", + "amt_rename", + tableSchema = "col_to_be_renamed STRING, col_no_rename STRING")( + setup = name => appendRowsAsSeparateFiles( + name, + numFiles = 2, + columnExprs = Seq( + "CONCAT('renamedVal', CAST(id AS STRING))", + "CONCAT('keptVal', CAST(id AS STRING))")), + inlineCheckpointTriggerActionsOrSQL = Some(name => Right( + s"""INSERT INTO $name SELECT CONCAT('renamedVal', CAST(id AS STRING)), + |CONCAT('keptVal', CAST(id AS STRING)) FROM range(2, 3)""".stripMargin))) { context => + val deltaLog = context.postCheckpointSnapshot.deltaLog + val provider = context.provider + + // Reconstruct the live AddFiles under a checkpoint whose metadata is `meta`, projecting the + // stats each file carries. + def reconstructStats(meta: Metadata): Seq[String] = + reconstructAddFiles(provider, deltaLog, meta) + .select("add.stats").collect() + .map(r => if (r.isNullAt(0)) "null" else r.getString(0)).toSeq + + // The sub-structs on disk are named `_` (a bare read + // reflects the physical layout verbatim; content_stats keys each sub-struct by column id). + val preRenameMeta = deltaLog.update().metadata + val onDisk = onDiskColumnSchema(provider, "content_stats") + val expectedSubStructs = preRenameMeta.dataSchema + .map(f => s"${f.name}_${DeltaColumnMapping.getColumnId(f)}").toSet + assert(onDisk.fieldNames.toSet == expectedSubStructs, + s"on-disk sub-structs ${onDisk.fieldNames.toSet} != expected $expectedSubStructs") + + // Before the rename, reconstruction carries every file's stats for both columns. + val statsBefore = reconstructStats(deltaLog.update().metadata) + assert(statsBefore.nonEmpty && statsBefore.forall(_.contains("minValues"))) + assert( + statsBefore.exists(_.contains("renamedVal")) && statsBefore.exists(_.contains("keptVal")), + s"expected both columns' stats before rename: $statsBefore") + + // After the rename (same id and physical name, new logical name), the field-id read remaps + // the on-disk `col_to_be_renamed` sub-struct onto the current `col_renamed`, so it still + // succeeds -- identical for both columns (the stats JSON is keyed by physical name, + // unchanged). + sql(s"ALTER TABLE ${context.tableName} RENAME COLUMN col_to_be_renamed TO col_renamed") + val statsAfter = reconstructStats(deltaLog.update().metadata) + assert(statsAfter.sorted == statsBefore.sorted, + s"stats changed across rename: $statsBefore -> $statsAfter") + + // Field-id read is what makes this work. With it off, the read falls back to + // name-matching: the on-disk `content_stats` struct still matches by name, but its renamed + // `col_to_be_renamed` sub-struct no longer matches the new `col_renamed`, so only that + // column's stats are lost -- `col_no_rename` still resolves by name. + val degraded = withSQLConf(SQLConf.PARQUET_FIELD_ID_READ_ENABLED.key -> "false") { + reconstructStats(deltaLog.update().metadata) + } + assert(degraded.exists(_.contains("keptVal")), + s"the un-renamed `col_no_rename` column's stats should survive name-matching: $degraded") + assert(degraded.forall(!_.contains("renamedVal")), + s"the renamed column's stats should be lost without field-id read: $degraded") + } + + testAcrossAMTCheckpointScenarios( + "partition values survive a column rename (field-id-based read)", + "amt_part_rename", + tableSchema = "id LONG, p STRING, keep_p STRING", + partitionColumns = Seq("p", "keep_p"))( + setup = name => appendRowsAsSeparateFiles( + name, + numFiles = 2, + columnExprs = Seq( + "CAST(id AS LONG)", + "CONCAT('pRenamedVal', CAST(id AS STRING))", + "CONCAT('pKeptVal', CAST(id AS STRING))")), + inlineCheckpointTriggerActionsOrSQL = Some(name => Right( + s"""INSERT INTO $name SELECT CAST(id AS LONG), + |CONCAT('pRenamedVal', CAST(id AS STRING)), + |CONCAT('pKeptVal', CAST(id AS STRING)) FROM range(2, 3)""".stripMargin))) { context => + val deltaLog = context.postCheckpointSnapshot.deltaLog + val provider = context.provider + + // Reconstruct the live AddFiles under a checkpoint whose metadata is `meta`, projecting the + // partition values each file carries. + def reconstructPartitions(meta: Metadata): Seq[Map[String, String]] = + reconstructAddFiles(provider, deltaLog, meta) + .select("add.partitionValues").collect().toSeq + .map(r => + if (r.isNullAt(0)) Map.empty[String, String] + else r.getMap[String, String](0).toMap) + + // The on-disk partition fields are still named by the pre-rename logical column names. + val onDisk = onDiskColumnSchema(provider, "partition") + assert(onDisk.fieldNames.toSet == Set("p", "keep_p")) + + // Before the rename, reconstruction carries every file's partition values for both columns. + val before = reconstructPartitions(deltaLog.update().metadata) + assert(before.nonEmpty && before.forall(m => m.size == 2 && m.values.forall(_ != null))) + val beforeValues = before.flatMap(_.values).toSet + assert(beforeValues.exists(_.startsWith("pRenamedVal")) && + beforeValues.exists(_.startsWith("pKeptVal")), + s"expected both partition columns' values before rename: $before") + + // After the rename (same id and physical name, new logical name), the field-id read remaps + // the on-disk `p` field onto the current `p_renamed`, so reconstruction still succeeds -- + // identical for both columns (the value map is keyed by physical name, unchanged). + sql(s"ALTER TABLE ${context.tableName} RENAME COLUMN p TO p_renamed") + val after = reconstructPartitions(deltaLog.update().metadata) + assert( + after.sortBy(_.toString) == before.sortBy(_.toString), + s"partition values changed across rename: $before -> $after") + + // Field-id read is what makes this work (see the content-stats test): with it off, the + // read falls back to name-matching, so only the renamed `p` field is lost -- it reads null + // under the new `p_renamed` name -- while `keep_p` still resolves by name. + val degraded = withSQLConf(SQLConf.PARQUET_FIELD_ID_READ_ENABLED.key -> "false") { + reconstructPartitions(deltaLog.update().metadata) + } + val degradedNonNull = degraded.flatMap(_.values).filter(_ != null).toSet + assert(degradedNonNull.exists(_.startsWith("pKeptVal")), + s"the un-renamed `keep_p` value should survive name-matching: $degraded") + assert(!degradedNonNull.exists(_.startsWith("pRenamedVal")), + s"the renamed partition value should be lost without field-id read: $degraded") + assert(degraded.exists(_.values.exists(_ == null)), + s"the renamed partition value should read null without field-id read: $degraded") + } + + testAcrossAMTCheckpointScenarios( + "manifest writes timestamps as int64 TIMESTAMP_MICROS, not INT96", + "amt_ts", + tableSchema = + "d_ts TIMESTAMP, d_ts_ntz TIMESTAMP_NTZ, p_ts TIMESTAMP, p_ts_ntz TIMESTAMP_NTZ", + partitionColumns = Seq("p_ts", "p_ts_ntz"))( + setup = name => appendRowsAsSeparateFiles( + name, + numFiles = 2, + columnExprs = Seq(tsExpr, tsNtzExpr, tsExpr, tsNtzExpr)), + inlineCheckpointTriggerActionsOrSQL = Some(name => Right( + s"""INSERT INTO $name SELECT $tsExpr, $tsNtzExpr, $tsExpr, $tsNtzExpr + |FROM range(2, 3)""".stripMargin))) { + context => + val deltaLog = context.postCheckpointSnapshot.deltaLog + val provider = context.provider + val hadoopConf = deltaLog.newDeltaHadoopConf() + // This small table is not leaf-packed, so it is a single root-resident manifest; reading the + // root covers every persisted timestamp value. + val files = provider.topLevelFiles.map(f => new File(f.getPath.toUri)) + assert(files.nonEmpty, "Expected at least one manifest file.") + + // content_stats names each sub-struct `_`; derive that suffix here. + val tsMeta = deltaLog.update().metadata + def cs(column: String): String = + s"content_stats.${column}_${DeltaColumnMapping.getColumnId(tsMeta.schema(column))}" + + files.foreach { file => + val primitives = typesByPath(footerSchema(hadoopConf, file)).collect { + case (path, field) if field.isPrimitive => path -> field.asPrimitiveType() + } + def assertTimestampMicros(name: String, adjustToUtc: Boolean): Unit = { + val prim = + primitives.getOrElse(name, fail(s"no primitive at '$name' in ${file.getName}")) + assert( + prim.getPrimitiveTypeName == PrimitiveType.PrimitiveTypeName.INT64, + s"'$name' should be int64 micros, got ${prim.getPrimitiveTypeName} in ${file.getName}") + prim.getLogicalTypeAnnotation match { + case ts: LogicalTypeAnnotation.TimestampLogicalTypeAnnotation => + assert( + ts.getUnit == LogicalTypeAnnotation.TimeUnit.MICROS, + s"'$name' should be MICROS, got ${ts.getUnit} in ${file.getName}") + assert( + ts.isAdjustedToUTC == adjustToUtc, + s"'$name' adjustToUtc should be $adjustToUtc in ${file.getName}") + case other => + fail(s"'$name' should carry a timestamp annotation, got $other in ${file.getName}") + } + } + // TimestampType -> adjust-to-utc; TimestampNTZType -> not. content_stats keys each + // sub-struct by `_`; partition keys by logical name. + assertTimestampMicros(s"${cs("d_ts")}.lower_bound", adjustToUtc = true) + assertTimestampMicros(s"${cs("d_ts")}.upper_bound", adjustToUtc = true) + assertTimestampMicros(s"${cs("d_ts_ntz")}.lower_bound", adjustToUtc = false) + assertTimestampMicros("partition.p_ts", adjustToUtc = true) + assertTimestampMicros("partition.p_ts_ntz", adjustToUtc = false) + } + } + + test("field-id read recovers a renamed static manifest_info column on the root") { + withTable("amt_static_rename") { + withSQLConf(leafPackingConfs: _*) { + createAMTTable("amt_static_rename", checkpointInterval = Int.MaxValue) + appendRowsAsSeparateFiles("amt_static_rename", numFiles = leafPackedFiles) + val deltaLog = deltaLogForName("amt_static_rename") + commitCheckpoint(deltaLog, incremental = false) + val provider = amtProvider(deltaLog.update()) + .getOrElse(fail("expected AMTCheckpointProvider")) + assert(provider.leaves.nonEmpty, "Expected leaf pointers in the root.") + val checkpoint = provider.checkpointAction + val persistedSchema = + AMTSingleAction.persistedSchema(checkpoint.metaData, checkpoint.protocol) + + // Read the root's physical columns verbatim (the persisted schema carries the field ids), + // then rewrite the root with its static `manifest_info` struct renamed to + // `manifest_info_new`, keeping the same Iceberg field id. A reader that matched columns by + // name would now miss it entirely. + val rootPath = provider.topLevelFiles.head.getPath.toString + val rootRows = allowReadWithinDeltaLog { + spark.read.schema(persistedSchema).parquet(rootPath).collect().toSeq + } + val renamedSchema = StructType(persistedSchema.map { field => + if (field.name == "manifest_info") field.copy(name = "manifest_info_new") else field + }) + withTempDir { dir => + val rewrittenDir = new File(dir, "root-1-dash") + withSQLConf(SQLConf.PARQUET_FIELD_ID_WRITE_ENABLED.key -> "true") { + spark.createDataFrame(rootRows.asJava, renamedSchema) + .coalesce(1).write.parquet(rewrittenDir.getAbsolutePath) + } + val rewrittenFile = Option(rewrittenDir.listFiles()).toSeq.flatten + .find(_.getName.endsWith(".parquet")) + .getOrElse(fail("no rewritten root parquet produced")) + + // The field really is renamed on disk (so a name-based read could not recover it). + val rewrittenSchema = footerSchema(deltaLog.newDeltaHadoopConf(), rewrittenFile) + assert( + rewrittenSchema.containsField("manifest_info_new") && + !rewrittenSchema.containsField("manifest_info"), + s"rewritten root should rename manifest_info: $rewrittenSchema") + + // Reading the rewritten root through the production path recovers every leaf pointer + // unchanged: `manifest_info` is resolved by its Iceberg field id, not its name. (If it + // were not, the DATA_MANIFEST rows would unwrap with a missing `manifest_info`.) + val rewrittenCheckpoint = checkpoint.copy( + contentRoot = checkpoint.contentRoot.copy( + path = rewrittenFile.getAbsolutePath, + sizeInBytes = rewrittenFile.length())) + val rewrittenProvider = AMTCheckpointProvider.fromCheckpoint( + deltaLog, rewrittenCheckpoint, provider.manifestCommitVersion) + assertLeavesEqual(rewrittenProvider.leaves, provider.leaves) + + // Field-id read is what makes this work: with it off, the read falls back to + // name-matching, so the requested `manifest_info` no longer matches the on-disk + // `manifest_info_new` and reads null. Decoding the manifest pointer then fails when + // `AMTSingleAction.validate` rejects a DATA_MANIFEST entry with no `manifest_info`, and + // the row decode surfaces it as EXPRESSION_DECODING_FAILED wrapping that requirement. + withSQLConf(SQLConf.PARQUET_FIELD_ID_READ_ENABLED.key -> "false") { + val error = intercept[SparkRuntimeException] { + AMTCheckpointProvider.fromCheckpoint( + deltaLog, rewrittenCheckpoint, provider.manifestCommitVersion) + } + assert(error.getCondition == "EXPRESSION_DECODING_FAILED", + s"Expected an EXPRESSION_DECODING_FAILED error, got ${error.getCondition}.") + assert(error.getCause.isInstanceOf[IllegalArgumentException], + s"Expected the decode to fail on the validate requirement, got ${error.getCause}.") + assert( + error.getCause.getMessage.contains( + "manifest_info must be set iff content_type is a manifest pointer"), + s"Expected a missing manifest_info requirement failure, " + + s"got ${error.getCause.getMessage}.") + } + } + } + } + } +} diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTIncrementalWriteCoreSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTIncrementalWriteCoreSuite.scala new file mode 100644 index 00000000000..dc964af6890 --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTIncrementalWriteCoreSuite.scala @@ -0,0 +1,567 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.amt + +// scalastyle:off import.ordering.noEmptyLine +import org.apache.spark.sql.delta.DeltaOperations +import org.apache.spark.sql.delta.sources.DeltaSQLConf + +/** + * The incremental-write engine: leafless/promoted-root basics, intermediate/window-commit + * assembly and replay correctness, and version bookkeeping. + */ +class AMTIncrementalWriteCoreSuite extends AMTIncrementalWriteTestBase { + + test("snapshot-backed AMT base bootstraps an incremental write from an empty snapshot") { + withTable("amt_snapshot_base_empty") { + val tableName = "amt_snapshot_base_empty" + createAMTTable(tableName, checkpointInterval = Int.MaxValue) + val deltaLog = deltaLogForName(tableName) + val snapshot = deltaLog.update() + assert(snapshot.numOfFiles === 0L) + assert(!snapshot.checkpointProvider.isInstanceOf[AMTCheckpointProvider]) + + val baseProvider = new BaseSnapshotActionsProvider(snapshot) + val base = baseProvider.load() + assert(base.metadata === snapshot.metadata) + assert(base.protocol === snapshot.protocol) + assert(base.setTransactions === snapshot.setTransactions) + assert(base.domainMetadatas === snapshot.domainMetadata) + assert(base.version === snapshot.version) + assert(base.fileActionsFromRoot.isEmpty) + assert(base.allLeafs.isEmpty) + assert(base.lastManifestCommitWithFullRewrite.isEmpty) + + val add = fakeAdd(1) + val attemptVersion = snapshot.version + 1 + val result = new IncrementalAMTWriter(spark, deltaLog).writeIncremental( + oldAMTActionsProvider = baseProvider, + intermediateLogCommits = Seq.empty, + attemptVersion = attemptVersion, + actionsToCommit = Seq(add), + trigger = "snapshot-bootstrap-test") + val metric = result.amtWriteMetrics + + assert(result.contentRootVersion === attemptVersion) + assert(result.checkpoint.version === attemptVersion) + assert(result.checkpoint.metaData === snapshot.metadata) + assert(result.checkpoint.protocol === snapshot.protocol) + assert(result.checkpoint.contentRoot.isIncremental.contains(true)) + assert(result.checkpoint.contentRoot.lastManifestCommitWithFullRewrite.contains( + attemptVersion)) + assert(result.includeActionsInCommitJson) + assert(result.leaves.isEmpty) + assert(metric.incremental === "true") + val incrementalMetrics = metric.incrementalWriteMetrics.getOrElse( + fail("The snapshot bootstrap must report incremental-write metrics.")) + assert(incrementalMetrics.numIntermediateCommits === 0) + + val rootPath = result.checkpoint.contentRoot + .getAbsolutePath(deltaLog.dataPath).toString + assert(trackingStatusToLocationsMap(rootPath) === + Map(Tracking.Status.Added -> Set(add.path))) + } + } + + test("snapshot-backed AMT base rejects a nonempty snapshot") { + withTable("amt_snapshot_base_nonempty") { + val tableName = "amt_snapshot_base_nonempty" + createAMTTable(tableName, checkpointInterval = Int.MaxValue) + val deltaLog = deltaLogForName(tableName) + deltaLog.startTransaction().commit(Seq(fakeAdd(1)), writeOperation) + val snapshot = deltaLog.update() + assert(!snapshot.checkpointProvider.isInstanceOf[AMTCheckpointProvider]) + assert(snapshot.numOfFiles === 1L) + + val error = intercept[IllegalArgumentException] { + new BaseSnapshotActionsProvider(snapshot) + } + assert(error.getMessage.contains("currently requires an empty snapshot")) + } + } + + test("snapshot-backed AMT base rejects a snapshot with an AMT checkpoint") { + withTable("amt_snapshot_base_existing_amt") { + val tableName = "amt_snapshot_base_existing_amt" + createAMTTable(tableName, checkpointInterval = Int.MaxValue) + val deltaLog = deltaLogForName(tableName) + commitCheckpoint(deltaLog, incremental = false) + val snapshot = deltaLog.update() + assert(snapshot.numOfFiles === 0L) + assert(snapshot.checkpointProvider.isInstanceOf[AMTCheckpointProvider]) + + val error = intercept[IllegalArgumentException] { + new BaseSnapshotActionsProvider(snapshot) + } + assert(error.getMessage.contains("must not already have an AMT checkpoint")) + } + } + + test("deleting a file from a promoted root drops it via replay, no leaf, no tombstone") { + withSQLConf(DeltaSQLConf.AMT_ENTRIES_PER_LEAF.key -> "8") { + withTables() { (baselineDeltaLog, amtDeltaLog) => + fullCheckpointPromotedToRoot(baselineDeltaLog, amtDeltaLog) + // Add two more files; all three live in the promoted root (no leaves, under the cap). + commitBoth(baselineDeltaLog, amtDeltaLog, Seq(fakeAdd(2), fakeAdd(3))) + commitCheckpoint(amtDeltaLog, incremental = true) + assert(leafPointers(amtDeltaLog.update()).isEmpty, + "The tree must stay leafless after a small append to a promoted root.") + // Delete one root-resident file. It has no back reference, so replay drops it (as on + // the deferred path): no leaf is touched and, deferred, no root tombstone is written. + commitBoth(baselineDeltaLog, amtDeltaLog, Seq(removeOf(amtDeltaLog, 2))) + createIncrementalAMTAndValidate( + baselineDeltaLog, + amtDeltaLog, + createIncrementalAMTWriteMetrics( + numRootEntriesExistingStatus = 2)) + val (rootLiveAdds, tombstones) = liveAddsAndTombstonesCountInRoot(amtDeltaLog) + assert(rootLiveAdds == 2, s"Two files must remain live in the root; got $rootLiveAdds.") + assert(tombstones == 0, + s"A deferred remove of a root-resident file writes no tombstone; got $tombstones.") + } + } + } + + test("deleting every file from a promoted root yields an empty, still-readable tree") { + withSQLConf(DeltaSQLConf.AMT_ENTRIES_PER_LEAF.key -> "8") { + withTables() { (baselineDeltaLog, amtDeltaLog) => + fullCheckpointPromotedToRoot(baselineDeltaLog, amtDeltaLog) + commitBoth(baselineDeltaLog, amtDeltaLog, Seq(fakeAdd(2), fakeAdd(3))) + commitCheckpoint(amtDeltaLog, incremental = true) + // Remove all three root-resident files across two intermediate commits. Every remove is a + // no-backref replay drop, so the new root ends with no live adds and the tree, having never + // had a leaf, reconstructs an empty live set. + commitBoth( + baselineDeltaLog, amtDeltaLog, Seq(removeOf(amtDeltaLog, 1), removeOf(amtDeltaLog, 2))) + commitBoth(baselineDeltaLog, amtDeltaLog, Seq(removeOf(amtDeltaLog, 3))) + createIncrementalAMTAndValidate( + baselineDeltaLog, + amtDeltaLog, + createIncrementalAMTWriteMetrics()) + assert(leafPointers(amtDeltaLog.update()).isEmpty, + "The tree must remain leafless.") + assert(livePathsInLatestAMTCheckpoint(amtDeltaLog).isEmpty, + "The AMT tree must reconstruct an empty live set.") + } + } + } + + test("deleting an old root-resident file drops it via replay, no MDV") { + withSQLConf(leafPackingConfs: _*) { + withTables() { (baselineDeltaLog, amtDeltaLog) => + commitBoth(baselineDeltaLog, amtDeltaLog, (1 to leafPackedFiles).map(fakeAdd)) + commitCheckpoint(amtDeltaLog, incremental = false) + val numLeafCountBefore = + amtProvider(amtDeltaLog.update()).getOrElse(fail("expected AMT")).leaves.size + assert(numLeafCountBefore >= 2, + s"Need a tree-shaped bootstrap; got $numLeafCountBefore leaves.") + // Append id=31 -> becomes root-resident (no leaf, no backref). + commitBoth(baselineDeltaLog, amtDeltaLog, Seq(fakeAdd(leafPackedFiles + 1))) + createIncrementalAMTAndValidate( + baselineDeltaLog, + amtDeltaLog, + createIncrementalAMTWriteMetrics( + numOldLeavesUntouched = numLeafCountBefore, + numRootEntriesExistingStatus = 1, + numLeavesExistingStatus = numLeafCountBefore)) + // Delete id=31 (root-resident): remove has NO backref -> dropped by replay, no MDV bit. + commitBoth(baselineDeltaLog, amtDeltaLog, Seq(removeOf(amtDeltaLog, leafPackedFiles + 1))) + createIncrementalAMTAndValidate( + baselineDeltaLog, + amtDeltaLog, + createIncrementalAMTWriteMetrics( + numOldLeavesUntouched = numLeafCountBefore, + numLeafMdvBitsAdded = 0, + numLeavesExistingStatus = numLeafCountBefore)) + } + } + } + + test("an add then delete of the same file within the intermediate commits is net-zero") { + withSQLConf(leafPackingConfs: _*) { + withTables() { (baselineDeltaLog, amtDeltaLog) => + commitBoth(baselineDeltaLog, amtDeltaLog, (1 to leafPackedFiles).map(fakeAdd)) + commitCheckpoint(amtDeltaLog, incremental = false) + val numLeafCountBefore = + amtProvider(amtDeltaLog.update()).getOrElse(fail("expected AMT")).leaves.size + assert(numLeafCountBefore >= 2, + s"Need a tree-shaped bootstrap; got $numLeafCountBefore leaves.") + // The intermediate commits add a file and remove it again: net-zero, so nothing reaches + // the new tree. + commitBoth(baselineDeltaLog, amtDeltaLog, Seq(fakeAdd(leafPackedFiles + 1))) + commitBoth(baselineDeltaLog, amtDeltaLog, Seq(removeOf(amtDeltaLog, leafPackedFiles + 1))) + createIncrementalAMTAndValidate( + baselineDeltaLog, + amtDeltaLog, + createIncrementalAMTWriteMetrics( + numOldLeavesUntouched = numLeafCountBefore, + numLeafMdvBitsAdded = 0, + numLeavesExistingStatus = numLeafCountBefore)) + } + } + } + + test("a leaf file removed then re-added at the same path is masked once") { + withSQLConf(leafPackingConfs: _*) { + withTables() { (baselineDeltaLog, amtDeltaLog) => + commitBoth(baselineDeltaLog, amtDeltaLog, (1 to leafPackedFiles).map(fakeAdd)) + commitCheckpoint(amtDeltaLog, incremental = false) + val leafToAddFileMapping = leafToAddFileMap(amtDeltaLog) + assert(leafToAddFileMapping.size >= 2, + s"Need a tree-shaped bootstrap; got ${leafToAddFileMapping.size} leaves.") + // Window: remove a leaf-resident file (it carries a backref) then re-add the SAME path. The + // old leaf entry is MDV-masked and the re-added copy becomes a root-resident EXISTING entry + // (a re-commit of an already-live key, not a new add); reconstructed once. The re-add keeps + // the back reference the file was stamped with: its path is still the one the leaf holds, + // and a commit reusing a leaf-resident path must carry that leaf's reference. + val victim = leafToAddFileMapping.toSeq.sortBy(_._1).head._2.head + commitBoth(baselineDeltaLog, amtDeltaLog, Seq(victim.remove)) + commitBoth(baselineDeltaLog, amtDeltaLog, Seq(victim)) + createIncrementalAMTAndValidate( + baselineDeltaLog, + amtDeltaLog, + createIncrementalAMTWriteMetrics( + numOldLeavesUpdated = 1, + numOldLeavesUntouched = leafToAddFileMapping.size - 1, + numRootEntriesExistingStatus = 1, + numLeafMdvBitsAdded = 1, + numLeavesModifiedStatus = 1, + numLeavesExistingStatus = leafToAddFileMapping.size - 1)) + } + } + } + + test("a leaf file re-added and removed again is masked once, not double-counted") { + withSQLConf(leafPackingConfs: _*) { + withTables() { (baselineDeltaLog, amtDeltaLog) => + commitBoth(baselineDeltaLog, amtDeltaLog, (1 to leafPackedFiles).map(fakeAdd)) + commitCheckpoint(amtDeltaLog, incremental = false) + val leafToAddFileMapping = leafToAddFileMap(amtDeltaLog) + assert(leafToAddFileMapping.size >= 2, + s"Need a tree-shaped bootstrap; got ${leafToAddFileMapping.size} leaves.") + // Remove a leaf-resident file, re-add the same path, then remove it again. Both removes + // carry the same back reference, so both target the same (leaf, position). Note the re-add + // can only follow a remove: re-adding a path that is still live would be an in-place file + // metadata update, which a WRITE is not allowed to perform. + val victim = leafToAddFileMapping.toSeq.sortBy(_._1).head._2.head + val victimLeaf = leafToAddFileMapping.toSeq.sortBy(_._1).head._1 + commitBoth(baselineDeltaLog, amtDeltaLog, Seq(victim.remove)) + commitBoth(baselineDeltaLog, amtDeltaLog, Seq(victim)) + commitBoth(baselineDeltaLog, amtDeltaLog, Seq(victim.remove)) + // The two removes share one (leaf, position), which the writer holds as a set, so the write + // reports a single MDV bit, matching what the leaf's bitmap actually gains. That + // agreement is what lets this go through the shared validator, whose second check + // derives the bits from the on-disk dv_cardinality delta. + createIncrementalAMTAndValidate( + baselineDeltaLog, + amtDeltaLog, + createIncrementalAMTWriteMetrics( + numOldLeavesUpdated = 1, + numOldLeavesUntouched = leafToAddFileMapping.size - 1, + numLeafMdvBitsAdded = 1, + numLeavesModifiedStatus = 1, + numLeavesExistingStatus = leafToAddFileMapping.size - 1)) + val maskedLeaf = leafPointers(amtDeltaLog.update()).getOrElse(victimLeaf, + fail(s"Leaf $victimLeaf must still be carried forward.")) + assert(mdvCardinality(maskedLeaf) == 1L, + s"The twice-removed position must be masked once; " + + s"got ${mdvCardinality(maskedLeaf)} bits.") + } + } + } + + test("a re-add at a different path leaves both files live, no double-count") { + withSQLConf(leafPackingConfs: _*) { + withTables() { (baselineDeltaLog, amtDeltaLog) => + commitBoth(baselineDeltaLog, amtDeltaLog, (1 to leafPackedFiles).map(fakeAdd)) + commitCheckpoint(amtDeltaLog, incremental = false) + val leafToAddFileMapping = leafToAddFileMap(amtDeltaLog) + assert(leafToAddFileMapping.size >= 2, + s"Need a tree-shaped bootstrap; got ${leafToAddFileMapping.size} leaves.") + // Delete a leaf-resident file, then add a NEW file at a different path. The leaf gets one + // MDV bit, the new file is a root-resident live add; both reconstruct exactly once. + val victim = leafToAddFileMapping.toSeq.sortBy(_._1).head._2.head + commitBoth(baselineDeltaLog, amtDeltaLog, Seq(victim.remove)) + commitBoth(baselineDeltaLog, amtDeltaLog, Seq(fakeAdd(leafPackedFiles + 1))) + createIncrementalAMTAndValidate( + baselineDeltaLog, + amtDeltaLog, + createIncrementalAMTWriteMetrics( + numOldLeavesUpdated = 1, + numOldLeavesUntouched = leafToAddFileMapping.size - 1, + numRootEntriesExistingStatus = 1, + numLeafMdvBitsAdded = 1, + numLeavesModifiedStatus = 1, + numLeavesExistingStatus = leafToAddFileMapping.size - 1)) + } + } + } + + test("an incremental AMT with no intermediate commits reconstructs unchanged") { + withSQLConf(leafPackingConfs: _*) { + withTables() { (baselineDeltaLog, amtDeltaLog) => + commitBoth(baselineDeltaLog, amtDeltaLog, (1 to leafPackedFiles).map(fakeAdd)) + commitCheckpoint(amtDeltaLog, incremental = false) + val numLeafCountBefore = + amtProvider(amtDeltaLog.update()).getOrElse(fail("expected AMT")).leaves.size + assert(numLeafCountBefore >= 2, + s"Need a tree-shaped bootstrap; got $numLeafCountBefore leaves.") + // Immediately checkpoint again with no intervening write: no shape change. + createIncrementalAMTAndValidate( + baselineDeltaLog, + amtDeltaLog, + createIncrementalAMTWriteMetrics( + numOldLeavesUntouched = numLeafCountBefore, + numLeavesExistingStatus = numLeafCountBefore)) + } + } + } + + test("a deferred incremental AMT folds in every intermediate commit and counts them") { + withSQLConf(leafPackingConfs: _*) { + withTables() { (baselineDeltaLog, amtDeltaLog) => + commitBoth(baselineDeltaLog, amtDeltaLog, (1 to leafPackedFiles).map(fakeAdd)) + commitCheckpoint(amtDeltaLog, incremental = false) + val numLeafCountBefore = + amtProvider(amtDeltaLog.update()).getOrElse(fail("expected AMT")).leaves.size + assert(numLeafCountBefore >= 2, + s"Need a tree-shaped bootstrap; got $numLeafCountBefore leaves.") + // Three separate writes between the full AMT and the deferred incremental. Each + // adds a distinct root-resident file; the deferred incremental must fold all three in. + val businessCommits = 3 + (leafPackedFiles + 1 to leafPackedFiles + businessCommits) + .foreach(fileID => commitBoth(baselineDeltaLog, amtDeltaLog, Seq(fakeAdd(fileID)))) + // The intermediate commits span [oldAMTVersion+1, attemptVersion): the three writes plus + // the OPTIMIZE CHECKPOINT commit that landed the bootstrap tree. Every appended file stays + // root-resident. + createIncrementalAMTAndValidate( + baselineDeltaLog, + amtDeltaLog, + createIncrementalAMTWriteMetrics( + numOldLeavesUntouched = numLeafCountBefore, + numRootEntriesExistingStatus = businessCommits, + numLeavesExistingStatus = numLeafCountBefore), + expectedNumIntermediateCommits = Some(businessCommits + 1)) + } + } + } + + test("replay re-derives the root across a chain of incremental AMTs") { + withSQLConf(leafPackingConfs: _*) { + withTables() { (baselineDeltaLog, amtDeltaLog) => + commitBoth(baselineDeltaLog, amtDeltaLog, (1 to leafPackedFiles).map(fakeAdd)) + commitCheckpoint(amtDeltaLog, incremental = false) + val leafToAddFileMapping = leafToAddFileMap(amtDeltaLog) + assert(leafToAddFileMapping.size >= 2, + s"Need a tree-shaped bootstrap; got ${leafToAddFileMapping.size} leaves.") + val numLeafCountBefore = leafToAddFileMapping.size + + // incr 1: append one file, which stays root-resident (carried pointers + 1 add <= cap). + commitBoth(baselineDeltaLog, amtDeltaLog, Seq(fakeAdd(leafPackedFiles + 1))) + createIncrementalAMTAndValidate( + baselineDeltaLog, + amtDeltaLog, + createIncrementalAMTWriteMetrics( + numOldLeavesUntouched = numLeafCountBefore, + numRootEntriesExistingStatus = 1, + numLeavesExistingStatus = numLeafCountBefore)) + + // incr 2: delete a leaf-resident file -> its leaf gets one MDV bit. The file appended by + // incr 1 must survive as a root-resident EXISTING entry, which is replay re-deriving the + // root's live set from the PREVIOUS incremental's root (part 1a). + val victim = leafToAddFileMapping.toSeq.sortBy(_._1).head._2.head + commitBoth(baselineDeltaLog, amtDeltaLog, Seq(victim.remove)) + createIncrementalAMTAndValidate( + baselineDeltaLog, + amtDeltaLog, + createIncrementalAMTWriteMetrics( + numOldLeavesUpdated = 1, + numOldLeavesUntouched = numLeafCountBefore - 1, + numRootEntriesExistingStatus = 1, + numLeafMdvBitsAdded = 1, + numLeavesModifiedStatus = 1, + numLeavesExistingStatus = numLeafCountBefore - 1)) + + // incr 3: append enough files to push the root past the cap, forcing a spill. + commitBoth( + baselineDeltaLog, amtDeltaLog, + (leafPackedFiles + 2 to leafPackedFiles + 15).map(fakeAdd)) + // Spills; the exact shape depends on spill order, so only the live set is asserted. + commitCheckpoint(amtDeltaLog, incremental = true) + assertLiveAddFilesEquals(baselineDeltaLog, amtDeltaLog) + } + } + } + + test("a long mixed chain of writes folds into ONE incremental AMT") { + withSQLConf(leafPackingConfs: _*) { + withTables() { (baselineDeltaLog, amtDeltaLog) => + commitBoth(baselineDeltaLog, amtDeltaLog, (1 to leafPackedFiles).map(fakeAdd)) + commitCheckpoint(amtDeltaLog, incremental = false) + val leafToAddFileMapping = leafToAddFileMap(amtDeltaLog) + assert(leafToAddFileMapping.size >= 2, + s"Need a tree-shaped bootstrap; got ${leafToAddFileMapping.size} leaves.") + + // The opposite packing: NOTHING is checkpointed until the very end, so instead of + // one write per incremental, all these interleaved appends and deletes land in a single + // incremental's intermediate commits, to be folded in at once. + val leafVictims = leafToAddFileMapping.toSeq.sortBy(_._1).flatMap(_._2.take(1)) + commitBoth(baselineDeltaLog, amtDeltaLog, Seq(fakeAdd(leafPackedFiles + 1))) + commitBoth(baselineDeltaLog, amtDeltaLog, Seq(leafVictims.head.remove)) + commitBoth( + baselineDeltaLog, amtDeltaLog, + Seq(fakeAdd(leafPackedFiles + 2), fakeAdd(leafPackedFiles + 3))) + commitBoth(baselineDeltaLog, amtDeltaLog, Seq(leafVictims(1).remove)) + // A root-resident file added and removed inside the same stream is net-zero. + commitBoth(baselineDeltaLog, amtDeltaLog, Seq(fakeAdd(leafPackedFiles + 4))) + commitBoth(baselineDeltaLog, amtDeltaLog, Seq(removeOf(amtDeltaLog, leafPackedFiles + 4))) + commitBoth(baselineDeltaLog, amtDeltaLog, Seq(fakeAdd(leafPackedFiles + 5))) + + // One incremental for all seven writes: the 4 surviving net-new files stay root-resident + // (2 leaf pointers + 4 adds is under the cap of 10), and the two leaf victims each + // contribute one MDV bit to their own leaf. + // The intermediate commits are the 7 writes plus the bootstrap's own OPTIMIZE CHECKPOINT. + createIncrementalAMTAndValidate( + baselineDeltaLog, + amtDeltaLog, + createIncrementalAMTWriteMetrics( + numOldLeavesUpdated = 2, + numOldLeavesUntouched = leafToAddFileMapping.size - 2, + numRootEntriesExistingStatus = 4, + numLeafMdvBitsAdded = 2, + numLeavesModifiedStatus = 2, + numLeavesExistingStatus = leafToAddFileMapping.size - 2), + expectedNumIntermediateCommits = Some(8)) + } + } + } + + test("re-adding an already-live file leaves it live exactly once, no double-count") { + withSQLConf(leafPackingConfs: _*) { + withTables() { (baselineDeltaLog, amtDeltaLog) => + commitBoth(baselineDeltaLog, amtDeltaLog, (1 to leafPackedFiles).map(fakeAdd)) + commitCheckpoint(amtDeltaLog, incremental = false) + val leafToAddFileMapping = leafToAddFileMap(amtDeltaLog) + assert(leafToAddFileMapping.size >= 2, + s"Need a tree-shaped bootstrap; got ${leafToAddFileMapping.size} leaves.") + val livePathsBefore = livePathsInLatestAMTCheckpoint(amtDeltaLog) + + // Re-commit a currently-live leaf file's AddFile with NO remove. + // The re-added AddFile carries the leaf back reference it was reconstructed with. + val liveLeafFile = leafToAddFileMapping.toSeq.sortBy(_._1).head._2.head + assert(liveLeafFile.backReference.isDefined, "The re-added file must be leaf-resident.") + commitBoth( + baselineDeltaLog, amtDeltaLog, Seq(liveLeafFile), + operation = DeltaOperations.ComputeStats(predicate = Nil)) + // The re-added AddFile carries the leaf back reference it was reconstructed with, so the + // writer recognizes it as a re-commit of an already-live file and keeps it EXISTING in the + // root (numRootEntriesExistingStatus = 1), not ADDED. The back reference marks the original + // leaf slot as superseded, so that leaf's MDV masks it (numOldLeavesUpdated = 1, + // numLeafMdvBitsAdded = 1). The root EXISTING copy and the masked leaf slot net to no + // change in the live set, and -- crucially -- the file is surfaced exactly once rather + // than once from the leaf and once from the root. + createIncrementalAMTAndValidate( + baselineDeltaLog, + amtDeltaLog, + createIncrementalAMTWriteMetrics( + numOldLeavesUpdated = 1, + numOldLeavesUntouched = leafToAddFileMapping.size - 1, + numRootEntriesExistingStatus = 1, + numLeafMdvBitsAdded = 1, + numLeavesModifiedStatus = 1, + numLeavesExistingStatus = leafToAddFileMapping.size - 1)) + assert(livePathsInLatestAMTCheckpoint(amtDeltaLog) == livePathsBefore, + "Re-adding an already-live file must not change the reconstructed live set.") + } + } + } + + test("a deferred incremental AMT describes attemptVersion - 1") { + withSQLConf(leafPackingConfs: _*) { + withTables() { (baselineDeltaLog, amtDeltaLog) => + commitBoth(baselineDeltaLog, amtDeltaLog, (1 to leafPackedFiles).map(fakeAdd)) + commitCheckpoint(amtDeltaLog, incremental = false) + val numLeafCountBefore = + amtProvider(amtDeltaLog.update()).getOrElse(fail("expected AMT")).leaves.size + assert(numLeafCountBefore >= 2, + s"Need a tree-shaped bootstrap; got $numLeafCountBefore leaves.") + commitBoth(baselineDeltaLog, amtDeltaLog, Seq(fakeAdd(leafPackedFiles + 1))) + val lastCommitted = amtDeltaLog.update().version + createIncrementalAMTAndValidate( + baselineDeltaLog, + amtDeltaLog, + createIncrementalAMTWriteMetrics( + numOldLeavesUntouched = numLeafCountBefore, + numRootEntriesExistingStatus = 1, + numLeavesExistingStatus = numLeafCountBefore)) + val provider = amtProvider(amtDeltaLog.update()).getOrElse(fail("expected AMT")) + // A deferred OPTIMIZE CHECKPOINT (no user actions) describes the last committed version, + // i.e. attemptVersion - 1. + assert(provider.checkpointAction.version == lastCommitted, + s"Deferred checkpoint must describe the last committed version $lastCommitted; " + + s"got ${provider.checkpointAction.version}.") + } + } + } + + test("lastManifestCommitWithFullRewrite is pinned across a chain of incremental AMTs") { + withSQLConf(leafPackingConfs: _*) { + withTables() { (baselineDeltaLog, amtDeltaLog) => + commitBoth(baselineDeltaLog, amtDeltaLog, (1 to leafPackedFiles).map(fakeAdd)) + commitCheckpoint(amtDeltaLog, incremental = false) + val numLeafCountBefore = + amtProvider(amtDeltaLog.update()).getOrElse(fail("expected AMT")).leaves.size + assert(numLeafCountBefore >= 2, + s"Need a tree-shaped bootstrap; got $numLeafCountBefore leaves.") + val fullMarker = amtProvider(amtDeltaLog.update()).getOrElse(fail("expected AMT")) + .checkpointAction.contentRoot.lastManifestCommitWithFullRewrite + assert(fullMarker.isDefined, "The full rewrite must set the last-full-rewrite marker.") + // CREATE TABLE is v0 and the 30-file write is v1, so the full rewrite committed at v2 and + // describes v1. + assert(fullMarker.contains(1L), s"The full rewrite must be pinned to v1; got $fullMarker.") + assertCheckpointDescribesVersion(amtDeltaLog, expectedVersion = 1L) + + // Two successive incrementals must both carry the SAME marker (pinned to the full rewrite), + // while each describes the write it followed: the write lands at v3 and its deferred + // checkpoint at v4 describes v3, then v5 / v6 describes v5. + commitBoth(baselineDeltaLog, amtDeltaLog, Seq(fakeAdd(leafPackedFiles + 1))) + createIncrementalAMTAndValidate( + baselineDeltaLog, + amtDeltaLog, + createIncrementalAMTWriteMetrics( + numOldLeavesUntouched = numLeafCountBefore, + numRootEntriesExistingStatus = 1, + numLeavesExistingStatus = numLeafCountBefore)) + assertCheckpointDescribesVersion(amtDeltaLog, expectedVersion = 3L) + commitBoth(baselineDeltaLog, amtDeltaLog, Seq(fakeAdd(leafPackedFiles + 2))) + createIncrementalAMTAndValidate( + baselineDeltaLog, + amtDeltaLog, + createIncrementalAMTWriteMetrics( + numOldLeavesUntouched = numLeafCountBefore, + numRootEntriesExistingStatus = 2, + numLeavesExistingStatus = numLeafCountBefore)) + assertCheckpointDescribesVersion(amtDeltaLog, expectedVersion = 5L) + val marker = amtProvider(amtDeltaLog.update()).getOrElse(fail("expected AMT")) + .checkpointAction.contentRoot.lastManifestCommitWithFullRewrite + assert(marker == fullMarker, + s"Incrementals must carry the full-rewrite marker forward unchanged: " + + s"full=$fullMarker incr=$marker.") + } + } + } +} diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTIncrementalWriteInlineSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTIncrementalWriteInlineSuite.scala new file mode 100644 index 00000000000..007397e6ae0 --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTIncrementalWriteInlineSuite.scala @@ -0,0 +1,914 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.amt + +// scalastyle:off import.ordering.noEmptyLine +import org.apache.spark.sql.delta.DeltaOperations +import org.apache.spark.sql.delta.sources.DeltaSQLConf +import org.apache.spark.sql.delta.test.shims.GridTestShim + +/** The inline emission path: statuses, tombstones, DELETE vs REPLACE, and inline re-adds. */ +class AMTIncrementalWriteInlineSuite extends AMTIncrementalWriteTestBase with GridTestShim { + + test("an inline write emits an incremental AMT that describes its own commit version") { + withSQLConf(DeltaSQLConf.AMT_ENTRIES_PER_LEAF.key -> "10") { + withTables() { (baselineDeltaLog, amtDeltaLog) => + val leavesBefore = setup(baselineDeltaLog, amtDeltaLog, entriesPerLeaf = 10, + initialIdRangeInLeaf = 1 to 30).map(_.location).toSet + assert(leavesBefore.size >= 2, + s"Need a tree-shaped bootstrap; got ${leavesBefore.size} leaves.") + + // Inline-append one net-new file. It stays root-resident (carried pointers + 1 add is under + // the cap), so no leaf is added or rewritten. + createIncrementalAMTAndValidate( + baselineDeltaLog, amtDeltaLog, + createIncrementalAMTWriteMetrics( + numOldLeavesUntouched = leavesBefore.size, + numRootEntriesAddedStatus = 1, + numLeavesExistingStatus = leavesBefore.size), + inlineAMTCommitActions = Some(Seq(fakeAdd(31)))) + assert(leafPointers(amtDeltaLog.update()).keySet == leavesBefore, + "An append below the spill threshold must add no leaf and rewrite none.") + } + } + } + + test("an inline-appended file is ADDED, then EXISTING once carried by the next commit") { + withSQLConf(DeltaSQLConf.AMT_ENTRIES_PER_LEAF.key -> "10") { + withTables() { (baselineDeltaLog, amtDeltaLog) => + setup(baselineDeltaLog, amtDeltaLog, entriesPerLeaf = 10, initialIdRangeInLeaf = 1 to 30) + // A full-rewrite root holds only leaf pointers, so it has no root-resident DATA entries. + val bootstrapCounts = rootDataEntryStatusToCount(amtDeltaLog) + assert(bootstrapCounts.isEmpty, + s"A full-rewrite root must hold no DATA entries; got $bootstrapCounts.") + // Append one net-new file INLINE: this commit inserts it, so it is ADDED. (A deferred fold + // would carry it forward as EXISTING, since that insert belongs to its own commit's CDF.) + withInline { commitBoth(baselineDeltaLog, amtDeltaLog, Seq(fakeAdd(31))) } + val afterFirst = rootDataEntryStatusToLocations(amtDeltaLog) + assert(afterFirst == Map(Tracking.Status.Added -> Set(fakeAdd(31).path)), + s"the freshly appended file 31 must be ADDED; got $afterFirst.") + assertLiveAddFilesEquals(baselineDeltaLog, amtDeltaLog) + // Append a second file INLINE: this commit inserts the second (32) as ADDED and carries the + // first (31), which decays to EXISTING. + withInline { commitBoth(baselineDeltaLog, amtDeltaLog, Seq(fakeAdd(32))) } + val afterSecond = rootDataEntryStatusToLocations(amtDeltaLog) + assert(afterSecond == Map( + Tracking.Status.Added -> Set(fakeAdd(32).path), + Tracking.Status.Existing -> Set(fakeAdd(31).path)), + s"file 32 must be ADDED and file 31 EXISTING; got $afterSecond.") + assertLiveAddFilesEquals(baselineDeltaLog, amtDeltaLog) + } + } + } + + test("an inline delete of a leaf-resident file masks it via a cumulative MDV") { + withSQLConf(DeltaSQLConf.AMT_ENTRIES_PER_LEAF.key -> "10") { + withTables() { (baselineDeltaLog, amtDeltaLog) => + val leavesBefore = setup(baselineDeltaLog, amtDeltaLog, entriesPerLeaf = 10, + initialIdRangeInLeaf = 1 to 30).map(_.location).toSet + assert(leavesBefore.size >= 2, + s"Need a tree-shaped bootstrap; got ${leavesBefore.size} leaves.") + val physicalEntriesBefore = leafLiveDataEntryCount(amtDeltaLog.update()) + assert(physicalEntriesBefore == 30, + "Every bootstrap file is spread across the AMT's leaves.") + + // Inline delete of a leaf-resident file: the owning leaf is carried forward by pointer and + // masked with one cumulative MDV bit; the leaf parquet keeps every physical entry. + createIncrementalAMTAndValidate( + baselineDeltaLog, amtDeltaLog, + createIncrementalAMTWriteMetrics( + numOldLeavesUpdated = 1, + numOldLeavesUntouched = leavesBefore.size - 1, + numLeafMdvBitsAdded = 1, + numLeafDeleteCDFBitsAdded = 1, + numLeavesExistingStatus = leavesBefore.size - 1, + numLeavesModifiedStatus = 1), + inlineAMTCommitActions = Some(Seq(removeOf(amtDeltaLog, 1)))) + assert(leafPointers(amtDeltaLog.update()).keySet == leavesBefore, + "The delete must carry every leaf forward, not rewrite or drop one.") + assert(leafLiveDataEntryCount(amtDeltaLog.update()) == physicalEntriesBefore, + "The carried leaf keeps every physical entry; a delete only sets the MDV.") + } + } + } + + test("an inline delete of a root-resident file writes a root tombstone for CDF") { + withSQLConf(DeltaSQLConf.AMT_ENTRIES_PER_LEAF.key -> "10") { + withTables() { (baselineDeltaLog, amtDeltaLog) => + val leavesBefore = setup(baselineDeltaLog, amtDeltaLog, entriesPerLeaf = 10, + initialIdRangeInLeaf = 1 to 30).map(_.location).toSet + assert(leavesBefore.size >= 2, + s"Need a tree-shaped bootstrap; got ${leavesBefore.size} leaves.") + + // Inline-append one net-new file; it stays root-resident (under the spill threshold), so + // its later remove carries no back reference. + val rootId = 31 + withInline { + commitBoth(baselineDeltaLog, amtDeltaLog, Seq(fakeAdd(rootId))) + } + val (rootLiveAddsBefore, tombstonesBefore) = liveAddsAndTombstonesCountInRoot(amtDeltaLog) + assert(rootLiveAddsBefore == 1L, + s"The appended file must be the one root-resident live add; got $rootLiveAddsBefore.") + assert(tombstonesBefore == 0L, "An append must not write a tombstone.") + + // Inline delete of the root-resident file. The deferred path drops such a file through + // replay with NO tombstone; inline instead has the remove in actionsToCommit, so it + // becomes a tracking=removed root entry for CDF. + createIncrementalAMTAndValidate( + baselineDeltaLog, amtDeltaLog, + createIncrementalAMTWriteMetrics( + numOldLeavesUntouched = leavesBefore.size, + numRootEntriesDeletedStatus = 1, + numLeavesExistingStatus = leavesBefore.size), + inlineAMTCommitActions = Some(Seq(removeOf(amtDeltaLog, rootId)))) + val (rootLiveAdds, tombstones) = liveAddsAndTombstonesCountInRoot(amtDeltaLog) + assert(tombstones == 1L, + s"Removing a root-resident file inline must leave one root tombstone; got $tombstones.") + assert(rootLiveAdds == 0L, + s"The removed file must no longer be a live root add; got $rootLiveAdds.") + // A no-backref remove is replay-resolved; it must not touch any leaf's MDV. + leafPointers(amtDeltaLog.update()).foreach { case (location, leaf) => + assert(mdvCardinality(leaf) == 0L, + s"Removing a root-resident file must not touch leaf $location's MDV.") + } + } + } + } + + test("an inline leaf delete stamps deleted_positions, and resets it on the next commit") { + withSQLConf(DeltaSQLConf.AMT_ENTRIES_PER_LEAF.key -> "10") { + withTables() { (baselineDeltaLog, amtDeltaLog) => + val leavesBefore = setup(baselineDeltaLog, amtDeltaLog, entriesPerLeaf = 10, + initialIdRangeInLeaf = 1 to 30).map(_.location).toSet + assert(leavesBefore.size >= 2, + s"Need a tree-shaped bootstrap; got ${leavesBefore.size} leaves.") + + // Inline delete of a leaf-resident file stamps this commit's deleted position on the owning + // leaf. deleted_positions is sourced from this commit's with-backref removes, so only the + // inline path populates it. + createIncrementalAMTAndValidate( + baselineDeltaLog, amtDeltaLog, + createIncrementalAMTWriteMetrics( + numOldLeavesUpdated = 1, + numOldLeavesUntouched = leavesBefore.size - 1, + numLeafMdvBitsAdded = 1, + numLeafDeleteCDFBitsAdded = 1, + numLeavesExistingStatus = leavesBefore.size - 1, + numLeavesModifiedStatus = 1), + inlineAMTCommitActions = Some(Seq(removeOf(amtDeltaLog, 1)))) + val stamped = + leafPointers(amtDeltaLog.update()).values.filter(leafDeletedPositions(_).nonEmpty).toSeq + assert(stamped.size == 1, + s"Exactly one leaf must carry this commit's deleted_positions; got ${stamped.size}.") + assert(leafDeletedPositions(stamped.head).size == 1, + s"deleted_positions must hold this commit's single deletion; " + + s"got ${leafDeletedPositions(stamped.head)}.") + assert(mdvCardinality(stamped.head) == 1L, + "The same leaf's cumulative MDV must also carry that one bit.") + + // A following inline commit that deletes from no leaf must RESET deleted_positions (it is + // per-commit, never the stale prior value); the cumulative MDV persists. + createIncrementalAMTAndValidate( + baselineDeltaLog, amtDeltaLog, + createIncrementalAMTWriteMetrics( + numOldLeavesUntouched = leavesBefore.size, + numRootEntriesAddedStatus = 1, + numLeavesExistingStatus = leavesBefore.size), + inlineAMTCommitActions = Some(Seq(fakeAdd(31)))) + val afterAppend = amtDeltaLog.update() + leafPointers(afterAppend).foreach { case (location, leaf) => + assert(leafDeletedPositions(leaf).isEmpty, + s"deleted_positions must reset on leaf $location when this commit deletes nothing " + + s"from it; got ${leafDeletedPositions(leaf)}.") + } + assert(leafPointers(afterAppend).values.map(mdvCardinality).sum == 1L, + "The cumulative MDV bit from the earlier delete must survive the append.") + } + } + } + + + test("an inline leaf-resident REPLACE stamps replaced_positions and re-adds MODIFIED") { + withSQLConf(DeltaSQLConf.AMT_ENTRIES_PER_LEAF.key -> "10") { + withTables() { (baselineDeltaLog, amtDeltaLog) => + val leavesBefore = setup(baselineDeltaLog, amtDeltaLog, entriesPerLeaf = 10, + initialIdRangeInLeaf = 1 to 30).map(_.location).toSet + assert(leavesBefore.size >= 2, s"Need a tree-shaped bootstrap; got ${leavesBefore.size}.") + + // Inline REPLACE of a leaf file: remove f, re-add it under a new DV, in one commit. + // The owning leaf is carried forward with the position masked in its cumulative MDV and + // recorded as this commit's replaced_positions (not deleted_positions). The re-added + // copy is a live MODIFIED root DATA entry. + createIncrementalAMTAndValidate( + baselineDeltaLog, amtDeltaLog, + createIncrementalAMTWriteMetrics( + numOldLeavesUpdated = 1, + numOldLeavesUntouched = leavesBefore.size - 1, + numRootEntriesModifiedStatus = 1, + numLeafMdvBitsAdded = 1, + numLeafReplaceCDFBitsAdded = 1, + numLeavesExistingStatus = leavesBefore.size - 1, + numLeavesModifiedStatus = 1), + inlineAMTCommitActions = Some(removeAndReAddWithDV(amtDeltaLog, Left(1)))) + val replaced = leafPointers(amtDeltaLog.update()).values + .filter(leafReplacedPositions(_).nonEmpty).toSeq + assert(replaced.size == 1, + s"Exactly one leaf must carry this commit's replaced_positions; got ${replaced.size}.") + assert(leafReplacedPositions(replaced.head).size == 1 && + leafDeletedPositions(replaced.head).isEmpty, + s"REPLACE sets replaced_positions, not deleted; got ${replaced.head.tracking}.") + assert( + rootDataEntryStatusToCount(amtDeltaLog).getOrElse(Tracking.Status.Modified, 0L) == 1L, + "The re-added copy must be a MODIFIED root DATA entry.") + assertLiveAddFilesEquals(baselineDeltaLog, amtDeltaLog) + } + } + } + + test("an inline root-resident REPLACE writes a REPLACED root entry, not DELETED") { + withSQLConf(DeltaSQLConf.AMT_ENTRIES_PER_LEAF.key -> "10") { + withTables() { (baselineDeltaLog, amtDeltaLog) => + val leavesBefore = setup(baselineDeltaLog, amtDeltaLog, entriesPerLeaf = 10, + initialIdRangeInLeaf = 1 to 30).map(_.location).toSet + assert(leavesBefore.size >= 2, s"Need a tree-shaped bootstrap; got ${leavesBefore.size}.") + // Append a net-new file; being root-resident, its later remove carries no back reference. + val rootId = 31 + withInline { commitBoth(baselineDeltaLog, amtDeltaLog, Seq(fakeAdd(rootId))) } + + // Inline REPLACE of the root-resident file: a no-backref remove whose path is re-added this + // commit becomes a REPLACED root DataEntry (buildRootRemoveEntries), and the re-added copy + // is MODIFIED -- unlike a pure root delete, which leaves a DELETED tombstone. + createIncrementalAMTAndValidate( + baselineDeltaLog, amtDeltaLog, + createIncrementalAMTWriteMetrics( + numOldLeavesUntouched = leavesBefore.size, + numRootEntriesModifiedStatus = 1, + numRootEntriesReplacedStatus = 1, + numLeavesExistingStatus = leavesBefore.size), + inlineAMTCommitActions = Some(removeAndReAddWithDV(amtDeltaLog, Left(rootId)))) + val rootCounts = rootDataEntryStatusToCount(amtDeltaLog) + assert(rootCounts.getOrElse(Tracking.Status.Replaced, 0L) == 1L, + s"A re-added root file must leave one REPLACED root entry; got $rootCounts.") + assert(rootCounts.getOrElse(Tracking.Status.Modified, 0L) == 1L, + s"The re-added copy must be MODIFIED; got $rootCounts.") + assertLiveAddFilesEquals(baselineDeltaLog, amtDeltaLog) + } + } + } + + test("a root DATA entry goes ADDED -> EXISTING -> MODIFIED across incremental writes") { + withSQLConf(DeltaSQLConf.AMT_ENTRIES_PER_LEAF.key -> "10") { + withTables() { (baselineDeltaLog, amtDeltaLog) => + val leavesBefore = setup(baselineDeltaLog, amtDeltaLog, entriesPerLeaf = 10, + initialIdRangeInLeaf = 1 to 30).map(_.location).toSet + val rootId = 31 + + // Append rootId inline: it is an ADDED root DATA entry. + withInline { commitBoth(baselineDeltaLog, amtDeltaLog, Seq(fakeAdd(rootId))) } + assert(rootDataEntryStatusToCount(amtDeltaLog) == Map(Tracking.Status.Added -> 1L), + s"the appended root file must start ADDED; got " + + s"${rootDataEntryStatusToCount(amtDeltaLog)}.") + + // Carry rootId forward under an unrelated inline append: rootId decays ADDED -> EXISTING + // while the filler is the new ADDED entry. + val fillerId = 32 + createIncrementalAMTAndValidate( + baselineDeltaLog, amtDeltaLog, + createIncrementalAMTWriteMetrics( + numOldLeavesUntouched = leavesBefore.size, + numRootEntriesAddedStatus = 1, + numRootEntriesExistingStatus = 1, + numLeavesExistingStatus = leavesBefore.size), + inlineAMTCommitActions = Some(Seq(fakeAdd(fillerId)))) + assert(rootDataEntryStatusToCount(amtDeltaLog) == + Map(Tracking.Status.Added -> 1L, Tracking.Status.Existing -> 1L), + s"rootId must decay to EXISTING while the filler is ADDED; " + + s"got ${rootDataEntryStatusToCount(amtDeltaLog)}.") + + // Re-add rootId with a new DV: its EXISTING copy becomes a REPLACED root entry and the + // re-added copy is MODIFIED. + createIncrementalAMTAndValidate( + baselineDeltaLog, amtDeltaLog, + createIncrementalAMTWriteMetrics( + numOldLeavesUntouched = leavesBefore.size, + numRootEntriesExistingStatus = 1, + numRootEntriesModifiedStatus = 1, + numRootEntriesReplacedStatus = 1, + numLeavesExistingStatus = leavesBefore.size), + inlineAMTCommitActions = Some(removeAndReAddWithDV(amtDeltaLog, Left(rootId)))) + val finalCounts = rootDataEntryStatusToCount(amtDeltaLog) + assert(finalCounts.getOrElse(Tracking.Status.Modified, 0L) == 1L && + finalCounts.getOrElse(Tracking.Status.Replaced, 0L) == 1L, + s"the re-added rootId must be MODIFIED with a REPLACED prior entry; got $finalCounts.") + } + } + } + + test("a freshly spilled leaf can have ADDED/MODIFIED/EXISTING and its manifest_info " + + "counts partition its entries") { + // A cap of 10 entries per leaf: 20 files pack into 2 whole leaves, and the 8 live entries this + // commit produces spill whole into one new leaf. + withSQLConf(DeltaSQLConf.AMT_ENTRIES_PER_LEAF.key -> "10") { + withTables() { (baselineDeltaLog, amtDeltaLog) => + // Old tree: 5 root-resident DATA entries over 2 leaves (files 1..20 packed into 2 whole + // leaves, then files 21..25 appended root-resident under the cap of 10). + val oldLeafLocations = setup(baselineDeltaLog, amtDeltaLog, entriesPerLeaf = 10, + initialIdRangeInLeaf = 1 to 20, initialIdRangeInRoot = 21 to 25, + rowCountPerDataEntry = 20).map(_.location).toSet + assert(rootDataEntryStatusToCount(amtDeltaLog).values.sum == 5, + "the old root must hold exactly the 5 root-resident DATA entries.") + + // A deferred (window) log commit appends files 26..28 (10 rows each); not checkpointed, so + // they enter the next incremental write as intermediate window commits. + val windowIds = 26 to 28 + commitBoth(baselineDeltaLog, amtDeltaLog, + windowIds.map(id => fakeAdd(id, dataChange = true, numRecords = 10L))) + + // One inline commit exercising every live-status class: + // - REPLACE 3 root files (remove + re-add under a new DV): 3 REPLACED + 3 MODIFIED. + // - DELETE 1 root file (remove, no re-add): 1 DELETED. + // - REPLACE 2 window files: 2 REPLACED + 2 MODIFIED. + // - REPLACE 1 leaf-resident file: masks its old slot in that leaf (which becomes + // MODIFIED) and re-adds it live as 1 MODIFIED. + // - 1 net-new insert: 1 ADDED. + // The untouched 5th root file and 3rd window file stay EXISTING. The 9 live entries + // (1 ADDED + 6 MODIFIED + 2 EXISTING) overflow the cap of 10 and spill whole into one new + // leaf; the 6 root tombstones (5 REPLACED + 1 DELETED) stay root-resident, while the + // replaced leaf slot is masked in its now-MODIFIED leaf. + val replaceRoots = + Seq(21, 22, 23).flatMap(id => removeAndReAddWithDV(amtDeltaLog, Left(id))) + val deleteRoot = Seq(removeOf(amtDeltaLog, 24)) + val replaceWindows = + Seq(26, 27).flatMap(id => removeAndReAddWithDV(amtDeltaLog, Left(id))) + // File 1 is leaf-resident (one of the 20 files packed into the 2 full-rewrite leaves). + val replaceLeaf = removeAndReAddWithDV(amtDeltaLog, Left(1)) + val netNewAdd = Seq(fakeAdd(29, dataChange = true, numRecords = 30L)) + val inlineActions = + replaceRoots ++ deleteRoot ++ replaceWindows ++ replaceLeaf ++ netNewAdd + + createIncrementalAMTAndValidate( + baselineDeltaLog, + amtDeltaLog, + createIncrementalAMTWriteMetrics( + numOldLeavesUpdated = 1, + numOldLeavesUntouched = 1, + numNewLeaves = 1, + numRootEntriesReplacedStatus = 5, + numRootEntriesDeletedStatus = 1, + numLeafMdvBitsAdded = 1, + numLeafReplaceCDFBitsAdded = 1, + numLeavesAddedStatus = 1, + numLeavesModifiedStatus = 1, + numLeavesExistingStatus = 1), + expectedNumIntermediateCommits = Some(2), + inlineAMTCommitActions = Some(inlineActions)) + + // Exactly one freshly spilled leaf, pointer ADDED, holding all 8 live entries. + val newLeaves = leafPointers(amtDeltaLog.update()) + val spilled = (newLeaves.keySet -- oldLeafLocations).toSeq + assert(spilled.size == 1, s"exactly one new leaf must spill; got $spilled.") + val spilledLeaf = newLeaves(spilled.head) + assert(spilledLeaf.tracking.status == Tracking.Status.Added, + s"a freshly spilled leaf pointer must be ADDED; got ${spilledLeaf.tracking.status}.") + assert(spilledLeaf.record_count == 9, + s"the spilled leaf must hold 9 entries; got ${spilledLeaf.record_count}.") + + // The leaf parquet's raw per-entry statuses: ADDED, MODIFIED, and EXISTING all present. + val provider = amtProvider(amtDeltaLog.update()).getOrElse(fail("expected AMT")) + val leafStatusToCount = trackingStatusToAddFileCountMap( + spilledLeaf.getAbsolutePath(provider.tableRoot).toString) + assert(leafStatusToCount == Map( + Tracking.Status.Added -> 1L, + Tracking.Status.Modified -> 6L, + Tracking.Status.Existing -> 2L), + s"leaf entries must be 1 ADDED + 6 MODIFIED + 2 EXISTING; got $leafStatusToCount.") + + // manifest_info counts each status separately: 1 ADDED, 6 MODIFIED, 2 EXISTING, and a live + // spilled leaf carries no tombstone counts. + val mi = spilledLeaf.manifest_info + assert(mi.added_files_count == 1 && mi.modified_files_count == 6 && + mi.existing_files_count == 2 && + mi.deleted_files_count == 0 && mi.replaced_files_count == 0, + s"leaf manifest_info must be added=1, modified=6, existing=2, no tombstones; got $mi.") + assert( + mi.added_files_count + mi.modified_files_count + mi.existing_files_count == + spilledLeaf.record_count.toInt, + s"added + modified + existing must partition the leaf's ${spilledLeaf.record_count} " + + s"entries; got $mi.") + + // Row counts sum physical records, so they diverge from the file counts here. ADDED rows = + // the 30-row net-new file 29; MODIFIED rows = 6 single-row re-adds; EXISTING rows = the + // 20-row root file 25 + the 10-row window file 28 = 30; no tombstone rows. + assert(mi.added_rows_count == 30L && mi.modified_rows_count == 6L && + mi.existing_rows_count == 30L && + mi.deleted_rows_count == 0L && mi.replaced_rows_count == 0L, + s"leaf row counts must be added=30, modified=6, existing=30, no tombstones; got $mi.") + } + } + } + + test("an inline commit deleting one leaf file and replacing another on the same leaf") { + withSQLConf(DeltaSQLConf.AMT_ENTRIES_PER_LEAF.key -> "10") { + withTables() { (baselineDeltaLog, amtDeltaLog) => + setup(baselineDeltaLog, amtDeltaLog, entriesPerLeaf = 10, initialIdRangeInLeaf = 1 to 30) + val leafToAddFileMapping = leafToAddFileMap(amtDeltaLog) + val (targetLeaf, files) = leafToAddFileMapping.toSeq.sortBy(_._1).find(_._2.size >= 2) + .getOrElse(fail("need a leaf holding at least two files.")) + val leavesBefore = leafToAddFileMapping.keySet + val fileToDelete = files.head + val fileToReplace = files(1) + + // One inline commit: DELETE fileToDelete (no re-add) and REPLACE fileToReplace (remove + + // re-add with a new DV). Both land on the same leaf, so its pointer carries this commit's + // deleted_positions AND replaced_positions (MODIFIED), with two masked MDV bits. + val replaceActions = removeAndReAddWithDV(amtDeltaLog, Right(fileToReplace)) + createIncrementalAMTAndValidate( + baselineDeltaLog, amtDeltaLog, + createIncrementalAMTWriteMetrics( + numOldLeavesUpdated = 1, + numOldLeavesUntouched = leavesBefore.size - 1, + numRootEntriesModifiedStatus = 1, + numLeafMdvBitsAdded = 2, + numLeafDeleteCDFBitsAdded = 1, + numLeafReplaceCDFBitsAdded = 1, + numLeavesModifiedStatus = 1, + numLeavesExistingStatus = leavesBefore.size - 1), + inlineAMTCommitActions = Some(fileToDelete.remove +: replaceActions)) + val touched = leafPointers(amtDeltaLog.update()).values + .filter(l => leafDeletedPositions(l).nonEmpty || leafReplacedPositions(l).nonEmpty).toSeq + assert(touched.size == 1, s"exactly one leaf must be touched; got ${touched.size}.") + assert(leafDeletedPositions(touched.head).size == 1 && + leafReplacedPositions(touched.head).size == 1, + s"leaf must carry a deleted AND replaced position; got ${touched.head.tracking}.") + assertLiveAddFilesEquals(baselineDeltaLog, amtDeltaLog) + } + } + } + + test("an overflowing tombstone leaf is born ADDED, decays to DELETED, then is dropped") { + withSQLConf(DeltaSQLConf.AMT_ENTRIES_PER_LEAF.key -> "10") { + withTables() { (baselineDeltaLog, amtDeltaLog) => + setup(baselineDeltaLog, amtDeltaLog, entriesPerLeaf = 10, + initialIdRangeInLeaf = 1 to 30, initialIdRangeInRoot = 31 to 36, + rowCountPerDataEntry = 10) + // File 37 arrives as a separate (uncheckpointed) log commit carrying 20 rows. + commitBoth(baselineDeltaLog, amtDeltaLog, + Seq(fakeAdd(37, dataChange = true, numRecords = 20L))) + // Inline: delete every root-resident file (no-backref -> DELETED tombstones) and append + // enough net-new files that the live adds spill into new leaves; the extra pointers push + // the tombstones past the cap, so they spill into their own leaf. + val actions = + (31 to 37).map(id => removeOf(amtDeltaLog, id)) ++ (130 to 150).map(fakeAdd) + withInline { commitBoth(baselineDeltaLog, amtDeltaLog, actions) } + + // The freshly spilled tombstone leaf (7 DELETED entries, no live file) is born ADDED, not + // DELETED, and no leaf is DELETED on the commit that writes it. + val bornLeaves = leafPointers(amtDeltaLog.update()) + val tombstoneLeaves = + bornLeaves.values.filter(_.manifest_info.deleted_files_count > 0).toSeq + assert(tombstoneLeaves.size == 1, + s"exactly one spilled tombstone leaf; got ${tombstoneLeaves.size}.") + val tombstoneLeaf = tombstoneLeaves.head + assert(tombstoneLeaf.tracking.status == Tracking.Status.Added, + s"a freshly spilled tombstone leaf is born ADDED; got ${tombstoneLeaf.tracking.status}.") + assert(tombstoneLeaf.manifest_info.added_files_count == 0 && + tombstoneLeaf.manifest_info.existing_files_count == 0 && + tombstoneLeaf.manifest_info.modified_files_count == 0, + s"the tombstone leaf holds no live entries; got ${tombstoneLeaf.manifest_info}.") + // The 7 DELETED tombstones carry their files' physical rows: six 10-row root files plus the + // 20-row window file 37 = 80 deleted rows, versus 7 deleted files, and no live rows. + val tombstoneMi = tombstoneLeaf.manifest_info + assert(tombstoneMi.deleted_files_count == 7 && tombstoneMi.deleted_rows_count == 80L && + tombstoneMi.replaced_rows_count == 0L, + s"tombstone leaf must count 7 deleted files and 80 deleted rows; got $tombstoneMi.") + assert(tombstoneMi.added_rows_count == 0L && tombstoneMi.existing_rows_count == 0L && + tombstoneMi.modified_rows_count == 0L, + s"the tombstone leaf holds no live rows; got $tombstoneMi.") + assert(!bornLeaves.values.exists(_.tracking.status == Tracking.Status.Deleted), + "no leaf is DELETED on the commit that spills the tombstone leaf.") + assertLiveAddFilesEquals(baselineDeltaLog, amtDeltaLog) + + // Next AMT: the carried tombstone leaf holds no live file, so it decays to DELETED. + val tombstoneLoc = tombstoneLeaf.location + commitCheckpoint(amtDeltaLog, incremental = true) + assert(leafPointers(amtDeltaLog.update())(tombstoneLoc).tracking.status == + Tracking.Status.Deleted, + "a carried leaf with no live file must decay to DELETED.") + + // Next AMT: the DELETED leaf is dropped. + commitCheckpoint(amtDeltaLog, incremental = true) + assert(!leafPointers(amtDeltaLog.update()).contains(tombstoneLoc), + "a DELETED leaf must be dropped by the next AMT.") + } + } + } + + test("a spilled tombstone leaf counts a REPLACED + DELETED mix") { + withSQLConf(DeltaSQLConf.AMT_ENTRIES_PER_LEAF.key -> "10") { + withTables() { (baselineDeltaLog, amtDeltaLog) => + setup(baselineDeltaLog, amtDeltaLog, entriesPerLeaf = 10, + initialIdRangeInLeaf = 1 to 30, initialIdRangeInRoot = 31 to 36, + rowCountPerDataEntry = 10) + // File 37 arrives as a separate (uncheckpointed) log commit carrying 20 rows. + commitBoth(baselineDeltaLog, amtDeltaLog, + Seq(fakeAdd(37, dataChange = true, numRecords = 20L))) + // Inline: DELETE 3 root-resident files (31..33) and REPLACE the other 4 (34..37, remove + + // re-add with a new DV), plus append net-new files so the tombstones overflow into a + // spilled leaf. + val actions = + Seq(31, 32, 33).map(id => removeOf(amtDeltaLog, id)) ++ + Seq(34, 35, 36, 37).flatMap(id => removeAndReAddWithDV(amtDeltaLog, Left(id))) ++ + (130 to 150).map(fakeAdd) + withInline { commitBoth(baselineDeltaLog, amtDeltaLog, actions) } + // The 7 tombstones spill into exactly one leaf, born ADDED (not DELETED) with no live + // entry; its manifest_info counts the 3 DELETED + 4 REPLACED mix. + val tombstoneLeaves = leafPointers(amtDeltaLog.update()).values.filter(l => + l.manifest_info.deleted_files_count + l.manifest_info.replaced_files_count > 0).toSeq + assert(tombstoneLeaves.size == 1, + s"the 7 tombstones must spill into exactly one leaf; got ${tombstoneLeaves.size}.") + val mi = tombstoneLeaves.head.manifest_info + assert(tombstoneLeaves.head.tracking.status == Tracking.Status.Added, + s"a freshly spilled tombstone leaf is born ADDED; got ${tombstoneLeaves.head.tracking}.") + assert(mi.deleted_files_count == 3 && mi.replaced_files_count == 4, + s"the tombstone leaf must count 3 DELETED + 4 REPLACED entries; got $mi.") + assert(mi.added_files_count == 0 && mi.existing_files_count == 0 && + mi.modified_files_count == 0, + s"the tombstone leaf holds no live entries; got $mi.") + // Row counts sum physical records, diverging from the file counts: the 3 DELETED tombstones + // are 10-row root files (30 rows); the 4 REPLACED tombstones are three 10-row root files + // plus the 20-row window file 37 (50 rows). + assert(mi.deleted_rows_count == 30L && mi.replaced_rows_count == 50L, + s"tombstone row counts must be 30 deleted + 50 replaced; got $mi.") + assert(mi.added_rows_count == 0L && mi.existing_rows_count == 0L && + mi.modified_rows_count == 0L, + s"the tombstone leaf holds no live rows; got $mi.") + assertLiveAddFilesEquals(baselineDeltaLog, amtDeltaLog) + } + } + } + + test("one inline write spills live leaves and a tombstone leaf, all born ADDED") { + withSQLConf(DeltaSQLConf.AMT_ENTRIES_PER_LEAF.key -> "10") { + withTables() { (baselineDeltaLog, amtDeltaLog) => + setup(baselineDeltaLog, amtDeltaLog, entriesPerLeaf = 10, + initialIdRangeInLeaf = 1 to 30, initialIdRangeInRoot = 31 to 37) + val actions = + (31 to 37).map(id => removeOf(amtDeltaLog, id)) ++ (130 to 150).map(fakeAdd) + withInline { commitBoth(baselineDeltaLog, amtDeltaLog, actions) } + // The single write spills freshly ADDED live leaves (from the net-new adds) and an ADDED + // tombstone leaf (from the overflowing DELETED tombstones); every newly written leaf is + // born ADDED, distinguished only by its manifest_info counts, and none is born DELETED. + val newLeaves = leafPointers(amtDeltaLog.update()).values + .filter(_.tracking.status == Tracking.Status.Added).toSeq + assert(newLeaves.exists(_.manifest_info.added_files_count > 0), + s"the net-new adds must spill into ADDED live leaves; got $newLeaves.") + assert(newLeaves.count(_.manifest_info.deleted_files_count > 0) == 1, + s"the overflowing tombstones must spill into one ADDED tombstone leaf; got $newLeaves.") + assert(!leafPointers(amtDeltaLog.update()).values.exists( + _.tracking.status == Tracking.Status.Deleted), + "no leaf is born DELETED.") + assertLiveAddFilesEquals(baselineDeltaLog, amtDeltaLog) + } + } + } + + test("an inline same-key re-add of an already-live file with dataChange=false is allowed") { + withSQLConf(DeltaSQLConf.AMT_ENTRIES_PER_LEAF.key -> "10") { + withTables() { (baselineDeltaLog, amtDeltaLog) => + setup(baselineDeltaLog, amtDeltaLog, entriesPerLeaf = 10, initialIdRangeInLeaf = 1 to 30) + val leafToAddFileMapping = leafToAddFileMap(amtDeltaLog) + assert(leafToAddFileMapping.size >= 2, + s"Need a tree-shaped bootstrap; got ${leafToAddFileMapping.size} leaves.") + val livePathsBefore = livePathsInLatestAMTCheckpoint(amtDeltaLog) + // The allowed counterpart to a rejected re-add, and the happy path of invariant (3): + // ComputeStats re-commits a currently-live leaf file with recomputed stats and + // dataChange=false. It stays EXISTING and the file is surfaced exactly once -- one + // EXISTING root entry, its old leaf slot MDV-masked. + val liveLeafFile = leafToAddFileMapping.toSeq.sortBy(_._1).head._2.head + assert(liveLeafFile.backReference.isDefined, "The re-added file must be leaf-resident.") + withInline { + amtDeltaLog.startTransaction().commit( + Seq(liveLeafFile.copy(dataChange = false)), + DeltaOperations.ComputeStats(predicate = Nil)) + } + assert(livePathsInLatestAMTCheckpoint(amtDeltaLog) == livePathsBefore, + "Re-adding an already-live file must not change the reconstructed live set.") + assert(rootDataEntryStatusToCount(amtDeltaLog) == Map(Tracking.Status.Existing -> 1L), + s"The re-added file must be a single EXISTING root entry; got " + + s"${rootDataEntryStatusToCount(amtDeltaLog)}.") + } + } + } + + /** + * A leaf-resident file F (back reference -> its leaf slot) AND a root-resident file R (no back + * reference) are each re-committed under their SAME (path, dv) key with dataChange=false in TWO + * commits -- first a deferred window commit, then an inline commit (metadata-only refreshes, e.g. + * ComputeStats). Both must stay live EXACTLY once as EXISTING root entries (F's old leaf slot + * MDV-masked), no matter what the inline re-add carries. + * + * `dropInlineBackReference` toggles how F's inline re-add is recognized as an already-live key: + * - false: it carries F's leaf back reference (recognized via the back reference); + * - true: it carries NO back reference (recognized via the pre-commit live set, since the + * window commit already put F's key there -- the branch that would silently regress if + * preCommitLiveKeys were dropped). + * R is root-resident (no leaf slot), so it is always recognized via the pre-commit live set -- + * every variant exercises that path for the root file. + */ + private def assertWindowThenInlineSameKeyReAddStaysExisting( + dropInlineBackReference: Boolean): Unit = { + withSQLConf(DeltaSQLConf.AMT_ENTRIES_PER_LEAF.key -> "10") { + withTables() { (baselineDeltaLog, amtDeltaLog) => + setup(baselineDeltaLog, amtDeltaLog, entriesPerLeaf = 10, initialIdRangeInLeaf = 1 to 30) + // Append one net-new file that stays root-resident (no leaf slot -> no back reference). + val rootId = 31 + withInline { commitBoth(baselineDeltaLog, amtDeltaLog, Seq(fakeAdd(rootId))) } + + val leafToAddFileMapping = leafToAddFileMap(amtDeltaLog) + assert(leafToAddFileMapping.size >= 2, + s"Need a tree-shaped bootstrap; got ${leafToAddFileMapping.size} leaves.") + val leafFile = leafToAddFileMapping.toSeq.sortBy(_._1).head._2.head + assert(leafFile.backReference.isDefined, "F must be leaf-resident (has a back reference).") + val rootPath = fakeAdd(rootId).path + val rootFile = liveAddFilesInLatestSnapshot(amtDeltaLog).find(_.path == rootPath) + .getOrElse(fail(s"root file $rootId is not live.")) + assert(rootFile.backReference.isEmpty, "R must be root-resident (no back reference).") + val livePathsBefore = livePathsInLatestAMTCheckpoint(amtDeltaLog) + + // Deferred window commit: re-add both F and R (same key, dataChange=false). F carries its + // leaf back reference; R carries none. + commitBoth( + baselineDeltaLog, amtDeltaLog, + Seq(leafFile.copy(dataChange = false), rootFile.copy(dataChange = false)), + operation = DeltaOperations.ComputeStats(predicate = Nil)) + + // Inline commit: re-add both again. The variant decides whether F's inline action carries + // its back reference; R never has one. F becomes a root EXISTING entry with its old leaf + // slot masked; R stays a root EXISTING entry -- two EXISTING root entries, one masked bit. + val inlineLeafReAdd = leafFile.copy( + dataChange = false, + backReference = if (dropInlineBackReference) None else leafFile.backReference) + createIncrementalAMTAndValidate( + baselineDeltaLog, amtDeltaLog, + createIncrementalAMTWriteMetrics( + numOldLeavesUpdated = 1, + numOldLeavesUntouched = leafToAddFileMapping.size - 1, + numRootEntriesExistingStatus = 2, + numLeafMdvBitsAdded = 1, + numLeavesModifiedStatus = 1, + numLeavesExistingStatus = leafToAddFileMapping.size - 1), + inlineAMTCommitActions = + Some(Seq(inlineLeafReAdd, rootFile.copy(dataChange = false))), + inlineOperation = DeltaOperations.ComputeStats(predicate = Nil)) + assert(livePathsInLatestAMTCheckpoint(amtDeltaLog) == livePathsBefore, + "Re-adding already-live files across a window + inline commit must not change the " + + "reconstructed live set.") + assert(rootDataEntryStatusToCount(amtDeltaLog) == Map(Tracking.Status.Existing -> 2L), + s"F and R must be two EXISTING root entries; got " + + s"${rootDataEntryStatusToCount(amtDeltaLog)}.") + } + } + } + + test("a window then inline same-key re-add (dataChange=false) of a leaf and a root file, " + + "the inline actions carrying back references, keeps both EXISTING") { + assertWindowThenInlineSameKeyReAddStaysExisting(dropInlineBackReference = false) + } + + test("a window then inline same-key re-add (dataChange=false) where the inline leaf " + + "action carries no back reference still keeps both EXISTING (recognized via the live set)") { + assertWindowThenInlineSameKeyReAddStaysExisting(dropInlineBackReference = true) + } + + test("a restore round-trip re-adds removed leaf and root files and drops the interim " + + "add, returning the reconstructed live set to the prior committed state") { + withSQLConf(DeltaSQLConf.AMT_ENTRIES_PER_LEAF.key -> "10") { + withTables() { (baselineDeltaLog, amtDeltaLog) => + // Commit-A / tree: the bootstrap adds files into a mixed committed state A that holds both + // leaf-resident files and root-resident files (it fills the root exactly, so adding more + // would spill those root entries into leaves -- keep it as the state we restore back to). + val rootIds = 31 to 37 + setup(baselineDeltaLog, amtDeltaLog, entriesPerLeaf = 10, + initialIdRangeInLeaf = 1 to 30, initialIdRangeInRoot = rootIds) + assert(rootIds.nonEmpty, "need at least one root-resident file.") + val stateA = livePathsInLatestAMTCheckpoint(amtDeltaLog) + + // Pick a leaf-resident (back-referenced) and a root-resident (no back reference) victim. + val leafVictim = leafToAddFileMap(amtDeltaLog).toSeq.sortBy(_._1).head._2.head + assert(leafVictim.backReference.isDefined, "leaf victim must be leaf-resident.") + val rootPaths = rootIds.map(id => fakeAdd(id).path).toSet + val rootVictim = liveAddFilesInLatestSnapshot(amtDeltaLog) + .find(a => a.backReference.isEmpty && rootPaths.contains(a.path)) + .getOrElse(fail("need a live root-resident file to remove.")) + val newId = 330 + + // Commit: remove both victims from the tree and add a new file; checkpoint -> state B, so + // the victims are gone (leaf slot masked, root tombstoned) and the new file is live. + commitBoth(baselineDeltaLog, amtDeltaLog, + Seq(leafVictim.remove, rootVictim.remove, fakeAdd(newId))) + commitCheckpoint(amtDeltaLog, incremental = true) + val stateB = livePathsInLatestAMTCheckpoint(amtDeltaLog) + assert(!stateB.contains(leafVictim.path) && !stateB.contains(rootVictim.path), + "both victims must be gone in state B.") + assert(stateB.contains(fakeAdd(newId).path), "the new file must be live in state B.") + + // Commit_Inline (restore): re-add both victims as fresh adds -- their tree slots are gone, + // so they carry no back reference and land as new root entries -- and remove the interim + // new file, restoring the reconstructed live set to state A. + withInline { + commitBoth(baselineDeltaLog, amtDeltaLog, Seq( + leafVictim.copy(backReference = None, dataChange = true), + rootVictim.copy(backReference = None, dataChange = true), + removeOf(amtDeltaLog, newId))) + } + assert(livePathsInLatestAMTCheckpoint(amtDeltaLog) == stateA, + s"the restore must return the reconstructed live set to state A; " + + s"got ${livePathsInLatestAMTCheckpoint(amtDeltaLog).diff(stateA)} extra / " + + s"${stateA.diff(livePathsInLatestAMTCheckpoint(amtDeltaLog))} missing.") + assertLiveAddFilesEquals(baselineDeltaLog, amtDeltaLog) + } + } + } + + /** + * A mixed old root deleted across every file-residence at once, folded into one incremental AMT + * on both the deferred and the inline route (the gridTest parameter). Steps: + * - Full-rewrite a tree of 2 leaves holding 20 [[DataEntry]]s; the root holds no DataEntry. + * - Add 8 more files and take a deferred manifest commit: they stay root-resident, so the root + * is now MIXED -- 8 live root DataEntries alongside the 2 leaf pointers. + * - IC-1: add 10 files. + * - IC-2: remove one IC-1 file -- cancels against its own add in replay. + * - IC-3: remove across three sources -- an old-root, two leaf and two IC-1 files. + * - IC-4: the same three-source remove, plus 10 new adds in the one commit. + * - IC-5: add 4 files and remove two more old-root files. + * - Final write: remove one file from each source (IC-4, IC-1, old-root, a leaf) plus 4 new + * adds, folded either as a deferred manifest commit (empty actionsToCommit) or inline (the + * final commit carries the actions -- the only route that writes tombstones). + * - Assert the folded tree reconstructs the expected live set and the per-route metrics. + * + * Residence decides how a remove resolves: a leaf-resident file is masked by an MDV bit on its + * carried pointer (no leaf rewrite); a root-resident file is dropped by replay (and only inline + * leaves a tracking=removed tombstone); an intermediate add's remove cancels against its own add. + */ + gridTest("a mixed old root plus multi-source deletes across intermediate commits" + + " folds into one incremental AMT")(Seq(false, true)) { inlineFinalWrite => + withSQLConf(DeltaSQLConf.AMT_ENTRIES_PER_LEAF.key -> "10") { + withTables() { (baselineDeltaLog, amtDeltaLog) => + // ---- Old root: 20 files packed into leaves, then 8 more added root-resident. ---- + // The full rewrite puts all 20 in leaves; the incremental that follows keeps its 8 net-new + // adds in the root (2 carried pointers + 8 adds == the cap of 10, so nothing spills). That + // leaves the old root genuinely mixed, which a full checkpoint alone can never produce. + setup(baselineDeltaLog, amtDeltaLog, entriesPerLeaf = 10, initialIdRangeInLeaf = 1 to 20) + val oldLeaves = leafToAddFileMap(amtDeltaLog) + + val rootIds = 101 to 108 + commitBoth(baselineDeltaLog, amtDeltaLog, rootIds.map(fakeAdd)) + createIncrementalAMTAndValidate( + baselineDeltaLog, + amtDeltaLog, + createIncrementalAMTWriteMetrics( + numOldLeavesUntouched = oldLeaves.size, + numRootEntriesExistingStatus = rootIds.size, + numLeavesExistingStatus = oldLeaves.size)) + val (rootLiveAddsBefore, _) = liveAddsAndTombstonesCountInRoot(amtDeltaLog) + assert(rootLiveAddsBefore == rootIds.size, + s"The old root must hold ${rootIds.size} live adds of its own; got $rootLiveAddsBefore.") + + // Leaf victims are picked against the ACTUAL hash-based assignment rather than assumed, so + // the removes carry the back references the writer really stamped. IC-3 and IC-4 take two + // each and the final write takes one more, so five are needed; taking three per leaf keeps + // that satisfied without depending on how the rewrite distributed the files. + val leafVictims = oldLeaves.toSeq.sortBy(_._1).flatMap(_._2.take(3)) + assert(leafVictims.size >= 5, s"Need >= 5 leaf victims; got ${leafVictims.size}.") + + // ---- IC-1: add 10 files. Root-resident (they do not spill until the fold). ---- + val ic1Ids = 201 to 210 + commitBoth(baselineDeltaLog, amtDeltaLog, ic1Ids.map(fakeAdd)) + + // ---- IC-2: delete one of IC-1's files -> cancels against its own add in replay. ---- + commitBoth(baselineDeltaLog, amtDeltaLog, Seq(removeOf(amtDeltaLog, ic1Ids.head))) + + // ---- IC-3: delete from THREE sources at once: old root, leaves, IC-1. ---- + commitBoth( + baselineDeltaLog, amtDeltaLog, + Seq(removeOf(amtDeltaLog, rootIds.head)) ++ + leafVictims.take(2).map(_.remove) ++ + ic1Ids.slice(1, 3).map(removeOf(amtDeltaLog, _))) + + // ---- IC-4: the same three-source delete, plus 10 more adds in the same commit. ---- + val ic4Ids = 301 to 310 + commitBoth( + baselineDeltaLog, amtDeltaLog, + Seq(removeOf(amtDeltaLog, rootIds(1))) ++ + leafVictims.slice(2, 4).map(_.remove) ++ + ic1Ids.slice(3, 5).map(removeOf(amtDeltaLog, _)) ++ + ic4Ids.map(fakeAdd)) + + // ---- IC-5: add 4, and delete 2 more of the old root's files. ---- + val ic5Ids = 401 to 404 + commitBoth( + baselineDeltaLog, amtDeltaLog, + ic5Ids.map(fakeAdd) ++ rootIds.slice(2, 4).map(removeOf(amtDeltaLog, _))) + + // ---- The final write: one file from each source removed, plus 4 new adds. ---- + val finalIds = 501 to 504 + val finalActions = + Seq( + removeOf(amtDeltaLog, ic4Ids.head), + removeOf(amtDeltaLog, ic1Ids(5)), + removeOf(amtDeltaLog, rootIds(4))) ++ + Seq(leafVictims(4).remove) ++ + finalIds.map(fakeAdd) + + // Live net-new adds surviving into the new root, by origin: + // old root 8 - 1 (IC-3) - 1 (IC-4) - 2 (IC-5) - 1 (final) = 3 + // IC-1 10 - 1 (IC-2) - 2 (IC-3) - 2 (IC-4) - 1 (final) = 4 + // IC-4 10 - 1 (final) = 9 + // IC-5 4 = 4 + // final 4 = 4 + val expectedLiveAdds = 3 + 4 + 9 + 4 + 4 + // Five leaf-resident removes across IC-3, IC-4 and the final write, each contributing one + // MDV bit. Which leaves those five land on follows the rewrite's hash placement, so derive + // the updated/untouched split from the victims actually chosen rather than assuming every + // leaf is hit. + val expectedMdvBits = 5 + val victimLeaves = leafVictims.take(expectedMdvBits) + .flatMap(_.backReference.map(_.manifest)).toSet + val leavesUpdated = victimLeaves.size + val leavesUntouched = oldLeaves.size - leavesUpdated + // Only the inline route sources CDF from actionsToCommit, so only it writes tombstones -- + // one per no-backref remove in the final commit (the IC-4, IC-1 and old-root files; the + // leaf one is masked instead). + val expectedTombstones = if (inlineFinalWrite) 3 else 0 + // spillIfNeeded moves whole cap-sized batches of live adds out until the root fits: + // while (fixedRootCount + spilled + remaining > cap && remaining.nonEmpty) + // with fixedRootCount = carried pointers + tombstones. + val fixedRootCount = oldLeaves.size + expectedTombstones + var spilled = 0 + var remaining = expectedLiveAdds + while (fixedRootCount + spilled + remaining > 10 && remaining > 0) { + remaining = math.max(0, remaining - 10) + spilled += 1 + } + // Under this branch's enriched metrics the writer also reports the new tree's leaf-pointer + // status mix (untouched -> EXISTING, MDV-grown -> MODIFIED, freshly spilled -> ADDED) and, + // on the inline route only, the per-commit deleted CDF bit for the one leaf-resident remove + // in the final commit (leafVictims(4)); the deferred route sources no CDF from its fold. + val expectedLeafDeleteCDFBits = if (inlineFinalWrite) 1 else 0 + // Of the live adds that remain in the root after spilling, the deferred fold proposes no + // insert (empty actionsToCommit), so every remaining root live add is EXISTING; the inline + // route spills them all (remaining == 0). + val rootExistingLiveAdds = if (inlineFinalWrite) 0 else remaining + val expectedMetrics = createIncrementalAMTWriteMetrics( + numOldLeavesUpdated = leavesUpdated, + numOldLeavesUntouched = leavesUntouched, + numNewLeaves = spilled, + numRootEntriesAddedStatus = remaining - rootExistingLiveAdds, + numRootEntriesExistingStatus = rootExistingLiveAdds, + numRootEntriesDeletedStatus = expectedTombstones, + numLeafMdvBitsAdded = expectedMdvBits, + numLeafDeleteCDFBitsAdded = expectedLeafDeleteCDFBits, + numLeavesAddedStatus = spilled, + numLeavesExistingStatus = leavesUntouched, + numLeavesModifiedStatus = leavesUpdated) + + if (inlineFinalWrite) { + // The inline route commits the actions and the tree in ONE commit, so the metrics come + // off that commit rather than a follow-up OPTIMIZE CHECKPOINT. + createIncrementalAMTAndValidate( + baselineDeltaLog, amtDeltaLog, expectedMetrics, + inlineAMTCommitActions = Some(finalActions)) + } else { + // The deferred route commits the actions as one more intermediate commit, then folds + // everything in with an empty actionsToCommit. + commitBoth(baselineDeltaLog, amtDeltaLog, finalActions) + createIncrementalAMTAndValidate( + baselineDeltaLog, amtDeltaLog, expectedMetrics, + // IC-1..IC-5, the final write, and the bootstrap incremental's own commit. + expectedNumIntermediateCommits = Some(7)) + } + + // The differential oracle above already pins the live set; state the total explicitly too, + // since the whole point of the scenario is that 39 files survive this churn. + val expectedLiveTotal = expectedLiveAdds + (20 - expectedMdvBits) + assert(livePathsInLatestAMTCheckpoint(amtDeltaLog).size == expectedLiveTotal, + s"The tree must reconstruct exactly $expectedLiveTotal live files; got " + + s"${livePathsInLatestAMTCheckpoint(amtDeltaLog).size}.") + } + } + } +} diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTIncrementalWriteInvariantSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTIncrementalWriteInvariantSuite.scala new file mode 100644 index 00000000000..aca46e373ba --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTIncrementalWriteInvariantSuite.scala @@ -0,0 +1,223 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.amt + +import org.apache.spark.sql.delta.DeltaOperations +import org.apache.spark.sql.delta.actions.{Action, AddFile, Metadata, Protocol} +import org.apache.spark.sql.delta.sources.DeltaSQLConf +import org.apache.spark.sql.delta.util.FileNames + +/** Invariant enforcement: rejecting commits that break a write- or commit-shape invariant. */ +class AMTIncrementalWriteInvariantSuite extends AMTIncrementalWriteTestBase { + + private def processedActions( + oldRootAdds: Seq[AddFile], + actionsToCommit: Seq[Action]): ProcessedActions = + new ProcessedActions( + oldAMTVersion = 0L, + oldRootAdds = oldRootAdds, + nonContentFromOldCheckpoint = Seq[Action](Protocol(), Metadata()), + windowCommits = Nil, + windowCommitActions = Nil, + attemptVersion = 1L, + actionsToCommit = actionsToCommit, + tableRoot = new org.apache.hadoop.fs.Path("s3://bucket/prefix/amt_test_table"), + useDeletionVectorObjectIdentity = true) + + test("writeIncremental rejects intermediate commits with a hole up to attemptVersion") { + withSQLConf(leafPackingConfs: _*) { + withTables() { (baselineDeltaLog, amtDeltaLog) => + commitBoth(baselineDeltaLog, amtDeltaLog, Seq(fakeAdd(1), fakeAdd(2))) + // A base AMT to build intermediate commits on. + commitCheckpoint(amtDeltaLog, incremental = false) + val snapshot = amtDeltaLog.update() + val provider = amtProvider(snapshot).getOrElse(fail("expected AMTCheckpointProvider")) + val oldAMTVersion = provider.checkpointAction.contentRoot.version + val intermediateLogCommits = snapshot.logSegment.deltas + .filter(f => FileNames.getFileVersion(f) > oldAMTVersion) + // They only reach snapshot.version, so [oldAMTVersion+1, snapshot.version+5) has a + // hole -> the Step-0 coverage assert must fire. + intercept[AssertionError] { + new IncrementalAMTWriter(spark, amtDeltaLog).writeIncremental( + oldAMTActionsProvider = new BaseAMTCheckpointActionsProvider(amtDeltaLog, provider), + intermediateLogCommits = intermediateLogCommits, + attemptVersion = snapshot.version + 5, + actionsToCommit = Seq.empty, + trigger = AMTTriggerMode.CheckpointIntervalIncremental.name) + } + } + } + } + + test("an inline no-backref remove with no originating AddFile is rejected") { + withSQLConf(leafPackingConfs: _*) { + withTables() { (baselineDeltaLog, amtDeltaLog) => + commitBoth(baselineDeltaLog, amtDeltaLog, (1 to leafPackedFiles).map(fakeAdd)) + commitCheckpoint(amtDeltaLog, incremental = false) + // A no-backref remove for a path that was never added has no originating AddFile in + // root + window, so buildRootRemoveEntries cannot build a CDF entry, and rejects it. + val ex = intercept[IllegalStateException] { + withInline { + amtDeltaLog.startTransaction().commit(Seq(fakeAdd(999999).remove), writeOperation) + } + } + assert(ex.getMessage.contains("No originating AddFile"), + s"expected the missing-origin invariant; got: ${ex.getMessage}") + } + } + } + + test("an inline same-key re-add of an already-live file with dataChange=true is rejected") { + withSQLConf(leafPackingConfs: _*) { + withTables() { (baselineDeltaLog, amtDeltaLog) => + commitBoth(baselineDeltaLog, amtDeltaLog, (1 to leafPackedFiles).map(fakeAdd)) + commitCheckpoint(amtDeltaLog, incremental = false) + val leafToAddFileMapping = leafToAddFileMap(amtDeltaLog) + assert(leafToAddFileMapping.size >= 2, + s"Need a tree-shaped bootstrap; got ${leafToAddFileMapping.size} leaves.") + // Re-commit a currently-live leaf file under its SAME (path, dv) key -- carrying the leaf + // back reference it was reconstructed with -- in one inline commit with dataChange = true. + // A same-key re-add of an already-live file is a metadata-only refresh + // (dataChange = false); a data-changing re-add is rejected by the incremental writer. + val liveLeafFile = leafToAddFileMapping.toSeq.sortBy(_._1).head._2.head + assert(liveLeafFile.backReference.isDefined, "The re-added file must be leaf-resident.") + val ex = withSQLConf( + DeltaSQLConf.DELTA_COMMIT_VALIDATE_EXPECTED_DATA_CHANGE_MODE.key -> + DeltaSQLConf.DataChangeValidationMode.OFF.toString) { + intercept[IllegalStateException] { + withInline { + amtDeltaLog.startTransaction().commit( + Seq(liveLeafFile.copy(dataChange = true)), + DeltaOperations.ComputeStats(predicate = Nil)) + } + } + } + assert(ex.getMessage.contains("dataChange=true is not allowed"), + s"expected the data-changing re-add rejection; got: ${ex.getMessage}") + } + } + } + + test("(1) a dataChange=true commit rejects a same-key re-add of an already-live file") { + val ex = intercept[IllegalStateException] { + processedActions( + oldRootAdds = Seq(fakeAdd(1)), + actionsToCommit = Seq(fakeAdd(1, dataChange = true))) + } + assert(ex.getMessage.contains("dataChange=true is not allowed"), + s"expected invariant (1); got: ${ex.getMessage}") + } + + test("(2) a dataChange=false commit with removes rejects a re-add of an already-live file") { + val ex = intercept[IllegalStateException] { + processedActions( + oldRootAdds = Seq(fakeAdd(1), fakeAdd(2)), + actionsToCommit = Seq( + fakeAdd(1).remove.copy(dataChange = false), + fakeAdd(2, dataChange = false))) + } + assert(ex.getMessage.contains("must not re-add an already-live file"), + s"expected invariant (2); got: ${ex.getMessage}") + } + + test("(3) a dataChange=false commit with no removes rejects a new (non-re-add) file") { + val ex = intercept[IllegalStateException] { + processedActions( + oldRootAdds = Seq(fakeAdd(1)), + actionsToCommit = Seq(fakeAdd(99, dataChange = false))) + } + assert(ex.getMessage.contains("must re-add only already-live files"), + s"expected invariant (3); got: ${ex.getMessage}") + } + + test("the three legal commit shapes construct and classify their adds correctly") { + // (1)-legal: a data-changing append of a genuinely new file -> not a re-add. + val append = processedActions( + oldRootAdds = Seq(fakeAdd(1)), + actionsToCommit = Seq(fakeAdd(2, dataChange = true))) + assert(append.reCommittedLiveAdd.isEmpty) + // (2)-legal: a metadata-only compaction removing a live file and adding a fresh one -> not a + // re-add of an already-live file. + val compaction = processedActions( + oldRootAdds = Seq(fakeAdd(1)), + actionsToCommit = Seq( + fakeAdd(1).remove.copy(dataChange = false), + fakeAdd(2, dataChange = false))) + assert(compaction.reCommittedLiveAdd.isEmpty) + // (3)-legal: a metadata-only stats refresh re-adding an already-live file under the same key. + val refresh = processedActions( + oldRootAdds = Seq(fakeAdd(1)), + actionsToCommit = Seq(fakeAdd(1, dataChange = false))) + assert(refresh.reCommittedLiveAdd.isDefined) + } + + /** A carried leaf pointer whose manifest_info carries the given per-status file counts. */ + private def carriedLeafPointer( + addedFiles: Int = 0, + existingFiles: Int = 0, + deletedFiles: Int = 0, + replacedFiles: Int = 0, + modifiedFiles: Int = 0): DataManifestEntry = + DataManifestEntry( + location = "metadata/leaf-guard.parquet", + file_format = AMTSingleAction.FileFormatParquet, + tracking = Tracking( + status = Tracking.Status.Existing, + snapshot_id = None, dv_snapshot_id = None, sequence_number = None, + file_sequence_number = None, first_row_id = None, + deleted_positions = None, replaced_positions = None), + record_count = + (addedFiles + existingFiles + deletedFiles + replacedFiles + modifiedFiles).toLong, + file_size_in_bytes = 100L, + manifest_info = ManifestInfo( + added_files_count = addedFiles, existing_files_count = existingFiles, + deleted_files_count = deletedFiles, replaced_files_count = replacedFiles, + modified_files_count = modifiedFiles, + added_rows_count = 0L, existing_rows_count = 0L, + deleted_rows_count = 0L, replaced_rows_count = 0L, modified_rows_count = 0L, + min_sequence_number = 0L, dv = None, dv_cardinality = None)) + + test("carryForwardOneLeaf rejects a carried leaf mixing live files and tombstones") { + withTables() { (_, amtDeltaLog) => + val writer = new IncrementalAMTWriter(spark, amtDeltaLog) + // A carried leaf whose manifest_info counts both a live file and a tombstone is a corrupt + // shape the writer does not support. + val mixed = carriedLeafPointer(existingFiles = 2, deletedFiles = 1) + val ex = intercept[IllegalStateException] { + writer.carryForwardOneLeaf(mixed, newMdvPositions = Seq.empty, + deletedPositions = Seq.empty, replacedPositions = Seq.empty) + } + assert(ex.getMessage.contains("both live files and tombstones"), + s"expected the live+tombstone-mix invariant; got: ${ex.getMessage}") + } + } + + test("carryForwardOneLeaf rejects new MDV positions on a leaf with no live file") { + withTables() { (_, amtDeltaLog) => + val writer = new IncrementalAMTWriter(spark, amtDeltaLog) + // A tombstone-only carried leaf holds no live entry, so it cannot gain new MDV positions -- + // there is nothing left to mask. + val tombstoneOnly = carriedLeafPointer(deletedFiles = 3) + val ex = intercept[IllegalStateException] { + writer.carryForwardOneLeaf(tombstoneOnly, newMdvPositions = Seq(0), + deletedPositions = Seq.empty, replacedPositions = Seq.empty) + } + assert(ex.getMessage.contains("no live file but gained new MDV positions"), + s"expected the dead-leaf-MDV invariant; got: ${ex.getMessage}") + } + } +} diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTIncrementalWriteSpillSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTIncrementalWriteSpillSuite.scala new file mode 100644 index 00000000000..3091cb2ec42 --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTIncrementalWriteSpillSuite.scala @@ -0,0 +1,293 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.amt + +import org.apache.spark.sql.delta.sources.DeltaSQLConf + +/** Root-vs-leaf spill decisions and leaf packing on the incremental AMT write path. */ +class AMTIncrementalWriteSpillSuite extends AMTIncrementalWriteTestBase { + + test("deferred append below the cap stays root-resident, no spill, leaf untouched") { + withSQLConf(DeltaSQLConf.AMT_ENTRIES_PER_LEAF.key -> "8") { + withTables() { (baselineDeltaLog, amtDeltaLog) => + setup(baselineDeltaLog, amtDeltaLog, entriesPerLeaf = 8, initialIdRangeInLeaf = 1 to 24) + // One net-new add stays root-resident: the root holds 3 leaf pointers plus this add = 4, + // which is under the cap of 8, so spillIfNeeded does not spill. + commitBoth(baselineDeltaLog, amtDeltaLog, Seq(fakeAdd(25))) + createIncrementalAMTAndValidate( + baselineDeltaLog, + amtDeltaLog, + createIncrementalAMTWriteMetrics( + numOldLeavesUntouched = 3, + numRootEntriesExistingStatus = 1, + numLeavesExistingStatus = 3)) + } + } + } + + test("deferred append exactly filling the root to the cap does not spill") { + withSQLConf(DeltaSQLConf.AMT_ENTRIES_PER_LEAF.key -> "8") { + withTables() { (baselineDeltaLog, amtDeltaLog) => + setup(baselineDeltaLog, amtDeltaLog, entriesPerLeaf = 8, initialIdRangeInLeaf = 1 to 24) + // Fill the root exactly to the cap: 3 leaf pointers + 5 adds = 8 == cap. spillIfNeeded + // loops while the total is `> cap`, so filling it exactly must not spill. + commitBoth(baselineDeltaLog, amtDeltaLog, (25 to 29).map(fakeAdd)) + createIncrementalAMTAndValidate( + baselineDeltaLog, + amtDeltaLog, + createIncrementalAMTWriteMetrics( + numOldLeavesUntouched = 3, + numRootEntriesExistingStatus = 5, + numLeavesExistingStatus = 3)) + } + } + } + + test("deferred append over the cap spills whole cap-sized batches") { + withSQLConf(DeltaSQLConf.AMT_ENTRIES_PER_LEAF.key -> "8") { + withTables() { (baselineDeltaLog, amtDeltaLog) => + setup(baselineDeltaLog, amtDeltaLog, entriesPerLeaf = 8, initialIdRangeInLeaf = 1 to 24) + // 12 net-new adds on a root already holding 3 leaf pointers, cap 8. spillIfNeeded trace: + // 3(fixed) + 0(spilled) + 12(remaining) = 15 > 8 -> spill a batch of 8 (remaining = 4) + // 3(fixed) + 1(spilled) + 4(remaining) = 8 == 8 -> stop. + // => 1 new leaf, and the 4 leftover adds stay root-resident. + val appendVersion = amtDeltaLog.update().version + 1 + commitBoth(baselineDeltaLog, amtDeltaLog, (25 to 36).map(fakeAdd)) + createIncrementalAMTAndValidate( + baselineDeltaLog, + amtDeltaLog, + createIncrementalAMTWriteMetrics( + numOldLeavesUntouched = 3, + numNewLeaves = 1, + numRootEntriesExistingStatus = 4, + numLeavesExistingStatus = 3, + numLeavesAddedStatus = 1)) + val newLeaves = amtProvider(amtDeltaLog.update()).getOrElse(fail("expected AMT")) + .leaves.filter(_.tracking.status == Tracking.Status.Added) + assert(newLeaves.size == 1, s"Expected one newly spilled leaf; found ${newLeaves.size}.") + assert(newLeaves.head.manifest_info.min_sequence_number == appendVersion, + s"The new leaf must use append sequence $appendVersion; got ${newLeaves.head}.") + } + } + } + + test("deferred append with entriesPerLeaf=1 spills every net-new add into its own leaf") { + withTables() { (baselineDeltaLog, amtDeltaLog) => + // Bootstrap at a cap that clusters into several leaves (a single manifest would be promoted + // into the root), then drop the cap to one so every net-new add spills into its own leaf. + withSQLConf(DeltaSQLConf.AMT_ENTRIES_PER_LEAF.key -> "8") { + setup(baselineDeltaLog, amtDeltaLog, entriesPerLeaf = 8, initialIdRangeInLeaf = 1 to 24) + } + withSQLConf(DeltaSQLConf.AMT_ENTRIES_PER_LEAF.key -> "1") { + // At a cap of one, each of the 3 adds spills into its own leaf and none stays in the root. + commitBoth(baselineDeltaLog, amtDeltaLog, Seq(fakeAdd(25), fakeAdd(26), fakeAdd(27))) + createIncrementalAMTAndValidate( + baselineDeltaLog, + amtDeltaLog, + createIncrementalAMTWriteMetrics( + numOldLeavesUntouched = 3, + numNewLeaves = 3, + numRootEntriesAddedStatus = 0, + numLeavesExistingStatus = 3, + numLeavesAddedStatus = 3)) + } + } + } + + test("a large deferred append spills multiple cap-sized leaves, each within the cap") { + withSQLConf(DeltaSQLConf.AMT_ENTRIES_PER_LEAF.key -> "8") { + withTables() { (baselineDeltaLog, amtDeltaLog) => + val oldExistingLeaves = + setup(baselineDeltaLog, amtDeltaLog, entriesPerLeaf = 8, initialIdRangeInLeaf = 1 to 24) + .map(_.location).toSet + // 30 net-new adds on a root already holding 3 leaf pointers, cap 8. spillIfNeeded trace: + // 3 + 0 + 30 = 33 > 8 -> spill 8 (remaining 22) + // 3 + 1 + 22 = 26 > 8 -> spill 8 (remaining 14) + // 3 + 2 + 14 = 19 > 8 -> spill 8 (remaining 6) + // 3 + 3 + 6 = 12 > 8 -> spill 6 (remaining 0) + // => 4 new leaves, 0 root-resident adds. + val appendVersion = amtDeltaLog.update().version + 1 + commitBoth(baselineDeltaLog, amtDeltaLog, (25 to 54).map(fakeAdd)) + createIncrementalAMTAndValidate( + baselineDeltaLog, + amtDeltaLog, + createIncrementalAMTWriteMetrics( + numOldLeavesUntouched = 3, + numNewLeaves = 4, + numRootEntriesAddedStatus = 0, + numLeavesExistingStatus = 3, + numLeavesAddedStatus = 4)) + // Every leaf this write SPILLED holds at most `cap` physical DATA entries, because + // spillIfNeeded moves whole cap-sized batches. The bootstrap's own leaves are excluded: a + // clustered full rewrite derives a leaf count from the cap but does not bound each leaf, so + // an uneven hash distribution can leave one holding more. + val provider = amtProvider(amtDeltaLog.update()).getOrElse(fail("expected AMT")) + val spilledLeaves = provider.leaves.filterNot(l => oldExistingLeaves.contains(l.location)) + assert(spilledLeaves.size == 4, + s"Expected 4 spilled leaves; found ${spilledLeaves.size}.") + spilledLeaves.foreach { leaf => + val entries = + trackingStatusToAddFileCountMap(leaf.getAbsolutePath(provider.tableRoot).toString) + .values.sum + assert(entries <= 8, + s"Spilled leaf ${leaf.location} holds $entries entries, over the cap of 8.") + assert(leaf.manifest_info.min_sequence_number == appendVersion, + s"Spilled leaf ${leaf.location} must use append sequence $appendVersion; got " + + s"${leaf.manifest_info.min_sequence_number}.") + } + } + } + } + + test("spill accounting includes carried leaf pointers in fixedRootCount") { + withSQLConf(DeltaSQLConf.AMT_ENTRIES_PER_LEAF.key -> "2") { + withTables() { (baselineDeltaLog, amtDeltaLog) => + // Seed enough files that the full rewrite writes MORE THAN ONE leaf; the exact count is + // non-deterministic (the clustered rewrite hashes files across partitions), so read it back + // rather than assume it. spillIfNeeded must then base its overflow on the carried count + // plus the new adds. + commitBoth(baselineDeltaLog, amtDeltaLog, (1 to 6).map(fakeAdd)) + commitCheckpoint(amtDeltaLog, incremental = false) + val numLeafCountBefore = + amtProvider(amtDeltaLog.update()).getOrElse(fail("expected AMT")).leaves.size + assert(numLeafCountBefore >= 2, + s"Need >= 2 carried leaves to exercise fixedRootCount; got $numLeafCountBefore.") + // Append one file. All carried pointers stay untouched. The carried leaf pointers already + // fill the root to the cap (fixedRootCount counts them, and numLeafCountBefore >= cap = 2), + // so the appended file has no root capacity and must spill into exactly one new leaf -- + // which is the accounting this test guards. + val oldAMTLeafToMDV = leafToLeafMDVCardinalityMap(amtDeltaLog) + commitBoth(baselineDeltaLog, amtDeltaLog, Seq(fakeAdd(7))) + val actualIncrementalMetrics = commitIncrementalCheckpointAndReturnMetrics(amtDeltaLog) + .getOrElse(fail("An incremental checkpoint must log metrics.")) + assert(actualIncrementalMetrics.numOldLeavesUntouched == numLeafCountBefore, + s"All $numLeafCountBefore carried leaves must be untouched by an append; got " + + s"${actualIncrementalMetrics.numOldLeavesUntouched}.") + assert(actualIncrementalMetrics.numNewLeaves == 1, + s"The appended file must spill into one new leaf; got " + + s"${actualIncrementalMetrics.numNewLeaves}.") + assert(actualIncrementalMetrics.numRootEntriesAddedStatus == 0 && + actualIncrementalMetrics.numRootEntriesExistingStatus == 0 && + actualIncrementalMetrics.numRootEntriesModifiedStatus == 0, + "The appended file must not land in the root (the carried pointers fill it).") + // The old tree is a fresh full rewrite, so it carries no DELETED tombstones to drop. + assertMetricsMatchTreeDelta(amtDeltaLog, oldAMTLeafToMDV, oldDeletedLeafCount = 0, + metrics = actualIncrementalMetrics) + assertLiveAddFilesEquals(baselineDeltaLog, amtDeltaLog) + } + } + } + + test("consecutive intermediate insert commits accumulate their adds in the root") { + withSQLConf(DeltaSQLConf.AMT_ENTRIES_PER_LEAF.key -> "8") { + withTables() { (baselineDeltaLog, amtDeltaLog) => + setup(baselineDeltaLog, amtDeltaLog, entriesPerLeaf = 8, initialIdRangeInLeaf = 1 to 24) + // Several separate INSERT commits, none checkpointed, each adding one net-new file: the + // adding analog of accumulating deferred leaf removes. The deferred incremental must fold + // every one into the root -- 3 leaf pointers + 4 adds = 7 <= cap 8, so all four stay + // root-resident and nothing spills. + commitBoth(baselineDeltaLog, amtDeltaLog, Seq(fakeAdd(25))) + commitBoth(baselineDeltaLog, amtDeltaLog, Seq(fakeAdd(26))) + commitBoth(baselineDeltaLog, amtDeltaLog, Seq(fakeAdd(27))) + commitBoth(baselineDeltaLog, amtDeltaLog, Seq(fakeAdd(28))) + createIncrementalAMTAndValidate( + baselineDeltaLog, + amtDeltaLog, + createIncrementalAMTWriteMetrics( + numOldLeavesUntouched = 3, + numRootEntriesExistingStatus = 4, + numLeavesExistingStatus = 3), + expectedNumIntermediateCommits = Some(5)) + } + } + } + + test("added actions fit in the root of a leafless AMT and are not spilled") { + withSQLConf(DeltaSQLConf.AMT_ENTRIES_PER_LEAF.key -> "8") { + withTables() { (baselineDeltaLog, amtDeltaLog) => + fullCheckpointPromotedToRoot(baselineDeltaLog, amtDeltaLog) + // No leaf pointers, so fixedRootCount is 0. The replay carries the promoted root's own file + // forward as a root-resident add, so the new root holds that one plus the 3 appended = 4, + // well under the cap of 8: nothing spills and every live add stays root-resident. + commitBoth(baselineDeltaLog, amtDeltaLog, (2 to 4).map(fakeAdd)) + createIncrementalAMTAndValidate( + baselineDeltaLog, + amtDeltaLog, + createIncrementalAMTWriteMetrics( + numRootEntriesExistingStatus = 4)) + } + } + } + + test("added actions exactly fill the root of a leafless AMT") { + withSQLConf(DeltaSQLConf.AMT_ENTRIES_PER_LEAF.key -> "8") { + withTables() { (baselineDeltaLog, amtDeltaLog) => + fullCheckpointPromotedToRoot(baselineDeltaLog, amtDeltaLog) + // fixedRootCount is 0 and the replay carries the bootstrap's own file forward as a + // root-resident add, so 8 live adds fill the root exactly to the cap. spillIfNeeded loops + // while the total is `> cap`, so an exactly-full root must not spill. + commitBoth(baselineDeltaLog, amtDeltaLog, (2 to 8).map(fakeAdd)) + createIncrementalAMTAndValidate( + baselineDeltaLog, + amtDeltaLog, + createIncrementalAMTWriteMetrics( + numRootEntriesExistingStatus = 8)) + } + } + } + + test("added actions overflow the root of a leafless AMT and spill into one leaf") { + withSQLConf(DeltaSQLConf.AMT_ENTRIES_PER_LEAF.key -> "8") { + withTables() { (baselineDeltaLog, amtDeltaLog) => + fullCheckpointPromotedToRoot(baselineDeltaLog, amtDeltaLog) + // 9 live adds (the bootstrap's file + 8 new) on a root with no pointers, cap 8: + // 0(fixed) + 0(spilled) + 9(remaining) = 9 > 8 -> spill a batch of 8 (remaining = 1) + // 0(fixed) + 1(spilled) + 1(remaining) = 2 <= 8 -> stop. + // => 1 new leaf, 1 root-resident add. + commitBoth(baselineDeltaLog, amtDeltaLog, (2 to 9).map(fakeAdd)) + createIncrementalAMTAndValidate( + baselineDeltaLog, + amtDeltaLog, + createIncrementalAMTWriteMetrics( + numNewLeaves = 1, + numRootEntriesExistingStatus = 1, + numLeavesAddedStatus = 1)) + } + } + } + + test("added actions overflow the root of a leafless AMT and spill into three leaves") { + withSQLConf(DeltaSQLConf.AMT_ENTRIES_PER_LEAF.key -> "8") { + withTables() { (baselineDeltaLog, amtDeltaLog) => + fullCheckpointPromotedToRoot(baselineDeltaLog, amtDeltaLog) + // 24 live adds (the bootstrap's file + 23 new) on a root with no pointers, cap 8: + // 0 + 0 + 24 = 24 > 8 -> spill 8 (remaining 16) + // 0 + 1 + 16 = 17 > 8 -> spill 8 (remaining 8) + // 0 + 2 + 8 = 10 > 8 -> spill 8 (remaining 0) + // => 3 new leaves, 0 root-resident adds. + commitBoth(baselineDeltaLog, amtDeltaLog, (2 to 24).map(fakeAdd)) + createIncrementalAMTAndValidate( + baselineDeltaLog, + amtDeltaLog, + createIncrementalAMTWriteMetrics( + numNewLeaves = 3, + numRootEntriesAddedStatus = 0, + numLeavesAddedStatus = 3)) + } + } + } +} diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTIncrementalWriteTestBase.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTIncrementalWriteTestBase.scala new file mode 100644 index 00000000000..6c20acdedb9 --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTIncrementalWriteTestBase.scala @@ -0,0 +1,596 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.amt + +import java.util.UUID + +import org.apache.spark.sql.delta.{DeltaLog, DeltaOperations, Snapshot} +import org.apache.spark.sql.delta.actions.{Action, AddFile, DeletionVectorDescriptor, RemoveFile} +import org.apache.spark.sql.delta.deletionvectors.ManifestBitmap + +import org.apache.spark.sql.SaveMode +import org.apache.spark.sql.functions.col + +/** Shared helpers for AMTIncrementalWrite* suites. */ +abstract class AMTIncrementalWriteTestBase extends AMTCheckpointTestBase { + + import testImplicits._ + + /** A deterministic fake data file. Paths are unique per id so live sets are easy to reason on. */ + protected def fakeAdd(fileID: Int): AddFile = fakeAdd(fileID, dataChange = true) + + protected def fakeAdd(fileID: Int, dataChange: Boolean, numRecords: Long = 1L): AddFile = + AddFile( + path = f"part-$fileID%05d.parquet", + partitionValues = Map.empty, + size = 100L + fileID, + modificationTime = 1000L + fileID, + dataChange = dataChange, + stats = s"""{"numRecords":$numRecords}""") + + /** Creates the non-AMT never-checkpointed baseline table and the AMT-backed subject table. */ + protected def createTables( + baselineNonAMTDeltaTable: String, + amtDeltaTable: String, + amtTableLocation: Option[String]): (DeltaLog, DeltaLog) = { + // Baseline: interval huge so it is never checkpointed; its allFiles is pure log replay. + sql( + s"""CREATE TABLE $baselineNonAMTDeltaTable (id INT) USING DELTA + |TBLPROPERTIES ( + | 'delta.columnMapping.mode' = 'id', + | 'delta.enableDeletionVectors' = 'true', + | 'delta.checkpointInterval' = '1000000')""".stripMargin) + // Subject: AMT-backed; we drive its checkpoints explicitly, so interval is huge too. + createAMTTable(amtDeltaTable, checkpointInterval = 1000000, location = amtTableLocation) + (deltaLogForName(baselineNonAMTDeltaTable), deltaLogForName(amtDeltaTable)) + } + + /** The operation every write in this suite is committed under. */ + protected def writeOperation: DeltaOperations.Operation = + DeltaOperations.Write(SaveMode.Append) + + /** Commits the identical `actions` as a "WRITE" commit to both tables. */ + protected def commitBoth( + baselineDeltaLog: DeltaLog, + amtDeltaLog: DeltaLog, + actions: Seq[Action], + operation: DeltaOperations.Operation = writeOperation): Unit = { + val baselineActions = actions.map { + case a: AddFile => a.copy(backReference = None) + case r: RemoveFile => r.copy(backReference = None) + case other => other + } + baselineDeltaLog.startTransaction().commit(baselineActions, operation) + amtDeltaLog.startTransaction().commit(actions, operation) + } + + /** The live-file path set of a table via its normal snapshot (log replay for the baseline). */ + protected def livePathsInLatestSnapshot(deltaLog: DeltaLog): Set[String] = + liveAddFilesInLatestSnapshot(deltaLog).map(_.path).toSet + + /** + * The live-file path set the AMT provider reconstructs from its manifest tree (root + leaves, + * MDV / tracking.status honored). + */ + protected def livePathsInLatestAMTCheckpoint(deltaLog: DeltaLog): Set[String] = { + val provider = amtProvider(deltaLog.update()) + .getOrElse(fail("AMT table must be checkpoint-provider-backed.")) + provider.loadActionsForStateReconstruction(spark, deltaLog) + .getOrElse(fail("AMT provider must contribute leaf-derived file actions.")) + .where("add is not null").select("add.path").as[String].collect().toSet + } + + /** DATA-entry tracking statuses that mark a root-resident entry as a CDF tombstone. */ + protected val tombstoneTrackingStatuses = Set(Tracking.Status.Deleted, Tracking.Status.Replaced) + + /** + * Physical DATA-row counts of a manifest parquet keyed by `tracking.status`. + */ + protected def trackingStatusToAddFileCountMap(absManifestPath: String): Map[Int, Long] = + withManifestDataEntries(Seq(absManifestPath)) { entries => + entries.groupBy(col("tracking.status").as("status")).count() + .as[(Int, Long)].collect().toMap + } + + /** + * DATA-entry `location`s (each `add.path`, a file's unique id) in a manifest parquet, keyed by + * `tracking.status`. Lets a test assert *which* file carries each status, not just the counts. + */ + protected def trackingStatusToLocationsMap(absManifestPath: String): Map[Int, Set[String]] = + withManifestDataEntries(Seq(absManifestPath)) { entries => + entries.select(col("tracking.status").as("status"), col("location")) + .as[(Int, String)].collect() + .groupBy(_._1).map { case (status, rows) => status -> rows.map(_._2).toSet } + } + + /** Root-resident DATA-entry counts keyed by tracking.status (DATA_MANIFEST pointers excluded). */ + protected def rootDataEntryStatusToCount(amtDeltaLog: DeltaLog): Map[Int, Long] = { + val provider = amtProvider(amtDeltaLog.update()).getOrElse(fail("expected AMT")) + trackingStatusToAddFileCountMap( + provider.checkpointAction.contentRoot.getAbsolutePath(provider.tableRoot).toString) + } + + /** Root-resident DATA-entry `location`s keyed by tracking.status (pointers excluded). */ + protected def rootDataEntryStatusToLocations(amtDeltaLog: DeltaLog): Map[Int, Set[String]] = { + val provider = amtProvider(amtDeltaLog.update()).getOrElse(fail("expected AMT")) + trackingStatusToLocationsMap( + provider.checkpointAction.contentRoot.getAbsolutePath(provider.tableRoot).toString) + } + + /** The live AddFiles of a table's latest snapshot, for building real removes. */ + protected def liveAddFilesInLatestSnapshot(deltaLog: DeltaLog): Seq[AddFile] = + liveAddFiles(deltaLog.update()) + + /** + * The leaf-resident live files of the AMT table grouped by their leaf's relative location, read + * from the reconstructed AddFiles' back references. Root-resident live files (no back reference) + * are excluded. Lets a scenario pick removes against the *actual* leaf assignment. + */ + protected def leafToAddFileMap(amtDeltaLog: DeltaLog): Map[String, Seq[AddFile]] = + liveAddFilesInLatestSnapshot(amtDeltaLog) + .flatMap(add => add.backReference.map(br => br.manifest -> add)) + .groupBy(_._1).map { case (leaf, pairs) => leaf -> pairs.map(_._2) } + + /** Asserts the table's current AMT checkpoint describes exactly `expectedVersion`. */ + protected def assertCheckpointDescribesVersion( + amtDeltaLog: DeltaLog, expectedVersion: Long): Unit = { + val provider = amtProvider(amtDeltaLog.update()).getOrElse(fail("expected AMT")) + assert(provider.checkpointAction.version == expectedVersion, + s"Checkpoint must describe version $expectedVersion; " + + s"got ${provider.checkpointAction.version}.") + } + + /** A real remove for the live file with the given id, carrying its stamped back reference. */ + protected def removeOf(amtDeltaLog: DeltaLog, fileID: Int): RemoveFile = { + val path = fakeAdd(fileID).path + val add = liveAddFilesInLatestSnapshot(amtDeltaLog).find(_.path == path) + .getOrElse(fail(s"fileID=$fileID ($path) is not live in the AMT table.")) + add.remove + } + + /** + * A descriptor-only synthetic deletion vector named `name` (a 'p'-type path descriptor pointing + * at no real bytes). + */ + protected def syntheticDv(name: String): DeletionVectorDescriptor = + DeletionVectorDescriptor.onDiskWithAbsolutePath( + // A `p` DV path must parse as an absolute URI (scheme required); the file is never read. + path = s"file:/$name", sizeInBytes = 5, cardinality = 5L, offset = Some(1)) + + /** + * Generates a Remove of the current file path (with whatever DV it has) plus an Add of the same + * file under a given DV. E.g. if the table holds file f1 with DV dv4, this returns a remove of + * (f1, dv4) and an add of (f1, given_dv). + */ + protected def removeAndReAddWithDV( + amtDeltaLog: DeltaLog, fileIdOrAddFile: Either[Int, AddFile], dvID: Int = 1): Seq[Action] = { + val liveAddFile = fileIdOrAddFile match { + case Left(fileID) => + val path = fakeAdd(fileID).path + liveAddFilesInLatestSnapshot(amtDeltaLog).find(_.path == path) + .getOrElse(fail(s"fileID=$fileID ($path) is not live in the AMT table.")) + case Right(add) => add + } + val newDv = syntheticDv(s"${liveAddFile.path}_dv_${dvID}") + assert(newDv != liveAddFile.deletionVector, + "new DV must differ from the file's current DV to change its (path, dv) key.") + // The re-add drops the back reference -- the Remove already carries it (masking the old leaf + // slot), and the re-added copy is a fresh (path, dv) key, not an existing leaf entry. It + // reports non-tight stats bounds (required for a DV-bearing file). + Seq( + liveAddFile.remove, + liveAddFile.copy( + deletionVector = newDv, + backReference = None, + stats = s"""{"numRecords":1,"tightBounds":false}""", + dataChange = true)) + } + + /** The current tree's leaf pointers keyed by relative `location`, with their MDV cardinality. */ + protected def leafToLeafMDVCardinalityMap(amtDeltaLog: DeltaLog): Map[String, Long] = { + val provider = amtProvider(amtDeltaLog.update()) + .getOrElse(fail("AMT table must be checkpoint-provider-backed.")) + provider.leaves.map(l => l.location -> l.manifest_info.dv_cardinality.getOrElse(0L)).toMap + } + + /** Root-resident DATA-entry counts (live adds, tombstones) read straight off the new root. */ + protected def liveAddsAndTombstonesCountInRoot(amtDeltaLog: DeltaLog): (Long, Long) = { + val provider = amtProvider(amtDeltaLog.update()) + .getOrElse(fail("AMT table must be checkpoint-provider-backed.")) + val byStatus = + trackingStatusToAddFileCountMap( + provider.checkpointAction.contentRoot.getAbsolutePath(provider.tableRoot).toString) + val liveAdds = + byStatus.filter { case (status, _) => + Tracking.Status.liveEntryStatuses.contains(status) }.values.sum + val tombstones = tombstoneTrackingStatuses.toSeq.map(byStatus.getOrElse(_, 0L)).sum + (liveAdds, tombstones) + } + + /** Decodes a manifest bitmap's set positions into a set (empty if the bytes are absent). */ + private def bitmapPositions(bytes: Array[Byte]): Set[Long] = + ManifestBitmap.fromSerializedByteArray(bytes).toArrayForTesting.toSet + + /** The per-commit CDF `tracking.deleted_positions` off a leaf pointer (empty if unset). */ + protected def leafDeletedPositions(leaf: DataManifestEntry): Set[Long] = + leaf.tracking.deleted_positions.map(bitmapPositions).getOrElse(Set.empty) + + /** The per-commit CDF `tracking.replaced_positions` off a leaf pointer (empty if unset). */ + protected def leafReplacedPositions(leaf: DataManifestEntry): Set[Long] = + leaf.tracking.replaced_positions.map(bitmapPositions).getOrElse(Set.empty) + + /** The current snapshot's leaf pointers, keyed by relative location. */ + protected def leafPointers(snapshot: Snapshot): Map[String, DataManifestEntry] = { + val provider = amtProvider(snapshot).getOrElse(fail("Snapshot has no AMTCheckpointProvider.")) + provider.leaves.map(l => l.location -> l).toMap + } + + /** The MDV cardinality numLeafCountBefore on a leaf pointer, 0 when it has none. */ + protected def mdvCardinality(leaf: DataManifestEntry): Long = + leaf.manifest_info.dv_cardinality.getOrElse(0L) + + /** + * Emits one incremental AMT and cross-checks its metrics three ways: the hand-specified + * `expectedAMTWriteMetrics` (author intent), the metrics the writer actually reported, and a + * structural derivation from the old tree vs. the new tree read straight off disk. Also compares + * the live files on baselineDeltaLog and amtDeltaLog. + * + * The emission route depends on `inlineAMTCommitActions`: + * - `None` (deferred): a follow-up OPTIMIZE CHECKPOINT folds the already-committed intermediate + * commits in with an empty actionsToCommit. + * - `Some(actions)` (inline): the actions ride the same commit that emits the tree + * ([[withInline]] sets the inline threshold to 1), so the metrics come off that commit and + * the checkpoint describes its OWN version -- unlike a deferred write (E1), which describes + * attemptVersion - 1. + */ + protected def createIncrementalAMTAndValidate( + baselineDeltaLog: DeltaLog, + amtDeltaLog: DeltaLog, + expectedAMTWriteMetrics: IncrementalAMTWriteMetrics, + expectedNumIntermediateCommits: Option[Int] = None, + inlineAMTCommitActions: Option[Seq[Action]] = None, + inlineOperation: DeltaOperations.Operation = writeOperation): Unit = { + // Snapshot the OLD tree's leaf pointers before the write: their MDV cardinality (for the shape + // derivation), the full pointers (to assert manifest_info counts stay immutable), and the count + // of DELETED tombstone pointers this rewrite is expected to drop. + val leafToLeafMDVCardinality = leafToLeafMDVCardinalityMap(amtDeltaLog) + val oldLeaves = leafPointers(amtDeltaLog.update()) + val oldDeletedLeafCount = amtProvider(amtDeltaLog.update()) + .getOrElse(fail("AMT table must be checkpoint-provider-backed.")) + .leaves.count(_.tracking.status == Tracking.Status.Deleted) + + val actualIncrementalAMTWriteMetrics = inlineAMTCommitActions match { + case Some(actions) => + val attemptVersion = amtDeltaLog.update().version + 1 + val metrics = withInline { + trackIncrementalAMTWriteMetrics(attemptVersion) { + commitBoth(baselineDeltaLog, amtDeltaLog, actions, inlineOperation) + } + }.getOrElse(fail("An inline incremental write must log IncrementalAMTWriteMetrics.")) + assertCheckpointDescribesVersion(amtDeltaLog, attemptVersion) + metrics + case None => + commitIncrementalCheckpointAndReturnMetrics(amtDeltaLog) + .getOrElse(fail("An incremental checkpoint must log IncrementalAMTWriteMetrics.")) + } + + // 1) Reported metrics match the author's hand-specified expectation. + assertIncrementalAMTWriteMetrics(actualIncrementalAMTWriteMetrics, expectedAMTWriteMetrics) + // 1b) The intermediate-commit count, for the tests that know exactly how many commits the + // window spans. Left unpinned otherwise, since it is the mechanical window size. + expectedNumIntermediateCommits.foreach { expected => + assert(actualIncrementalAMTWriteMetrics.numIntermediateCommits == expected, + s"Expected $expected intermediate commits; " + + s"got ${actualIncrementalAMTWriteMetrics.numIntermediateCommits}.") + } + // 2) Reported metrics match a structural derivation from old-tree -> new-tree on disk. + assertMetricsMatchTreeDelta( + amtDeltaLog, leafToLeafMDVCardinality, oldDeletedLeafCount, + actualIncrementalAMTWriteMetrics) + // 3) Differential live set + white-box tree invariants. + assertLiveAddFilesEquals(baselineDeltaLog, amtDeltaLog) + // 4) Carried-forward leaves keep their immutable manifest_info file counts. + assertCarriedLeafCountsImmutable(oldLeaves, amtDeltaLog) + } + + /** + * Carried-forward leaves (present in both the old and new tree by location) keep their + * manifest_info file/row counts unchanged -- those counts describe the leaf as written and are + * immutable across trees; only the MDV and tracking evolve. + */ + protected def assertCarriedLeafCountsImmutable( + oldLeaves: Map[String, DataManifestEntry], amtDeltaLog: DeltaLog): Unit = { + def fileAndRowCounts( + e: DataManifestEntry): (Int, Int, Int, Int, Int, Long, Long, Long, Long, Long) = + (e.manifest_info.added_files_count, e.manifest_info.existing_files_count, + e.manifest_info.deleted_files_count, e.manifest_info.replaced_files_count, + e.manifest_info.modified_files_count, + e.manifest_info.added_rows_count, e.manifest_info.existing_rows_count, + e.manifest_info.deleted_rows_count, e.manifest_info.replaced_rows_count, + e.manifest_info.modified_rows_count) + val newLeaves = leafPointers(amtDeltaLog.update()) + oldLeaves.foreach { case (location, oldLeaf) => + newLeaves.get(location).foreach { newLeaf => + assert(fileAndRowCounts(newLeaf) == fileAndRowCounts(oldLeaf), + s"Carried leaf $location: manifest_info file/row counts must be immutable; " + + s"old=${fileAndRowCounts(oldLeaf)} new=${fileAndRowCounts(newLeaf)}") + } + } + } + + + /** Asserts `actual` matches `expected` on all fields but the derived numIntermediateCommits. */ + protected def assertIncrementalAMTWriteMetrics( + actualMetrics: IncrementalAMTWriteMetrics, + expectedMetrics: IncrementalAMTWriteMetrics): Unit = { + val normalized = + expectedMetrics.copy(numIntermediateCommits = actualMetrics.numIntermediateCommits) + assert(actualMetrics == normalized, + s"Incremental shape mismatch.\n expected: $normalized\n actual: $actualMetrics") + } + + /** + * Derives the incremental write's shape from the structural difference between the old tree's + * leaf pointers (`oldAMTLeafToMDV`: location -> MDV cardinality, captured before the write) + * and the + * new tree read straight off disk, then asserts the reported `metrics` equal that derivation. + * This is independent of the writer's self-reported counters -- a leaf carried forward but + * silently re-packed, or a miscounted metric, diverges here. + * - a carried leaf whose MDV grew -> existing-leaf-updated, and the growth is MDV bits added; + * - a carried leaf whose MDV is unchanged -> existing-leaf-untouched; + * - a leaf location absent from the old tree -> a new (spilled) leaf; + * - root-resident live adds / tombstones are read directly off the new root. + */ + protected def assertMetricsMatchTreeDelta( + amtDeltaLog: DeltaLog, + oldAMTLeafToMDV: Map[String, Long], + oldDeletedLeafCount: Int, + metrics: IncrementalAMTWriteMetrics): Unit = { + val newAMTLeafToMDV = leafToLeafMDVCardinalityMap(amtDeltaLog) + // Materialize the carried leaves once (a lazy key-view would give inconsistent re-traversals). + val carriedLeafLocations: Seq[String] = + newAMTLeafToMDV.keys.filter(oldAMTLeafToMDV.contains).toSeq + val perLeafBitsAdded: Seq[Long] = + carriedLeafLocations.map(loc => newAMTLeafToMDV(loc) - oldAMTLeafToMDV(loc)) + val updated = perLeafBitsAdded.count(_ != 0L) + val untouched = perLeafBitsAdded.count(_ == 0L) + val newLeaves = newAMTLeafToMDV.keys.count(loc => !oldAMTLeafToMDV.contains(loc)) + val mdvBitsAdded = perLeafBitsAdded.sum + val rootDataEntryStatusToCountMap = rootDataEntryStatusToCount(amtDeltaLog) + + // Per-status breakdown over every leaf pointer in the new tree, derived independently of the + // writer's self-report; numStaleDeletedLeavesDropped is the old tree's DELETED tombstone count. + val newLeafPointers = amtProvider(amtDeltaLog.update()) + .getOrElse(fail("AMT table must be checkpoint-provider-backed.")).leaves + val statusToLeafCountMapping = + newLeafPointers.groupBy(_.tracking.status).map { case (s, ps) => s -> ps.size } + val deleteCDFBitsAdded = newLeafPointers.map(p => leafDeletedPositions(p).size).sum + val replaceCDFBitsAdded = newLeafPointers.map(p => leafReplacedPositions(p).size).sum + + val derived = metrics.copy( + numOldLeavesUpdated = updated, + numOldLeavesUntouched = untouched, + numNewLeaves = newLeaves, + numRootEntriesAddedStatus = + rootDataEntryStatusToCountMap.getOrElse(Tracking.Status.Added, 0L).toInt, + numRootEntriesExistingStatus = + rootDataEntryStatusToCountMap.getOrElse(Tracking.Status.Existing, 0L).toInt, + numRootEntriesModifiedStatus = + rootDataEntryStatusToCountMap.getOrElse(Tracking.Status.Modified, 0L).toInt, + numRootEntriesReplacedStatus = + rootDataEntryStatusToCountMap.getOrElse(Tracking.Status.Replaced, 0L).toInt, + numRootEntriesDeletedStatus = + rootDataEntryStatusToCountMap.getOrElse(Tracking.Status.Deleted, 0L).toInt, + numLeafMdvBitsAdded = mdvBitsAdded.toInt, + numLeafDeleteCDFBitsAdded = deleteCDFBitsAdded.toInt, + numLeafReplaceCDFBitsAdded = replaceCDFBitsAdded.toInt, + numLeavesAddedStatus = statusToLeafCountMapping.getOrElse(Tracking.Status.Added, 0), + numLeavesExistingStatus = statusToLeafCountMapping.getOrElse(Tracking.Status.Existing, 0), + numLeavesModifiedStatus = statusToLeafCountMapping.getOrElse(Tracking.Status.Modified, 0), + numLeavesDeletedStatus = statusToLeafCountMapping.getOrElse(Tracking.Status.Deleted, 0), + numStaleDeletedLeavesDropped = oldDeletedLeafCount) + assert(metrics == derived, + s"Reported metrics disagree with the old->new tree delta.\n" + + s" reported: $metrics\n derived: $derived\n" + + s" oldAMTLeafToMDV: $oldAMTLeafToMDV\n newAMTLeafToMDV: $newAMTLeafToMDV") + } + + /** + * Asserts the AMT tree reconstructs exactly the baseline's (and its own) live path set, and runs + * the white-box structural validation ([[assertTreeInvariants]]) on the same tree. + */ + protected def assertLiveAddFilesEquals( + baselineDeltaLog: DeltaLog, amtDeltaLog: DeltaLog): Unit = { + assert( + livePathsInLatestAMTCheckpoint(amtDeltaLog) == livePathsInLatestSnapshot(baselineDeltaLog), + "The AMT tree must reconstruct exactly the baseline table's live file set.") + assert(livePathsInLatestAMTCheckpoint(amtDeltaLog) == livePathsInLatestSnapshot(amtDeltaLog), + "The AMT table's reconstruction must equal its own allFiles.") + assertTreeInvariants(amtDeltaLog) + } + + /** + * Performs following validations on the underlying AMT (from latest snapshot): + * - each leaf pointer's `dv_cardinality` equals its decoded MDV bitmap size, and every MDV + * position is in range of that leaf's physical entry count (no stale / out-of-range mask); + * - conservation: the reader's live count equals root-resident live adds plus, over every leaf, + * (physical leaf entries - MDV cardinality) + * - no path is surfaced more than once by the reader. + */ + protected def assertTreeInvariants(amtDeltaLog: DeltaLog): Unit = { + val snapshot = amtDeltaLog.update() + val provider = amtProvider(snapshot) + .getOrElse(fail("AMT table must be checkpoint-provider-backed.")) + val tableRoot = provider.tableRoot + + // Root: only its live-status DATA entries (added/existing) reconstruct as live files. + val rootPath = provider.checkpointAction.contentRoot.getAbsolutePath(tableRoot).toString + val statusToCountMap = trackingStatusToAddFileCountMap(rootPath) + val rootLiveAdds = + statusToCountMap + .filter { case (status, _) => Tracking.Status.liveEntryStatuses.contains(status) } + .values.sum + + // Leaves: each pointer's MDV must be internally consistent, and the unmasked LIVE entries are + // the leaf's live contribution. A tombstone-only leaf (born ADDED but holding no live entry) + // and a DELETED leaf both contribute nothing live, matching the reader. + val leafLiveContribution = provider.leaves + .filter(_.tracking.status != Tracking.Status.Deleted) + .map { leaf => + val leafPath = leaf.getAbsolutePath(tableRoot).toString + val statusToCountMapForLeaf = trackingStatusToAddFileCountMap(leafPath) + val physicalEntries = statusToCountMapForLeaf.values.sum + val livePhysicalEntries = statusToCountMapForLeaf + .filter { case (status, _) => Tracking.Status.liveEntryStatuses.contains(status) } + .values.sum + val mdvDeclaredCardinality = leaf.manifest_info.dv_cardinality.getOrElse(0L) + val decoded = leaf.manifest_info.dv.map(ManifestBitmap.fromSerializedByteArray) + .getOrElse(ManifestBitmap.fromPositions(Seq.empty)) + assert(decoded.cardinality == mdvDeclaredCardinality, + s"Leaf ${leaf.location}: dv_cardinality=$mdvDeclaredCardinality but the decoded MDV " + + s"has ${decoded.cardinality} bits.") + assert(mdvDeclaredCardinality <= physicalEntries, + s"Leaf ${leaf.location}: MDV cardinality ($mdvDeclaredCardinality) exceeds physical " + + s"entries " + + s"($physicalEntries).") + decoded.toArrayForTesting.foreach { pos => + assert(pos >= 0 && pos < physicalEntries, + s"Leaf ${leaf.location}: MDV position $pos is out of range [0, $physicalEntries).") + } + livePhysicalEntries - mdvDeclaredCardinality + }.sum + + val reconstructed = provider.loadActionsForStateReconstruction(spark, amtDeltaLog) + .getOrElse(fail("AMT provider must contribute leaf-derived file actions.")) + .where("add is not null").select("add.path").as[String].collect().toSeq + assert(reconstructed.distinct.size == reconstructed.size, + s"The reader must not surface a duplicate path: " + + s"${reconstructed.diff(reconstructed.distinct).distinct}") + assert(reconstructed.size.toLong == rootLiveAdds + leafLiveContribution, + s"Tree conservation mismatch: reconstructed=${reconstructed.size}, " + + s"rootLiveAdds=$rootLiveAdds, leafLiveContribution=$leafLiveContribution.") + } + + /** + * Runs `body` with a fresh baseline+AMT table pair, cleaning both up afterward. + * + * Every test gets its own pair rather than sharing one: `withTable` drops the catalog entry but + * these tables are catalog-managed, so commit backfill can still be writing files when the next + * test CREATEs at the same path, which fails with DELTA_CREATE_TABLE_WITH_NON_EMPTY_LOCATION. + * A UUID keeps the names unique across shards and across reruns, where an earlier run's files may + * still sit under the warehouse directory. + */ + protected def withTables(amtTableLocation: Option[String] = None)( + body: (DeltaLog, DeltaLog) => Unit): Unit = { + val uniqueSuffix = UUID.randomUUID().toString.replace("-", "_") + val baselineNonAMTDeltaTable = s"amt_diff_baseline_$uniqueSuffix" + val amtDeltaTable = s"amt_diff_subject_$uniqueSuffix" + withTable(baselineNonAMTDeltaTable, amtDeltaTable) { + val (baselineDeltaLog, amtDeltaLog) = + createTables(baselineNonAMTDeltaTable, amtDeltaTable, amtTableLocation) + body(baselineDeltaLog, amtDeltaLog) + } + } + + /** Metrics with fields defaulted to 0; numIntermediateCommits is derived by the assertion. */ + // scalastyle:off argcount + protected def createIncrementalAMTWriteMetrics( + numOldLeavesUpdated: Int = 0, + numOldLeavesUntouched: Int = 0, + numNewLeaves: Int = 0, + numRootEntriesAddedStatus: Int = 0, + numRootEntriesExistingStatus: Int = 0, + numRootEntriesModifiedStatus: Int = 0, + numRootEntriesReplacedStatus: Int = 0, + numRootEntriesDeletedStatus: Int = 0, + numLeafMdvBitsAdded: Int = 0, + numLeafDeleteCDFBitsAdded: Int = 0, + numLeafReplaceCDFBitsAdded: Int = 0, + numLeavesAddedStatus: Int = 0, + numLeavesExistingStatus: Int = 0, + numLeavesModifiedStatus: Int = 0, + numLeavesDeletedStatus: Int = 0, + numStaleDeletedLeavesDropped: Int = 0): IncrementalAMTWriteMetrics = + IncrementalAMTWriteMetrics( + numIntermediateCommits = 0, + numOldLeavesUpdated = numOldLeavesUpdated, + numOldLeavesUntouched = numOldLeavesUntouched, + numNewLeaves = numNewLeaves, + numRootEntriesAddedStatus = numRootEntriesAddedStatus, + numRootEntriesExistingStatus = numRootEntriesExistingStatus, + numRootEntriesModifiedStatus = numRootEntriesModifiedStatus, + numRootEntriesReplacedStatus = numRootEntriesReplacedStatus, + numRootEntriesDeletedStatus = numRootEntriesDeletedStatus, + numLeafMdvBitsAdded = numLeafMdvBitsAdded, + numLeafDeleteCDFBitsAdded = numLeafDeleteCDFBitsAdded, + numLeafReplaceCDFBitsAdded = numLeafReplaceCDFBitsAdded, + numLeavesAddedStatus = numLeavesAddedStatus, + numLeavesExistingStatus = numLeavesExistingStatus, + numLeavesModifiedStatus = numLeavesModifiedStatus, + numLeavesDeletedStatus = numLeavesDeletedStatus, + numStaleDeletedLeavesDropped = numStaleDeletedLeavesDropped) + // scalastyle:on argcount + + /** + * Bootstraps a tree: the `initialIdRangeInLeaf` files packed into whole leaves via a full + * checkpoint, then (when `initialIdRangeInRoot` is non-empty) that id range appended + * root-resident via an incremental checkpoint. The caller must have set AMT_ENTRIES_PER_LEAF to + * `entriesPerLeaf`. Returns the leaf pointers of the bootstrapped tree. + */ + protected def setup( + baselineDeltaLog: DeltaLog, + amtDeltaLog: DeltaLog, + entriesPerLeaf: Int, + initialIdRangeInLeaf: Range, + initialIdRangeInRoot: Range = Range(0, 0), + rowCountPerDataEntry: Long = 1L): Seq[DataManifestEntry] = { + require(initialIdRangeInLeaf.size > entriesPerLeaf, + s"initialIdRangeInLeaf (${initialIdRangeInLeaf.size}) must exceed entriesPerLeaf " + + s"($entriesPerLeaf).") + val numLeaves = math.ceil(initialIdRangeInLeaf.size.toDouble / entriesPerLeaf).toInt + require(initialIdRangeInRoot.size <= entriesPerLeaf - numLeaves, + s"initialIdRangeInRoot (${initialIdRangeInRoot.size}) must be <= " + + s"${entriesPerLeaf - numLeaves} (entriesPerLeaf minus $numLeaves leaf pointers) so the " + + "root-resident adds do not spill.") + def adds(ids: Range): Seq[AddFile] = + ids.map(id => fakeAdd(id, dataChange = true, numRecords = rowCountPerDataEntry)) + commitBoth(baselineDeltaLog, amtDeltaLog, adds(initialIdRangeInLeaf)) + commitCheckpoint(amtDeltaLog, incremental = false) + assert(leafPointers(amtDeltaLog.update()).size == numLeaves, + s"the full checkpoint must pack $numLeaves leaves.") + if (initialIdRangeInRoot.nonEmpty) { + commitBoth(baselineDeltaLog, amtDeltaLog, adds(initialIdRangeInRoot)) + commitCheckpoint(amtDeltaLog, incremental = true) + assert(leafPointers(amtDeltaLog.update()).size == numLeaves, + "the root-resident adds must not spill any leaf.") + } + leafPointers(amtDeltaLog.update()).values.toSeq + } + + /** + * Bootstraps a leafless full AMT: a single live file clusters into one manifest, which the full + * rewrite promotes to the root. Returns after asserting the tree really has no leaf pointers. + */ + protected def fullCheckpointPromotedToRoot( + baselineDeltaLog: DeltaLog, amtDeltaLog: DeltaLog): Unit = { + commitBoth(baselineDeltaLog, amtDeltaLog, Seq(fakeAdd(1))) + commitCheckpoint(amtDeltaLog, incremental = false) + val leafCount = amtProvider(amtDeltaLog.update()).getOrElse(fail("expected AMT")).leaves.size + assert(leafCount == 0, + s"A single-manifest full rewrite must be promoted to the root; got $leafCount leaves.") + } +} diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTIncrementalWriteTrackingMDVSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTIncrementalWriteTrackingMDVSuite.scala new file mode 100644 index 00000000000..2fa2b19c1dd --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTIncrementalWriteTrackingMDVSuite.scala @@ -0,0 +1,534 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.amt + +import org.apache.spark.sql.delta.DeltaLog +import org.apache.spark.sql.delta.actions.AddFile +import org.apache.spark.sql.delta.sources.DeltaSQLConf + +/** + * Leaf lifecycle: carry-forward, MDV masking of removed files, and the leaf-pointer + * tracking.status transition chains across a sequence of incremental rewrites. + */ +class AMTIncrementalWriteTrackingMDVSuite extends AMTIncrementalWriteTestBase { + + test("deleting one leaf-resident file sets a single MDV bit on its leaf") { + withSQLConf(DeltaSQLConf.AMT_ENTRIES_PER_LEAF.key -> "5") { + withTables() { (baselineDeltaLog, amtDeltaLog) => + // 15 files at cap 5 clusters into several leaves; a bootstrap that produced one manifest + // would be promoted into the root, leaving no leaf to mask. + commitBoth(baselineDeltaLog, amtDeltaLog, (1 to 15).map(fakeAdd)) + commitCheckpoint(amtDeltaLog, incremental = false) + val leafCount = leafToAddFileMap(amtDeltaLog).size + assert(leafCount >= 2, s"Need a tree-shaped bootstrap; got $leafCount leaves.") + // Delete one file, so exactly its own leaf is updated and every sibling stays untouched. + val victim = leafToAddFileMap(amtDeltaLog).toSeq.sortBy(_._1).head._2.head + commitBoth(baselineDeltaLog, amtDeltaLog, Seq(victim.remove)) + createIncrementalAMTAndValidate( + baselineDeltaLog, + amtDeltaLog, + createIncrementalAMTWriteMetrics( + numOldLeavesUpdated = 1, + numOldLeavesUntouched = leafCount - 1, + numLeafMdvBitsAdded = 1, + numLeavesModifiedStatus = 1, + numLeavesExistingStatus = leafCount - 1)) + } + } + } + + test("MDV masking applies independently across two distinct leaves") { + withSQLConf(DeltaSQLConf.AMT_ENTRIES_PER_LEAF.key -> "5") { + withTables() { (baselineDeltaLog, amtDeltaLog) => + // 15 files at cap 5 -> ceil(15/5)=3 leaves, all non-empty (the full-rewrite fills them). + commitBoth(baselineDeltaLog, amtDeltaLog, (1 to 15).map(fakeAdd)) + commitCheckpoint(amtDeltaLog, incremental = false) + val leafToAddFileMapping = leafToAddFileMap(amtDeltaLog) + assert(leafToAddFileMapping.size >= 2, + s"Need >= 2 leaves; got ${leafToAddFileMapping.size}.") + // Pick one file from each of two distinct leaves, so exactly two leaves are updated. + val twoLeaves = leafToAddFileMapping.toSeq.sortBy(_._1).take(2) + val victims = twoLeaves.map { case (_, files) => files.head } + val untouchedLeaves = leafToAddFileMapping.size - 2 + commitBoth(baselineDeltaLog, amtDeltaLog, victims.map(_.remove)) + createIncrementalAMTAndValidate( + baselineDeltaLog, + amtDeltaLog, + createIncrementalAMTWriteMetrics( + numOldLeavesUpdated = 2, + numOldLeavesUntouched = untouchedLeaves, + numLeafMdvBitsAdded = 2, + numLeavesModifiedStatus = 2, + numLeavesExistingStatus = untouchedLeaves)) + } + } + } + + test("deleting two files from the SAME leaf adds two bits to one leaf") { + withSQLConf(DeltaSQLConf.AMT_ENTRIES_PER_LEAF.key -> "5") { + withTables() { (baselineDeltaLog, amtDeltaLog) => + commitBoth(baselineDeltaLog, amtDeltaLog, (1 to 15).map(fakeAdd)) + commitCheckpoint(amtDeltaLog, incremental = false) + val leafToAddFileMapping = leafToAddFileMap(amtDeltaLog) + // A leaf holding >= 2 files; delete two of them -> one leaf updated, two bits. + val (_, files) = leafToAddFileMapping.toSeq.sortBy(_._1).find(_._2.size >= 2) + .getOrElse(fail("Expected some leaf to hold >= 2 files at cap 5 with 15 files.")) + val untouchedLeaves = leafToAddFileMapping.size - 1 + commitBoth(baselineDeltaLog, amtDeltaLog, files.take(2).map(_.remove)) + createIncrementalAMTAndValidate( + baselineDeltaLog, + amtDeltaLog, + createIncrementalAMTWriteMetrics( + numOldLeavesUpdated = 1, + numOldLeavesUntouched = untouchedLeaves, + numLeafMdvBitsAdded = 2, + numLeavesModifiedStatus = 1, + numLeavesExistingStatus = untouchedLeaves)) + } + } + } + + test("multiple deferred intermediate commits accumulate all their leaf removes in the MDV") { + withSQLConf(DeltaSQLConf.AMT_ENTRIES_PER_LEAF.key -> "5") { + withTables() { (baselineDeltaLog, amtDeltaLog) => + // 15 files at cap 5 clusters into several leaves rather than one promoted root manifest. + commitBoth(baselineDeltaLog, amtDeltaLog, (1 to 15).map(fakeAdd)) + commitCheckpoint(amtDeltaLog, incremental = false) + val leafToAddFileMapping = leafToAddFileMap(amtDeltaLog) + assert(leafToAddFileMapping.size >= 2, + s"Need a tree-shaped bootstrap; got ${leafToAddFileMapping.size} leaves.") + // One victim from each of two distinct leaves, so the two intermediate commits land their + // bits on different carried pointers. + val twoLeaves = leafToAddFileMapping.toSeq.sortBy(_._1).take(2) + val victimFromLeaf1 = twoLeaves.head._2.head + val victimFromLeaf2 = twoLeaves(1)._2.head + // Two separate intermediate commits each remove one leaf file; the deferred incremental + // must accumulate both bits though neither is this commit's own action. + commitBoth(baselineDeltaLog, amtDeltaLog, Seq(victimFromLeaf1.remove)) + commitBoth(baselineDeltaLog, amtDeltaLog, Seq(victimFromLeaf2.remove)) + createIncrementalAMTAndValidate( + baselineDeltaLog, + amtDeltaLog, + createIncrementalAMTWriteMetrics( + numOldLeavesUpdated = 2, + numOldLeavesUntouched = leafToAddFileMapping.size - 2, + numLeafMdvBitsAdded = 2, + numLeavesModifiedStatus = 2, + numLeavesExistingStatus = leafToAddFileMapping.size - 2)) + } + } + } + + test("consecutive incremental AMT deletes accumulate MDV bits across writes") { + withSQLConf(DeltaSQLConf.AMT_ENTRIES_PER_LEAF.key -> "5") { + withTables() { (baselineDeltaLog, amtDeltaLog) => + // 15 files at cap 5 clusters into several leaves rather than one promoted root manifest. + commitBoth(baselineDeltaLog, amtDeltaLog, (1 to 15).map(fakeAdd)) + commitCheckpoint(amtDeltaLog, incremental = false) + val leafToAddFileMapping = leafToAddFileMap(amtDeltaLog) + assert(leafToAddFileMapping.size >= 2, + s"Need a tree-shaped bootstrap; got ${leafToAddFileMapping.size} leaves.") + // Both victims share one leaf, so each write updates that same carried pointer. + val victims = leafToAddFileMapping.toSeq.sortBy(_._1).find(_._2.size >= 2) + .getOrElse(fail("Expected some leaf to hold >= 2 files."))._2.take(2) + val untouched = leafToAddFileMapping.size - 1 + commitBoth(baselineDeltaLog, amtDeltaLog, Seq(victims.head.remove)) + createIncrementalAMTAndValidate( + baselineDeltaLog, + amtDeltaLog, + createIncrementalAMTWriteMetrics( + numOldLeavesUpdated = 1, + numOldLeavesUntouched = untouched, + numLeafMdvBitsAdded = 1, + numLeavesModifiedStatus = 1, + numLeavesExistingStatus = untouched)) + // The second write adds only its own bit; the leaf's cumulative MDV covers both, checked + // by the live-set baselineDeltaLog. + commitBoth(baselineDeltaLog, amtDeltaLog, Seq(victims(1).remove)) + createIncrementalAMTAndValidate( + baselineDeltaLog, + amtDeltaLog, + createIncrementalAMTWriteMetrics( + numOldLeavesUpdated = 1, + numOldLeavesUntouched = untouched, + numLeafMdvBitsAdded = 1, + numLeavesModifiedStatus = 1, + numLeavesExistingStatus = untouched)) + } + } + } + + test("incremental AMT writes handles paths with spaces correctly") { + withTempDir { baseDir => + // A leaf pointer's `location` and a remove's stamped `backReference.manifest` are both raw, + // table-root-relative AMT paths. A mismatch here would silently no-op the MDV, leaving the + // removed file live, which createIncrementalAMTAndValidate's live-set oracle catches. + val tableRoot = new java.io.File(baseDir, "amt tbl") + withTables(amtTableLocation = Some(tableRoot.toString)) { + (baselineDeltaLog, amtDeltaLog) => + withSQLConf(DeltaSQLConf.AMT_ENTRIES_PER_LEAF.key -> "5") { + // 15 files at cap 5 clusters into several leaves rather than one promoted root manifest. + commitBoth(baselineDeltaLog, amtDeltaLog, (1 to 15).map(fakeAdd)) + commitCheckpoint(amtDeltaLog, incremental = false) + val leafToAddFileMapping = leafToAddFileMap(amtDeltaLog) + assert(leafToAddFileMapping.size >= 2, + s"Need a tree-shaped bootstrap; got ${leafToAddFileMapping.size} leaves.") + // Every leaf key must be table-root-relative, never the absolute path that would make the + // string match below succeed only by coincidence. + leafToAddFileMapping.keys.foreach { leaf => + assert(leaf.startsWith("metadata/") && !leaf.contains(baseDir.toString), + s"Leaf key must be table-root-relative; got $leaf.") + } + val victim = leafToAddFileMapping.toSeq.sortBy(_._1).head._2.head + commitBoth(baselineDeltaLog, amtDeltaLog, Seq(victim.remove)) + createIncrementalAMTAndValidate( + baselineDeltaLog, + amtDeltaLog, + createIncrementalAMTWriteMetrics( + numOldLeavesUpdated = 1, + numOldLeavesUntouched = leafToAddFileMapping.size - 1, + numLeafMdvBitsAdded = 1, + numLeavesModifiedStatus = 1, + numLeavesExistingStatus = leafToAddFileMapping.size - 1)) + } + } + } + } + + test("an incremental AMT carries the leaf parquet forward byte-for-byte") { + withSQLConf(DeltaSQLConf.AMT_ENTRIES_PER_LEAF.key -> "5") { + withTables() { (baselineDeltaLog, amtDeltaLog) => + // 15 files at cap 5 clusters into several leaves; a single manifest would be promoted into + // the root, leaving no leaf parquet to carry forward. + commitBoth(baselineDeltaLog, amtDeltaLog, (1 to 15).map(fakeAdd)) + commitCheckpoint(amtDeltaLog, incremental = false) + val provider = amtProvider(amtDeltaLog.update()).getOrElse(fail("expected AMT")) + assert(provider.leaves.size >= 2, + s"Need a tree-shaped bootstrap; got ${provider.leaves.size} leaves.") + // Fingerprint every leaf parquet on disk (path, length, modification time) pre-write. + val fsRoot = provider.tableRoot.getFileSystem(amtDeltaLog.newDeltaHadoopConf()) + def leafFingerprints(p: AMTCheckpointProvider): Set[(String, Long, Long)] = + p.leaves.map { leaf => + val st = fsRoot.getFileStatus(leaf.getAbsolutePath(p.tableRoot)) + (leaf.location, st.getLen, st.getModificationTime) + }.toSet + val before = leafFingerprints(provider) + // A delete of a leaf-resident file: the pointer's MDV changes, but the leaf FILE must not. + val leafToAddFileMapping = leafToAddFileMap(amtDeltaLog) + val victim = leafToAddFileMapping.toSeq.sortBy(_._1).head._2.head + commitBoth(baselineDeltaLog, amtDeltaLog, Seq(victim.remove)) + createIncrementalAMTAndValidate( + baselineDeltaLog, + amtDeltaLog, + createIncrementalAMTWriteMetrics( + numOldLeavesUpdated = 1, + numOldLeavesUntouched = leafToAddFileMapping.size - 1, + numLeafMdvBitsAdded = 1, + numLeavesModifiedStatus = 1, + numLeavesExistingStatus = leafToAddFileMapping.size - 1)) + val after = + leafFingerprints(amtProvider(amtDeltaLog.update()).getOrElse(fail("expected AMT"))) + assert(after == before, + s"Incremental must carry leaves forward untouched.\n before=$before\n after=$after") + } + } + } + + test("an untouched sibling leaf keeps an empty MDV while another is masked") { + withSQLConf(DeltaSQLConf.AMT_ENTRIES_PER_LEAF.key -> "5") { + withTables() { (baselineDeltaLog, amtDeltaLog) => + commitBoth(baselineDeltaLog, amtDeltaLog, (1 to 15).map(fakeAdd)) // 3 leaves at cap 5. + commitCheckpoint(amtDeltaLog, incremental = false) + val leafToAddFileMapping = leafToAddFileMap(amtDeltaLog) + assert(leafToAddFileMapping.size >= 2, + s"Need >= 2 leaves; got ${leafToAddFileMapping.size}.") + // Delete one file from exactly one leaf; the other leaves must keep an empty MDV. + val (victimLeaf, victimFiles) = leafToAddFileMapping.toSeq.minBy(_._1) + commitBoth(baselineDeltaLog, amtDeltaLog, Seq(victimFiles.head.remove)) + createIncrementalAMTAndValidate( + baselineDeltaLog, + amtDeltaLog, + createIncrementalAMTWriteMetrics( + numOldLeavesUpdated = 1, + numOldLeavesUntouched = leafToAddFileMapping.size - 1, + numLeafMdvBitsAdded = 1, + numLeavesModifiedStatus = 1, + numLeavesExistingStatus = leafToAddFileMapping.size - 1)) + val provider = amtProvider(amtDeltaLog.update()).getOrElse(fail("expected AMT")) + provider.leaves.foreach { leaf => + val card = leaf.manifest_info.dv_cardinality.getOrElse(0L) + if (leaf.location == victimLeaf) { + assert(card == 1L, s"The victim leaf must carry exactly one MDV bit; got $card.") + } else { + assert(card == 0L, s"Sibling leaf ${leaf.location} must keep an empty MDV; got $card.") + } + } + } + } + } + + test("a stale DELETED leaf is dropped by the next incremental rewrite") { + withSQLConf(leafPackingConfs: _*) { + withTables() { (baselineDeltaLog, amtDeltaLog) => + // First reach the fully-masked state: every leaf carried as a DELETED tombstone. + commitBoth(baselineDeltaLog, amtDeltaLog, (1 to leafPackedFiles).map(fakeAdd)) + commitCheckpoint(amtDeltaLog, incremental = false) + val leafCount = leafToAddFileMap(amtDeltaLog).size + assert(leafCount >= 2, s"Need a tree-shaped bootstrap; got $leafCount leaves.") + commitBoth(baselineDeltaLog, amtDeltaLog, + leafToAddFileMap(amtDeltaLog).values.flatten.toSeq.map(_.remove)) + commitCheckpoint(amtDeltaLog, incremental = true) + assert(amtProvider(amtDeltaLog.update()).getOrElse(fail("expected AMT")) + .leaves.count(_.tracking.status == Tracking.Status.Deleted) == leafCount, + "precondition: the first incremental rewrite must leave DELETED tombstones.") + + // A second bare incremental rewrite carries nothing new, so it must drop the stale DELETED + // pointers and report them as numStaleDeletedLeavesDropped, leaving an empty tree. + val metrics = commitIncrementalCheckpointAndReturnMetrics(amtDeltaLog) + .getOrElse(fail("An incremental checkpoint must log metrics.")) + assert(metrics.numStaleDeletedLeavesDropped == leafCount, + s"All $leafCount stale DELETED leaves must be dropped; got " + + s"${metrics.numStaleDeletedLeavesDropped}.") + assert(amtProvider(amtDeltaLog.update()).getOrElse(fail("expected AMT")).leaves.isEmpty, + "The stale DELETED pointers must be gone from the new tree.") + assert(livePathsInLatestAMTCheckpoint(amtDeltaLog).isEmpty, + "The AMT tree must reconstruct an empty live set.") + } + } + } + + test("deleting every leaf-resident file masks every leaf fully, marking each DELETED") { + withSQLConf(leafPackingConfs: _*) { + withTables() { (baselineDeltaLog, amtDeltaLog) => + // The extreme end of per-leaf MDV masking: mask EVERY position on EVERY leaf, not one. + // Each leaf's cumulative MDV then covers all its entries, so its pointer is carried as a + // DELETED tombstone (the reader skips it) and the tree reconstructs an empty live set. + commitBoth(baselineDeltaLog, amtDeltaLog, (1 to leafPackedFiles).map(fakeAdd)) + commitCheckpoint(amtDeltaLog, incremental = false) + val leafToAddFileMapping = leafToAddFileMap(amtDeltaLog) + assert(leafToAddFileMapping.size >= 2, + s"Need a tree-shaped bootstrap; got ${leafToAddFileMapping.size} leaves.") + val allLeafFiles = leafToAddFileMapping.values.flatten.toSeq + commitBoth(baselineDeltaLog, amtDeltaLog, allLeafFiles.map(_.remove)) + createIncrementalAMTAndValidate( + baselineDeltaLog, + amtDeltaLog, + createIncrementalAMTWriteMetrics( + numOldLeavesUpdated = leafToAddFileMapping.size, + numLeafMdvBitsAdded = allLeafFiles.size, + numLeavesDeletedStatus = leafToAddFileMapping.size)) + // Every carried leaf pointer is now a DELETED tombstone. + val statuses = amtProvider(amtDeltaLog.update()).getOrElse(fail("expected AMT")) + .leaves.map(_.tracking.status) + assert(statuses.forall(_ == Tracking.Status.Deleted) && + statuses.size == leafToAddFileMapping.size, + s"Every fully-masked leaf must be DELETED; got $statuses.") + assert(livePathsInLatestSnapshot(baselineDeltaLog).isEmpty, + "The baseline table must have no live files.") + assert(livePathsInLatestAMTCheckpoint(amtDeltaLog).isEmpty, + "The AMT tree must reconstruct an empty live set.") + + // The next incremental rewrite drops every DELETED tombstone, leaving a fully empty tree: + // no leaf pointers, and a root that holds no DATA entries. + commitCheckpoint(amtDeltaLog, incremental = true) + val provider = amtProvider(amtDeltaLog.update()).getOrElse(fail("expected AMT")) + assert(provider.leaves.isEmpty, + s"The next rewrite must drop all DELETED leaves; got ${provider.leaves.size}.") + assert(liveAddsAndTombstonesCountInRoot(amtDeltaLog) == (0L, 0L), + "The new root must hold no DATA entries (no live adds, no tombstones).") + assert(livePathsInLatestAMTCheckpoint(amtDeltaLog).isEmpty, + "The fully empty tree must reconstruct an empty live set.") + } + } + } + + /** Human-readable [[Tracking.Status]] name, used in transition-failure messages. */ + private def statusName(status: Int): String = status match { + case Tracking.Status.Added => "ADDED" + case Tracking.Status.Existing => "EXISTING" + case Tracking.Status.Modified => "MODIFIED" + case Tracking.Status.Deleted => "DELETED" + case Tracking.Status.Replaced => "REPLACED" + case other => s"status($other)" + } + + /** The leaf pointer's tracking.status at `location`, or None if it is no longer listed. */ + private def leafStatusAt(amtDeltaLog: DeltaLog, location: String): Option[Int] = + amtProvider(amtDeltaLog.update()).getOrElse(fail("expected AMT")) + .leaves.find(_.location == location).map(_.tracking.status) + + /** The leaf at `location` must currently carry `expected`; `step` labels the transition edge. */ + private def assertLeafStatus( + amtDeltaLog: DeltaLog, location: String, expected: Int, step: String): Unit = { + val actual = leafStatusAt(amtDeltaLog, location) + assert(actual.contains(expected), + s"$step: leaf $location must be ${statusName(expected)}; " + + s"got ${actual.map(statusName).getOrElse("")}.") + } + + /** The leaf's still-live files (read via back reference), sorted by path. */ + private def liveLeafFiles(amtDeltaLog: DeltaLog, location: String): Seq[AddFile] = + leafToAddFileMap(amtDeltaLog).getOrElse(location, Seq.empty).sortBy(_.path) + + /** + * Bootstraps a multi-leaf full AMT ([[leafPackedFiles]] files) and returns the location of its + * largest leaf, asserting that leaf starts ADDED. The largest leaf packs at least + * `ceil(leafPackedFiles / numLeaves)` files -- headroom to mask across several commits before it + * empties. Every transition chain below begins from this freshly written ADDED leaf. + */ + private def bootstrapTargetLeaf(baselineDeltaLog: DeltaLog, amtDeltaLog: DeltaLog): String = { + commitBoth(baselineDeltaLog, amtDeltaLog, (1 to leafPackedFiles).map(fakeAdd)) + commitCheckpoint(amtDeltaLog, incremental = false) + val leafToAddFileMapping = leafToAddFileMap(amtDeltaLog) + assert(leafToAddFileMapping.size >= 2, + s"Need a tree-shaped bootstrap; got ${leafToAddFileMapping.size} leaves.") + val target = + leafToAddFileMapping.toSeq.sortBy { case (loc, files) => (-files.size, loc) }.head._1 + assertLeafStatus(amtDeltaLog, target, Tracking.Status.Added, "bootstrap") + target + } + + /** Removes `files` from both tables, then lands one incremental AMT checkpoint. */ + private def removeFilesAndCheckpoint( + baselineDeltaLog: DeltaLog, amtDeltaLog: DeltaLog, files: Seq[AddFile]): Unit = { + commitBoth(baselineDeltaLog, amtDeltaLog, files.map(_.remove)) + commitCheckpoint(amtDeltaLog, incremental = true) + } + + test("ADDED -> EXISTING (a freshly written leaf carried untouched)") { + withSQLConf(leafPackingConfs: _*) { + withTables() { (baselineDeltaLog, amtDeltaLog) => + val leaf = bootstrapTargetLeaf(baselineDeltaLog, amtDeltaLog) + // A bare incremental rewrite carries the leaf forward with no new masking. + commitCheckpoint(amtDeltaLog, incremental = true) + assertLeafStatus(amtDeltaLog, leaf, Tracking.Status.Existing, "ADDED -> EXISTING") + assertLiveAddFilesEquals(baselineDeltaLog, amtDeltaLog) + } + } + } + + test("ADDED -> DELETED -> removed (fully masked, then dropped)") { + withSQLConf(leafPackingConfs: _*) { + withTables() { (baselineDeltaLog, amtDeltaLog) => + val leaf = bootstrapTargetLeaf(baselineDeltaLog, amtDeltaLog) + // Mask every entry of the leaf: its cumulative MDV covers the whole leaf -> DELETED. + removeFilesAndCheckpoint(baselineDeltaLog, amtDeltaLog, liveLeafFiles(amtDeltaLog, leaf)) + assertLeafStatus(amtDeltaLog, leaf, Tracking.Status.Deleted, "ADDED -> DELETED") + // The next rewrite drops the stale DELETED tombstone. + commitCheckpoint(amtDeltaLog, incremental = true) + assert(leafStatusAt(amtDeltaLog, leaf).isEmpty, + s"DELETED -> removed: leaf $leaf must be dropped by the next rewrite.") + assertLiveAddFilesEquals(baselineDeltaLog, amtDeltaLog) + } + } + } + + test("ADDED -> MODIFIED (a freshly written leaf partially masked)") { + withSQLConf(leafPackingConfs: _*) { + withTables() { (baselineDeltaLog, amtDeltaLog) => + val leaf = bootstrapTargetLeaf(baselineDeltaLog, amtDeltaLog) + // Mask one entry: some live, some masked -> MODIFIED. + removeFilesAndCheckpoint( + baselineDeltaLog, amtDeltaLog, Seq(liveLeafFiles(amtDeltaLog, leaf).head)) + assertLeafStatus(amtDeltaLog, leaf, Tracking.Status.Modified, "ADDED -> MODIFIED") + assertLiveAddFilesEquals(baselineDeltaLog, amtDeltaLog) + } + } + } + + test("ADDED -> EXISTING -> EXISTING (carried untouched twice)") { + withSQLConf(leafPackingConfs: _*) { + withTables() { (baselineDeltaLog, amtDeltaLog) => + val leaf = bootstrapTargetLeaf(baselineDeltaLog, amtDeltaLog) + commitCheckpoint(amtDeltaLog, incremental = true) + assertLeafStatus(amtDeltaLog, leaf, Tracking.Status.Existing, "ADDED -> EXISTING") + commitCheckpoint(amtDeltaLog, incremental = true) + assertLeafStatus(amtDeltaLog, leaf, Tracking.Status.Existing, "EXISTING -> EXISTING") + assertLiveAddFilesEquals(baselineDeltaLog, amtDeltaLog) + } + } + } + + test("ADDED -> EXISTING -> DELETED -> removed") { + withSQLConf(leafPackingConfs: _*) { + withTables() { (baselineDeltaLog, amtDeltaLog) => + val leaf = bootstrapTargetLeaf(baselineDeltaLog, amtDeltaLog) + commitCheckpoint(amtDeltaLog, incremental = true) + assertLeafStatus(amtDeltaLog, leaf, Tracking.Status.Existing, "ADDED -> EXISTING") + // Mask every entry of the carried leaf -> DELETED. + removeFilesAndCheckpoint(baselineDeltaLog, amtDeltaLog, liveLeafFiles(amtDeltaLog, leaf)) + assertLeafStatus(amtDeltaLog, leaf, Tracking.Status.Deleted, "EXISTING -> DELETED") + commitCheckpoint(amtDeltaLog, incremental = true) + assert(leafStatusAt(amtDeltaLog, leaf).isEmpty, + s"DELETED -> removed: leaf $leaf must be dropped by the next rewrite.") + assertLiveAddFilesEquals(baselineDeltaLog, amtDeltaLog) + } + } + } + + test("ADDED -> EXISTING -> MODIFIED -> EXISTING") { + withSQLConf(leafPackingConfs: _*) { + withTables() { (baselineDeltaLog, amtDeltaLog) => + val leaf = bootstrapTargetLeaf(baselineDeltaLog, amtDeltaLog) + commitCheckpoint(amtDeltaLog, incremental = true) + assertLeafStatus(amtDeltaLog, leaf, Tracking.Status.Existing, "ADDED -> EXISTING") + removeFilesAndCheckpoint( + baselineDeltaLog, amtDeltaLog, Seq(liveLeafFiles(amtDeltaLog, leaf).head)) + assertLeafStatus(amtDeltaLog, leaf, Tracking.Status.Modified, "EXISTING -> MODIFIED") + // A carried leaf whose MDV does not grow this commit falls back to EXISTING. + commitCheckpoint(amtDeltaLog, incremental = true) + assertLeafStatus(amtDeltaLog, leaf, Tracking.Status.Existing, "MODIFIED -> EXISTING") + assertLiveAddFilesEquals(baselineDeltaLog, amtDeltaLog) + } + } + } + + test("ADDED -> EXISTING -> MODIFIED -> MODIFIED") { + withSQLConf(leafPackingConfs: _*) { + withTables() { (baselineDeltaLog, amtDeltaLog) => + val leaf = bootstrapTargetLeaf(baselineDeltaLog, amtDeltaLog) + commitCheckpoint(amtDeltaLog, incremental = true) + assertLeafStatus(amtDeltaLog, leaf, Tracking.Status.Existing, "ADDED -> EXISTING") + removeFilesAndCheckpoint( + baselineDeltaLog, amtDeltaLog, Seq(liveLeafFiles(amtDeltaLog, leaf).head)) + assertLeafStatus(amtDeltaLog, leaf, Tracking.Status.Modified, "EXISTING -> MODIFIED") + // Mask one more still-live entry: the MDV grows again but some entries remain live. + removeFilesAndCheckpoint( + baselineDeltaLog, amtDeltaLog, Seq(liveLeafFiles(amtDeltaLog, leaf).head)) + assertLeafStatus(amtDeltaLog, leaf, Tracking.Status.Modified, "MODIFIED -> MODIFIED") + assertLiveAddFilesEquals(baselineDeltaLog, amtDeltaLog) + } + } + } + + test("ADDED -> EXISTING -> MODIFIED -> DELETED") { + withSQLConf(leafPackingConfs: _*) { + withTables() { (baselineDeltaLog, amtDeltaLog) => + val leaf = bootstrapTargetLeaf(baselineDeltaLog, amtDeltaLog) + commitCheckpoint(amtDeltaLog, incremental = true) + assertLeafStatus(amtDeltaLog, leaf, Tracking.Status.Existing, "ADDED -> EXISTING") + removeFilesAndCheckpoint( + baselineDeltaLog, amtDeltaLog, Seq(liveLeafFiles(amtDeltaLog, leaf).head)) + assertLeafStatus(amtDeltaLog, leaf, Tracking.Status.Modified, "EXISTING -> MODIFIED") + // Mask every remaining live entry: the cumulative MDV now covers the whole leaf -> DELETED. + removeFilesAndCheckpoint(baselineDeltaLog, amtDeltaLog, liveLeafFiles(amtDeltaLog, leaf)) + assertLeafStatus(amtDeltaLog, leaf, Tracking.Status.Deleted, "MODIFIED -> DELETED") + assertLiveAddFilesEquals(baselineDeltaLog, amtDeltaLog) + } + } + } +} diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTInheritanceReadSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTInheritanceReadSuite.scala new file mode 100644 index 00000000000..a5f9f5ce048 --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTInheritanceReadSuite.scala @@ -0,0 +1,428 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.amt + +import org.apache.spark.sql.delta.DeltaLog +import org.apache.spark.sql.delta.actions.{Checkpoint, ContentRoot} +import org.apache.spark.sql.delta.deletionvectors.ManifestBitmap +import org.apache.spark.sql.delta.implicits.amtSingleActionEncoder +import org.apache.spark.sql.delta.util.FileNames +import org.apache.hadoop.fs.Path + +import org.apache.spark.sql.functions.col + +/** + * The AMT writer materializes every leaf value today, so no tree it produces exercises + * inheritance. These tests therefore hand-build manifest trees with deliberately null leaf + * tracking. + */ +class AMTInheritanceReadSuite extends AMTCheckpointTestBase { + + import testImplicits._ + + /** File sequence number assigned on the root pointer in inheritance scenarios. */ + private val inheritedFileSequenceNumber = 42L + + /** The row-tracking view of one file the reader reconstructed from a manifest tree. */ + private case class ReconstructedFile( + path: String, + baseRowId: Option[Long], + defaultRowCommitVersion: Option[Long]) + + /** A leaf manifest written to disk, with the entries it was written from. */ + private case class WrittenLeaf(location: String, sizeInBytes: Long, entries: Seq[DataEntry]) + + /** + * An AMT table whose manifest tree a test replaces with a hand-built one. + * + * @param deltaLog The table's log. + * @param base A real emitted checkpoint action, whose protocol and metadata the synthetic trees + * reuse so only the `contentRoot` differs. + */ + private case class SyntheticTree(deltaLog: DeltaLog, base: Checkpoint) { + + /** Writes `rows` as a single manifest parquet, returning its stored location and size. */ + private def writeManifest(file: Path, rows: Seq[AMTSingleAction]): (String, Long) = { + val hadoopConf = deltaLog.newDeltaHadoopConf() + AMTWriteHelper.writeAMTParquet( + spark, hadoopConf, file, base.metaData, base.protocol, rows) + val fs = file.getFileSystem(hadoopConf) + val location = AMTUtils.relativizeLocation(deltaLog.dataPath.toString, file.toString) + (location, fs.getFileStatus(file).getLen) + } + + /** Writes `entries` to a fresh leaf manifest, in the order given. */ + def writeLeaf(entries: Seq[DataEntry]): WrittenLeaf = { + val metadataDir = FileNames.amtMetadataDirPath(deltaLog.dataPath) + val leafFile = FileNames.newAMTLeafManifestFile(metadataDir) + val (location, sizeInBytes) = writeManifest(leafFile, entries.map(_.wrap)) + // A row's inherited row id depends on the entries physically before it, so tests that state + // expected row ids are only meaningful if the rows landed in the order they were written. + val onDisk = allowReadWithinDeltaLog { + spark.read.parquet(leafFile.toString) + .orderBy(col("_metadata.row_index")) + .select(col("location")).as[String].collect().toSeq + } + assert(onDisk == entries.map(_.location), + s"leaf rows must keep the order they were written in; got $onDisk") + WrittenLeaf(location, sizeInBytes, entries) + } + + /** + * Writes `rootEntries` as the root manifest and returns the files a reader reconstructs from + * the resulting tree. + */ + def reconstruct(rootEntries: Seq[AMTSingleAction]): Map[String, ReconstructedFile] = { + val metadataDir = FileNames.amtMetadataDirPath(deltaLog.dataPath) + val (location, sizeInBytes) = + writeManifest(FileNames.newAMTRootManifestFile(metadataDir), rootEntries) + val checkpoint = + base.copy(contentRoot = + ContentRoot(path = location, sizeInBytes = sizeInBytes, version = base.version)) + val provider = AMTCheckpointProvider.fromCheckpoint( + deltaLog, checkpoint, manifestCommitVersion = checkpoint.version) + provider.loadActionsForStateReconstruction(spark, deltaLog) + .getOrElse(fail("AMT provider must contribute reconstructed actions.")) + .where("add is not null") + .select(col("add.path"), col("add.baseRowId"), col("add.defaultRowCommitVersion")) + .as[(String, Option[Long], Option[Long])] + .collect() + .map { case (path, baseRowId, commitVersion) => + path -> ReconstructedFile(path, baseRowId, commitVersion) + } + .toMap + } + } + + /** + * Creates an AMT table with a real emitted tree, then hands `body` the fixture it + * uses to replace that tree with a hand-built one. + */ + private def withSyntheticTree(tableName: String)(body: SyntheticTree => Unit): Unit = { + withTable(tableName) { + createAMTTable(tableName, checkpointInterval = 2) + sql(s"INSERT INTO $tableName VALUES (1)") + // Emits a real AMT checkpoint action, whose non-content state the synthetic trees reuse. + sql(s"INSERT INTO $tableName VALUES (2)") + val deltaLog = deltaLogForName(tableName) + val checkpointAction = amtProvider(deltaLog.update()) + .getOrElse(fail("expected an AMT-backed checkpoint provider")) + .checkpointAction + body(SyntheticTree(deltaLog, checkpointAction)) + } + } + + /** Tracking with nothing assigned, as a writer records it before the commit version is known. */ + private def unassignedTracking(status: Int = Tracking.Status.Added): Tracking = Tracking( + status = status, + snapshot_id = None, + dv_snapshot_id = None, + sequence_number = None, + file_sequence_number = None, + first_row_id = None, + deleted_positions = None, + replaced_positions = None) + + /** + * Tracking for a root `DATA_MANIFEST` pointer, which is written at commit time and so carries + * the explicit values its leaf's entries inherit. + */ + private def pointerTracking( + sequenceNumber: Option[Long] = Some(inheritedFileSequenceNumber), + fileSequenceNumber: Option[Long] = Some(inheritedFileSequenceNumber), + firstRowId: Option[Long] = Some(1000L)): Tracking = + unassignedTracking().copy( + sequence_number = sequenceNumber, + file_sequence_number = fileSequenceNumber, + first_row_id = firstRowId) + + /** A leaf DATA entry for a data file of `recordCount` physical rows. */ + private def dataEntry( + path: String, + recordCount: Long, + tracking: Tracking = unassignedTracking()): DataEntry = DataEntry( + location = path, + file_format = AMTSingleAction.FileFormatParquet, + tracking = tracking, + record_count = recordCount, + file_size_in_bytes = 128L) + + /** A root pointer to `leaf`, optionally masking some of its positions with an MDV. */ + private def leafPointer( + leaf: WrittenLeaf, + tracking: Tracking = pointerTracking(), + mdvPositions: Seq[Long] = Seq.empty): DataManifestEntry = { + val mdv = Some(mdvPositions).filter(_.nonEmpty) + DataManifestEntry( + location = leaf.location, + file_format = AMTSingleAction.FileFormatParquet, + tracking = tracking, + record_count = leaf.entries.size.toLong, + file_size_in_bytes = leaf.sizeInBytes, + manifest_info = ManifestInfo( + added_files_count = leaf.entries.size, + existing_files_count = 0, + deleted_files_count = 0, + replaced_files_count = 0, + modified_files_count = 0, + added_rows_count = leaf.entries.map(_.record_count).sum, + existing_rows_count = 0L, + deleted_rows_count = 0L, + replaced_rows_count = 0L, + modified_rows_count = 0L, + min_sequence_number = 0L, + dv = mdv.map(positions => + AMTUtils.serializeMdv(ManifestBitmap.fromPositions(positions.map(_.toInt)))), + dv_cardinality = mdv.map(_.size.toLong))) + } + + /** The messages of `error` and of every exception it wraps. */ + private def causeMessages(error: Throwable): String = + Iterator.iterate(Option(error))(_.flatMap(e => Option(e.getCause))) + .takeWhile(_.isDefined) + .flatten + .map(e => String.valueOf(e.getMessage)) + .mkString("\n") + + test("leaf entries inherit the file sequence number and row ids of their root pointer") { + withSyntheticTree("amt_inherit_all") { tree => + val leaf = tree.writeLeaf(Seq( + dataEntry("data/a.parquet", recordCount = 10L), + dataEntry("data/b.parquet", recordCount = 20L), + dataEntry("data/c.parquet", recordCount = 30L))) + val files = tree.reconstruct(Seq(leafPointer(leaf).wrap)) + + assert(files.keySet == Set("data/a.parquet", "data/b.parquet", "data/c.parquet")) + assert(files.values.forall(_.defaultRowCommitVersion.contains(inheritedFileSequenceNumber)), + s"every entry must inherit the pointer's file sequence number; got ${files.values}") + // Row ids run from the pointer's first_row_id in the leaf's physical order, each file + // starting where the previous one's records ended. + assert(files("data/a.parquet").baseRowId.contains(1000L)) + assert(files("data/b.parquet").baseRowId.contains(1010L)) + assert(files("data/c.parquet").baseRowId.contains(1030L)) + } + } + + test("a materialized first_row_id is kept and does not advance the ones after it") { + withSyntheticTree("amt_inherit_mixed") { tree => + val materialized = unassignedTracking().copy(first_row_id = Some(5000L)) + val leaf = tree.writeLeaf(Seq( + dataEntry("data/a.parquet", recordCount = 10L), + dataEntry("data/b.parquet", recordCount = 20L, tracking = materialized), + dataEntry("data/c.parquet", recordCount = 30L))) + val files = tree.reconstruct(Seq(leafPointer(leaf).wrap)) + + assert(files("data/a.parquet").baseRowId.contains(1000L)) + assert(files("data/b.parquet").baseRowId.contains(5000L)) + // Only the entries that inherit a row id contribute to the running offset, so `c` follows + // `a` rather than `b`. + assert(files("data/c.parquet").baseRowId.contains(1010L)) + } + } + + test("an MDV-masked entry is skipped but still advances the row ids after it") { + withSyntheticTree("amt_inherit_mdv") { tree => + val leaf = tree.writeLeaf(Seq( + dataEntry("data/a.parquet", recordCount = 10L), + dataEntry("data/b.parquet", recordCount = 20L), + dataEntry("data/c.parquet", recordCount = 30L))) + val files = tree.reconstruct(Seq(leafPointer(leaf, mdvPositions = Seq(1L)).wrap)) + + assert(files.keySet == Set("data/a.parquet", "data/c.parquet"), + "the MDV-masked entry must not appear in the reconstructed live set") + assert(files("data/a.parquet").baseRowId.contains(1000L)) + // Masking `b` must not pull `c`'s row id down to 1010. + assert(files("data/c.parquet").baseRowId.contains(1030L)) + } + } + + test("a non-live entry with a materialized row id does not consume the inherited range") { + withSyntheticTree("amt_inherit_dead_entry") { tree => + val deletedTracking = unassignedTracking(Tracking.Status.Deleted).copy( + sequence_number = Some(9L), + file_sequence_number = Some(9L), + first_row_id = Some(5000L)) + val leaf = tree.writeLeaf(Seq( + dataEntry("data/a.parquet", recordCount = 10L), + dataEntry("data/b.parquet", recordCount = 20L, + tracking = deletedTracking), + dataEntry("data/c.parquet", recordCount = 30L))) + val files = tree.reconstruct(Seq(leafPointer(leaf).wrap)) + + assert(files.keySet == Set("data/a.parquet", "data/c.parquet")) + assert(files("data/a.parquet").baseRowId.contains(1000L)) + assert(files("data/c.parquet").baseRowId.contains(1010L)) + } + } + + test("manifest filters stay local to their leaf while masked entries advance inheritance") { + withSyntheticTree("amt_inherit_mdv_two_leaves") { tree => + val first = tree.writeLeaf(Seq( + dataEntry("data/a.parquet", recordCount = 10L), + dataEntry("data/b.parquet", recordCount = 20L))) + val second = tree.writeLeaf(Seq( + dataEntry("data/c.parquet", recordCount = 5L), + dataEntry("data/d.parquet", recordCount = 7L))) + val files = tree.reconstruct(Seq( + leafPointer( + first, + pointerTracking(firstRowId = Some(1000L)), + mdvPositions = Seq(0L)).wrap, + leafPointer(second, pointerTracking(firstRowId = Some(9000L))).wrap)) + + assert(files.keySet == Set("data/b.parquet", "data/c.parquet", "data/d.parquet")) + assert(files("data/b.parquet").baseRowId.contains(1010L)) + assert(files("data/c.parquet").baseRowId.contains(9000L)) + assert(files("data/d.parquet").baseRowId.contains(9005L)) + } + } + + test("each leaf inherits from its own root pointer") { + withSyntheticTree("amt_inherit_two_leaves") { tree => + val first = tree.writeLeaf(Seq( + dataEntry("data/a.parquet", recordCount = 10L), + dataEntry("data/b.parquet", recordCount = 20L))) + val second = tree.writeLeaf(Seq( + dataEntry("data/c.parquet", recordCount = 5L), + dataEntry("data/d.parquet", recordCount = 7L))) + val files = tree.reconstruct(Seq( + leafPointer(first).wrap, + leafPointer(second, pointerTracking( + sequenceNumber = Some(99L), + fileSequenceNumber = Some(99L), firstRowId = Some(9000L))).wrap)) + + assert(files("data/b.parquet").defaultRowCommitVersion.contains(inheritedFileSequenceNumber)) + assert(files("data/d.parquet").defaultRowCommitVersion.contains(99L)) + assert(files("data/a.parquet").baseRowId.contains(1000L)) + assert(files("data/b.parquet").baseRowId.contains(1010L)) + assert(files("data/c.parquet").baseRowId.contains(9000L)) + assert(files("data/d.parquet").baseRowId.contains(9005L)) + } + } + + test("a root pointer that assigns a file sequence number but no first_row_id is rejected") { + withSyntheticTree("amt_inherit_no_row_ids") { tree => + val leaf = tree.writeLeaf(Seq( + dataEntry("data/a.parquet", recordCount = 10L), + dataEntry("data/b.parquet", recordCount = 20L))) + val error = intercept[Exception] { + tree.reconstruct(Seq(leafPointer(leaf, pointerTracking(firstRowId = None)).wrap)) + } + assert(causeMessages(error).contains("tracking.first_row_id must not be null")) + } + } + + test("a root pointer that declares no inheritable tracking is rejected") { + withSyntheticTree("amt_inherit_nothing") { tree => + val leaf = tree.writeLeaf(Seq( + dataEntry("data/a.parquet", recordCount = 10L), + dataEntry("data/b.parquet", recordCount = 20L))) + val error = intercept[Exception] { + tree.reconstruct(Seq(leafPointer(leaf, unassignedTracking()).wrap)) + } + assert(causeMessages(error).contains("tracking.sequence_number must not be null")) + } + } + + test("root-resident entries are read verbatim") { + withSyntheticTree("amt_inherit_root_entries") { tree => + val leaf = tree.writeLeaf(Seq(dataEntry("data/leaf.parquet", recordCount = 10L))) + // A root DATA entry has nothing above it to inherit from, so it keeps its own tracking even + // while a sibling pointer hands values down to its leaf. + val files = tree.reconstruct(Seq( + dataEntry("data/inline.parquet", recordCount = 4L).wrap, + leafPointer(leaf).wrap)) + + assert(files("data/inline.parquet").defaultRowCommitVersion.isEmpty) + assert(files("data/inline.parquet").baseRowId.isEmpty) + assert(files("data/leaf.parquet").defaultRowCommitVersion + .contains(inheritedFileSequenceNumber)) + assert(files("data/leaf.parquet").baseRowId.contains(1000L)) + } + } + + test("a live entry that is not ADDED and has no file sequence number is rejected") { + withSyntheticTree("amt_inherit_malformed") { tree => + // Only an ADDED entry may leave its file sequence number to inheritance. An EXISTING entry + // predates the tree, so there is no value it could correctly inherit. + val leaf = tree.writeLeaf(Seq( + dataEntry("data/a.parquet", recordCount = 10L, + tracking = unassignedTracking(Tracking.Status.Existing).copy( + sequence_number = Some(7L), + first_row_id = Some(9L))))) + val error = intercept[Exception] { + tree.reconstruct(Seq(leafPointer(leaf).wrap)) + } + val messages = causeMessages(error) + assert(messages.contains("tracking.file_sequence_number is null"), messages) + assert(messages.contains("EXISTING"), messages) + } + } + + test("a live entry that is not ADDED and has no first_row_id is rejected") { + withSyntheticTree("amt_inherit_malformed_row_id") { tree => + val leaf = tree.writeLeaf(Seq( + dataEntry("data/a.parquet", recordCount = 10L, + tracking = unassignedTracking(Tracking.Status.Existing) + .copy( + sequence_number = Some(7L), + file_sequence_number = Some(8L))))) + val error = intercept[Exception] { + tree.reconstruct(Seq(leafPointer(leaf).wrap)) + } + val messages = causeMessages(error) + assert(messages.contains("tracking.first_row_id is null"), messages) + assert(messages.contains("EXISTING"), messages) + } + } + + test("a materialized file_sequence_number on a leaf entry is read without inheriting") { + Tracking.Status.liveEntryStatuses.foreach { status => + withSyntheticTree(s"amt_materialized_leaf_${Tracking.Status.nameOf(status)}") { tree => + val materialized = 77L + val materializedFirstRowId = + if (status == Tracking.Status.Added) None else Some(1000L) + val materializedSequenceNumber = + if (status == Tracking.Status.Added) None else Some(76L) + val leaf = tree.writeLeaf(Seq( + dataEntry("data/a.parquet", recordCount = 10L, + tracking = unassignedTracking(status).copy( + sequence_number = materializedSequenceNumber, + file_sequence_number = Some(materialized), + first_row_id = materializedFirstRowId)))) + val files = tree.reconstruct(Seq( + leafPointer(leaf).wrap)) + assert(files("data/a.parquet").defaultRowCommitVersion.contains(materialized), + s"${Tracking.Status.nameOf(status)} must keep its materialized value") + } + } + } + + test("a materialized file_sequence_number on a root-resident entry is read without inheriting") { + Tracking.Status.liveEntryStatuses.foreach { status => + withSyntheticTree(s"amt_materialized_root_${Tracking.Status.nameOf(status)}") { tree => + val materialized = 55L + val files = tree.reconstruct(Seq( + dataEntry("data/inline.parquet", recordCount = 4L, + tracking = unassignedTracking(status).copy(file_sequence_number = Some(materialized))) + .wrap)) + assert(files("data/inline.parquet").defaultRowCommitVersion.contains(materialized), + s"${Tracking.Status.nameOf(status)} must keep its materialized value") + } + } + } +} diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTInheritanceSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTInheritanceSuite.scala new file mode 100644 index 00000000000..579bb6e0262 --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTInheritanceSuite.scala @@ -0,0 +1,247 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.amt + +import org.apache.spark.SparkFunSuite + +/** + * Unit tests for the Iceberg V4 tracking inheritance rules. + */ +class AMTInheritanceSuite extends SparkFunSuite { + + /** The statuses an entry can carry that are not `ADDED`. */ + private val nonAddedStatuses: Seq[Int] = (Tracking.Status.all - Tracking.Status.Added).toSeq + + /** A root `DATA_MANIFEST` parent that declares a file sequence number and a starting row id. */ + private val fullParent = InheritableTracking( + sequence_number = 42L, + file_sequence_number = 42L, + first_row_id = 1000L) + + private def childTracking( + status: Int = Tracking.Status.Added, + snapshotId: Option[Long] = None, + dvSnapshotId: Option[Long] = None, + sequenceNumber: Option[Long] = None, + fileSequenceNumber: Option[Long] = None, + firstRowId: Option[Long] = None, + deletedPositions: Option[Array[Byte]] = None, + replacedPositions: Option[Array[Byte]] = None): Tracking = Tracking( + status = status, + snapshot_id = snapshotId, + dv_snapshot_id = dvSnapshotId, + sequence_number = sequenceNumber, + file_sequence_number = fileSequenceNumber, + first_row_id = firstRowId, + deleted_positions = deletedPositions, + replaced_positions = replacedPositions) + + private def resolve( + child: Tracking, + parent: InheritableTracking = fullParent, + prefixSumRecordCountForNullFirstRowId: Long = 0L): Tracking = + Tracking.resolve( + child, parent, prefixSumRecordCountForNullFirstRowId, + childEntryLocationForLogging = "data/part-0.parquet") + + /** A root `DATA_MANIFEST` entry carrying the given tracking. */ + private def parentPointer(tracking: Tracking): DataManifestEntry = DataManifestEntry( + location = "_delta_log/_amt/leaf-0.parquet", + file_format = AMTSingleAction.FileFormatParquet, + tracking = tracking, + record_count = 1L, + file_size_in_bytes = 1L, + manifest_info = ManifestInfo( + added_files_count = 1, + existing_files_count = 0, + deleted_files_count = 0, + replaced_files_count = 0, + modified_files_count = 0, + added_rows_count = 1L, + existing_rows_count = 0L, + deleted_rows_count = 0L, + replaced_rows_count = 0L, + modified_rows_count = 0L, + min_sequence_number = 0L, + dv = None, + dv_cardinality = None)) + + test("an ADDED entry with null tracking inherits every inheritable field") { + val resolved = resolve(childTracking()) + assert(resolved.file_sequence_number.contains(42L)) + assert(resolved.sequence_number.contains(42L)) + assert(resolved.first_row_id.contains(1000L)) + } + + test("materialized child values are kept in preference to the parent's") { + val resolved = resolve(childTracking( + sequenceNumber = Some(2L), + fileSequenceNumber = Some(3L), + firstRowId = Some(4L)), + prefixSumRecordCountForNullFirstRowId = 500L) + assert(resolved.sequence_number.contains(2L)) + assert(resolved.file_sequence_number.contains(3L)) + // The prefix sum applies only to an inherited row id, not to a materialized one. + assert(resolved.first_row_id.contains(4L)) + } + + test("first_row_id is inherited only by ADDED entries") { + val added = resolve(childTracking(status = Tracking.Status.Added)) + assert(added.first_row_id.contains(1000L)) + nonAddedStatuses.foreach { status => + val resolved = resolve(childTracking( + status = status, + sequenceNumber = Some(7L), + fileSequenceNumber = Some(9L), + firstRowId = Some(8L))) + assert(resolved.first_row_id.contains(8L)) + } + } + + test("an inherited first_row_id is the parent's value plus the prefix sum") { + val resolved = resolve(childTracking(), prefixSumRecordCountForNullFirstRowId = 250L) + assert(resolved.first_row_id.contains(1250L)) + } + + test("a parent that assigns a file sequence number but no row id is rejected") { + val parent = parentPointer(childTracking( + sequenceNumber = Some(42L), + fileSequenceNumber = Some(42L))) + val error = intercept[IllegalStateException](InheritableTracking(parent)) + assert(error.getMessage.contains("tracking.first_row_id must not be null")) + } + + test("a parent that assigns row ids but no file sequence number is rejected") { + val parent = parentPointer(childTracking( + sequenceNumber = Some(42L), + firstRowId = Some(1000L))) + val error = intercept[IllegalStateException](InheritableTracking(parent)) + assert(error.getMessage.contains("tracking.file_sequence_number must not be null")) + } + + test("a parent that assigns file sequence and row ID but no sequence number is rejected") { + val parent = parentPointer(childTracking( + fileSequenceNumber = Some(42L), + firstRowId = Some(1000L))) + val error = intercept[IllegalStateException](InheritableTracking(parent)) + assert(error.getMessage.contains("tracking.sequence_number must not be null")) + } + + test("sequence_number is inherited only by ADDED entries") { + val added = resolve(childTracking(status = Tracking.Status.Added)) + assert(added.sequence_number.contains(42L)) + nonAddedStatuses.foreach { status => + val resolved = resolve(childTracking( + status = status, + sequenceNumber = Some(7L), + fileSequenceNumber = Some(8L), + firstRowId = Some(9L))) + assert(resolved.sequence_number.contains(7L)) + } + } + + test("file_sequence_number is inherited only by ADDED entries") { + val added = resolve(childTracking(status = Tracking.Status.Added)) + assert(added.file_sequence_number.contains(42L)) + nonAddedStatuses.foreach { status => + val resolved = resolve(childTracking( + status = status, + sequenceNumber = Some(7L), + fileSequenceNumber = Some(8L), + firstRowId = Some(9L))) + assert(resolved.file_sequence_number.contains(8L)) + } + } + + test("status and the CDF position bitmaps pass through untouched") { + val deleted = Array[Byte](1, 2, 3) + val replaced = Array[Byte](4, 5) + val resolved = resolve(childTracking( + status = Tracking.Status.Added, + deletedPositions = Some(deleted), + replacedPositions = Some(replaced))) + assert(resolved.status == Tracking.Status.Added) + assert(resolved.deleted_positions.contains(deleted)) + assert(resolved.replaced_positions.contains(replaced)) + } + + test("a parent that declares no inheritable tracking is rejected") { + val error = intercept[IllegalStateException] { + InheritableTracking(parentPointer(childTracking())) + } + assert(error.getMessage.contains("tracking.sequence_number must not be null")) + } + + test("a null file_sequence_number on a non-ADDED entry is rejected") { + nonAddedStatuses.foreach { status => + val error = intercept[IllegalStateException] { + resolve(childTracking( + status = status, + sequenceNumber = Some(7L), + firstRowId = Some(9L))) + } + assert(error.getMessage.contains("tracking.file_sequence_number is null")) + assert(error.getMessage.contains(Tracking.Status.nameOf(status))) + assert(error.getMessage.contains("data/part-0.parquet")) + } + } + + test("a null first_row_id on a non-ADDED entry is rejected") { + nonAddedStatuses.foreach { status => + val error = intercept[IllegalStateException] { + resolve(childTracking( + status = status, + sequenceNumber = Some(7L), + fileSequenceNumber = Some(8L))) + } + assert(error.getMessage.contains("tracking.first_row_id is null")) + assert(error.getMessage.contains(Tracking.Status.nameOf(status))) + assert(error.getMessage.contains("data/part-0.parquet")) + } + } + + test("a null sequence_number on a non-ADDED entry is rejected") { + nonAddedStatuses.foreach { status => + val error = intercept[IllegalStateException] { + resolve(childTracking( + status = status, + fileSequenceNumber = Some(8L), + firstRowId = Some(9L))) + } + assert(error.getMessage.contains("tracking.sequence_number is null")) + assert(error.getMessage.contains(Tracking.Status.nameOf(status))) + assert(error.getMessage.contains("data/part-0.parquet")) + } + } + + test("InheritableTracking projects only the inheritable fields of a parent") { + val parent = parentPointer(Tracking( + status = Tracking.Status.Existing, + snapshot_id = Some(7L), + dv_snapshot_id = Some(8L), + sequence_number = Some(42L), + file_sequence_number = Some(43L), + first_row_id = Some(1000L), + deleted_positions = Some(Array[Byte](1)), + replaced_positions = Some(Array[Byte](2)))) + assert(InheritableTracking(parent) == + InheritableTracking( + sequence_number = 42L, + file_sequence_number = 43L, + first_row_id = 1000L)) + } +} diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTPartitionValuesSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTPartitionValuesSuite.scala new file mode 100644 index 00000000000..d362ad374d1 --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTPartitionValuesSuite.scala @@ -0,0 +1,164 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.amt + +import org.apache.spark.sql.delta.actions.AddFile +import org.apache.hadoop.fs.Path + +import org.apache.spark.sql.DataFrame +import org.apache.spark.sql.functions.col +import org.apache.spark.sql.types.{BinaryType, BooleanType, ByteType, DataType, DateType, DecimalType, DoubleType, FloatType, IntegerType, LongType, MapType, ShortType, StringType, StructField, StructType, TimestampNTZType, TimestampType} + +/** + * Tests for [[AMTPartitionValues]], the converter between Delta's physical-name partition string + * map and the typed Iceberg partition struct a manifest persists. Covers the conversion in + * isolation, on hand-built entries, and end to end against a real checkpoint's leaves. + */ +class AMTPartitionValuesSuite extends AMTCheckpointTestBase { + + import testImplicits._ + + /** A stand-in table root; the entries here carry no DV, so it is only a path placeholder. */ + private val tableRoot = new Path("file:/tmp/amt-test-table") + + /** A minimal leaf tracking envelope (ADDED status). */ + private def addedTracking: Tracking = Tracking( + status = Tracking.Status.Added, + snapshot_id = None, + dv_snapshot_id = None, + sequence_number = None, + file_sequence_number = None, + first_row_id = None, + deleted_positions = None, + replaced_positions = None) + + private def sampleAddFile: AddFile = AddFile( + path = "part-00000.parquet", + partitionValues = Map("p" -> "1"), + size = 1024L, + modificationTime = 100L, + dataChange = true, + stats = """{"numRecords":42}""") + + /** A DataFrame of one wrapped DATA entry carrying `partitionValues`. */ + private def entryWith(partitionValues: Map[String, String]) = + Seq(sampleAddFile.copy(partitionValues = partitionValues)) + .map(DataEntry.fromAddFile(_, addedTracking, tableRoot).wrap).toDS().toDF() + + /** + * Asserts the persisted `partition` field for column `name` has exactly the `expected` Iceberg + * type and the string form of its value. + */ + private def assertPartitionField( + persisted: DataFrame, + name: String, + expected: (DataType, String)): Unit = { + val field = persisted.schema("partition").dataType.asInstanceOf[StructType](name) + val row = persisted.select(s"partition.$name").head() + val actual = (field.dataType, String.valueOf(row.get(0))) + assert(actual == expected, s"partition.$name\n expected=$expected\n actual=$actual") + } + + test("forWrite persists partition values as the typed struct, keyed by logical name") { + withAllTypesTable("amt_partition_write") { deltaLog => + appendRowsAsSeparateFiles( + "amt_partition_write", numFiles = 1, columnExprs = allTypesTableColumnExprs) + val snapshot = deltaLog.update() + val partitionSchema = snapshot.metadata.partitionSchema + val add = liveAddFiles(snapshot).head + val entry = DataEntry.fromAddFile( + add, AMTWriteHelper.addedTrackingForDataEntry(), deltaLog.dataPath) + val input = Seq(entry.wrap).toDS().toDF() + val persisted = AMTPartitionValues.forWrite(input, partitionSchema) + + // The struct keeps the partition columns' *logical* names, though the source map is keyed by + // physical name -- an Iceberg reader resolves them by field id, so the name is informational. + val partition = persisted.schema("partition").dataType.asInstanceOf[StructType] + assert(partition.fieldNames.toSeq == partitionableTestColumns.map("p_" + _.name)) + + assertPartitionField(persisted, "p_int", (IntegerType, "0")) + assertPartitionField(persisted, "p_long", (LongType, "0")) + assertPartitionField(persisted, "p_short", (ShortType, "0")) + assertPartitionField(persisted, "p_byte", (ByteType, "0")) + assertPartitionField(persisted, "p_str", (StringType, "row0")) + assertPartitionField(persisted, "p_date", (DateType, "2026-07-25")) + assertPartitionField(persisted, "p_ts", (TimestampType, "2026-07-25 01:02:03.456")) + assertPartitionField(persisted, "p_ts_ntz", (TimestampNTZType, "2026-07-25T01:02:03.456")) + assertPartitionField(persisted, "p_dec", (DecimalType(9, 3), "0.000")) + assertPartitionField(persisted, "p_bool", (BooleanType, "false")) + assertPartitionField(persisted, "p_float", (FloatType, "0.0")) + assertPartitionField(persisted, "p_double", (DoubleType, "0.0")) + // Binary's string form is not stable, so assert only that it keeps its type. + assert(partition("p_binary").dataType == BinaryType, + s"p_binary should stay binary: ${partition("p_binary")}") + } + } + + test("forRead reproduces the partition values the delta log holds, for every type") { + withAllTypesTable("amt_partition_roundtrip") { deltaLog => + appendRowsAsSeparateFiles( + "amt_partition_roundtrip", numFiles = leafPackedFiles, + columnExprs = allTypesTableColumnExprs) + commitCheckpoint(deltaLog, incremental = false) + val snapshot = deltaLog.update() + val provider = amtProvider(snapshot).getOrElse(fail("expected AMTCheckpointProvider")) + val partitionSchema = snapshot.metadata.partitionSchema + val leaves = provider.liveLeafManifestAbsolutePaths.map(_.toString) + assert(leaves.nonEmpty, "Expected at least one leaf manifest.") + + // Read the commit json directly rather than going through the snapshot: an AMT snapshot + // reconstructs its AddFiles through `forRead`, so comparing against it would compare + // `forRead` with itself and hold even if the conversion were wrong. + val loggedSchema = StructType(Seq(StructField("add", StructType(Seq( + StructField("partitionValues", MapType(StringType, StringType))))))) + val logged = spark.read + .schema(loggedSchema) + .json(new Path(deltaLog.logPath, "*.json").toString) + .where(col("add").isNotNull) + .select(col("add.partitionValues")) + .collect() + .map(_.getMap[String, String](0).toMap) + .toSet + assert(logged.size == leafPackedFiles, + s"Expected one logged add per file, got ${logged.size}.") + + // forRead reconstructs the physical-name partition-value map from the leaves' DATA entries. + val reconstructed = withManifestDataEntries(leaves) { entries => + AMTPartitionValues.forRead(entries, partitionSchema) + .select(col("partition")) + .collect() + .map(_.getMap[String, String](0).toMap) + .toSet + } + assert(reconstructed == logged, + s"forRead did not reproduce the logged partition values.\n" + + s" logged=$logged\n reconstructed=$reconstructed") + } + } + + test("partition adapter rejects an invalid non-null typed value") { + val schema = StructType(Seq(StructField("p", IntegerType))) + val input = entryWith(Map("p" -> "not-an-int")) + + // `forWrite` casts with ANSI on, so an unparseable value fails the write rather than silently + // persisting a null partition value. + val error = intercept[Exception] { + AMTPartitionValues.forWrite(input, schema).collect() + } + assert(error.toString.contains("CAST_INVALID_INPUT")) + } +} diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTSingleActionSerializerSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTSingleActionSerializerSuite.scala index 958eb0ebdb0..c3b88a2378e 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTSingleActionSerializerSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTSingleActionSerializerSuite.scala @@ -18,12 +18,15 @@ package org.apache.spark.sql.delta.amt import java.util.UUID -import org.apache.spark.sql.delta.actions.{AddFile, DeletionVectorDescriptor} +import org.apache.spark.sql.delta.actions.{Action, AddFile, DeletionVectorDescriptor, InMemoryLogReplay} import org.apache.spark.sql.delta.storage.dv.DeletionVectorStore import org.apache.hadoop.fs.Path import org.apache.spark.sql.QueryTest +import org.apache.spark.sql.catalyst.expressions.GenericInternalRow +import org.apache.spark.sql.catalyst.util.GenericArrayData import org.apache.spark.sql.test.SharedSparkSession +import org.apache.spark.sql.types.{StringType, StructField, StructType} /** * Shape, builder, invariant, and parquet round-trip tests for the AMT @@ -74,10 +77,12 @@ class AMTSingleActionSerializerSuite extends QueryTest with SharedSparkSession { existing_files_count = 0, deleted_files_count = 0, replaced_files_count = 0, + modified_files_count = 5, added_rows_count = 42L, existing_rows_count = 0L, deleted_rows_count = 0L, replaced_rows_count = 0L, + modified_rows_count = 7L, min_sequence_number = 3L, dv = None, dv_cardinality = None) @@ -87,7 +92,7 @@ class AMTSingleActionSerializerSuite extends QueryTest with SharedSparkSession { "content_type", "format_version", "location", "file_format", "tracking", "deletion_vector", "spec_id", "partition", "sort_order_id", "record_count", "file_size_in_bytes", "content_stats", "manifest_info", "key_metadata", - "split_offsets")) + "split_offsets", "tags")) } test("closed constant sets match Iceberg V4 integer codes") { @@ -103,13 +108,62 @@ class AMTSingleActionSerializerSuite extends QueryTest with SharedSparkSession { val add = sampleAddFile val entry = AMTSingleAction.fromAddFile(add, addedTracking, tableRoot) assert(entry.content_type == AMTSingleAction.ContentType.Type.Data) - assert(entry.location == add.path) + assert(entry.location == "part-00000.parquet") assert(entry.record_count == add.numPhysicalRecords.get) assert(entry.file_size_in_bytes == add.size) assert(entry.manifest_info.isEmpty) assert(entry.tracking == addedTracking) } + test("fromAddFile decodes Delta paths into raw DATA locations") { + val add = sampleAddFile.copy(path = "dir/part%20with%25percent.parquet") + val entry = AMTSingleAction.fromAddFile(add, addedTracking, tableRoot) + assert(entry.location == "dir/part with%percent.parquet") + } + + test("toAddFile encodes raw DATA locations into Delta paths") { + val add = DataEntry( + location = "dir/part with%percent.parquet", + file_format = AMTSingleAction.FileFormatParquet, + tracking = addedTracking, + record_count = 10L, + file_size_in_bytes = 100L).toAddFile(tableRoot) + assert(add.path == "dir/part%20with%25percent.parquet") + } + + test("fromAddFile relativizes absolute in-table Delta paths") { + val deltaPath = "file:/tmp/amt-test-table/dir/part%20with%20space.parquet" + val entry = AMTSingleAction.fromAddFile( + sampleAddFile.copy(path = deltaPath), + addedTracking, + tableRoot + ) + assert(entry.location == "dir/part with space.parquet") + assert(entry.unwrap.asInstanceOf[DataEntry].toAddFile(tableRoot).path == + "dir/part%20with%20space.parquet") + } + + test("toAddFile keeps a relative in-table location relative") { + val add = DataEntry( + location = "dir/part with space.parquet", + file_format = AMTSingleAction.FileFormatParquet, + tracking = addedTracking, + record_count = 10L, + file_size_in_bytes = 100L).toAddFile(tableRoot) + assert(add.path == "dir/part%20with%20space.parquet") + } + + test("DATA location conversion preserves out-of-root absolute Delta paths") { + val deltaPath = "file:/tmp/other-table/dir/part%20with%20space.parquet" + val entry = AMTSingleAction.fromAddFile( + sampleAddFile.copy(path = deltaPath), + addedTracking, + tableRoot + ) + assert(entry.location == "file:/tmp/other-table/dir/part with space.parquet") + assert(entry.unwrap.asInstanceOf[DataEntry].toAddFile(tableRoot).path == deltaPath) + } + test("DataManifestEntry wraps to a DATA_MANIFEST root entry") { val info = sampleManifestInfo val entry = DataManifestEntry( @@ -134,6 +188,7 @@ class AMTSingleActionSerializerSuite extends QueryTest with SharedSparkSession { deletion_vector: Option[DeletionVector] = None, sort_order_id: Option[Int] = None, manifest_info: Option[ManifestInfo] = None, + tags: Option[Map[String, String]] = None, tracking: Tracking = addedTracking): AMTSingleAction = AMTSingleAction( content_type = content_type, format_version = AMTSingleAction.FormatVersionV4, @@ -142,14 +197,15 @@ class AMTSingleActionSerializerSuite extends QueryTest with SharedSparkSession { tracking = tracking, deletion_vector = deletion_vector, spec_id = None, - partition = Partition(), + partition = None, sort_order_id = sort_order_id, record_count = 1L, file_size_in_bytes = 1L, content_stats = None, manifest_info = manifest_info, key_metadata = None, - split_offsets = None) + split_offsets = None, + tags = tags) private def assertRejected(substring: String)(build: => AMTSingleAction): Unit = { val ex = intercept[IllegalArgumentException](build) @@ -223,7 +279,7 @@ class AMTSingleActionSerializerSuite extends QueryTest with SharedSparkSession { test("parquet round-trip preserves every binary field") { // Binary columns are `Array[Byte]`, whose case-class `==` is reference equality, so this - // asserts the bytes structurally. Populate all five Option[Array[Byte]] fields across the + // asserts the bytes structurally. Populate all four Option[Array[Byte]] fields across the // row and its sub-structs on a single DATA_MANIFEST entry (the only kind that reaches // `manifest_info.dv`) so none is silently dropped by wrap/unwrap or the encoder. withTempDir { dir => @@ -231,7 +287,6 @@ class AMTSingleActionSerializerSuite extends QueryTest with SharedSparkSession { val deletedPos = Array[Byte](5, 6) val replacedPos = Array[Byte](7, 8, 9) val manifestDv = Array[Byte](10, 11) - val rawStats = Array[Byte](12, 13, 14) val entry = DataManifestEntry( location = "dm.parquet", file_format = AMTSingleAction.FileFormatParquet, @@ -240,7 +295,6 @@ class AMTSingleActionSerializerSuite extends QueryTest with SharedSparkSession { record_count = 1L, file_size_in_bytes = 1L, manifest_info = sampleManifestInfo.copy(dv = Some(manifestDv), dv_cardinality = Some(2L)), - content_stats = Some(ContentStats(Some(rawStats))), key_metadata = Some(keyMeta)).wrap val path = new java.io.File(dir, "binary").getCanonicalPath spark.createDataset(Seq(entry)).write.parquet(path) @@ -254,8 +308,6 @@ class AMTSingleActionSerializerSuite extends QueryTest with SharedSparkSession { "tracking.replaced_positions did not round-trip.") assert(r.manifest_info.flatMap(_.dv).exists(_.sameElements(manifestDv)), "manifest_info.dv did not round-trip.") - assert(r.content_stats.flatMap(_.raw_stats).exists(_.sameElements(rawStats)), - "content_stats.raw_stats did not round-trip.") } } @@ -304,48 +356,40 @@ class AMTSingleActionSerializerSuite extends QueryTest with SharedSparkSession { // Framing bytes that DeletionVectorStore adds around the raw bitmap on disk (length + checksum). private val dvFraming = DeletionVectorStore.getTotalSizeOfDVFieldsInFile(0) - test("DeletionVector.fromDescriptor maps a UUID-relative DV to an absolute location") { + test("u DV round trip") { val id = UUID.randomUUID() - val dv = DeletionVectorDescriptor.onDiskWithRelativePath( - id = id, sizeInBytes = 20, cardinality = 3L, offset = Some(8)) + val dv = DeletionVectorDescriptor.onDiskWithUuidRelativePath( + id = id, + randomPrefix = "test%dv%prefix-", + sizeInBytes = 20, + cardinality = 3L, + offset = Some(8)) val amtDv = DeletionVector.fromDescriptor(dv, tableRoot) - // location is the absolute path the u-DV resolves to under the table root. - assert(amtDv.location == dv.absolutePath(tableRoot).toString) + assert(amtDv.location == + s"test%dv%prefix-/${DeletionVectorDescriptor.assembleDeletionVectorFileName(id)}") + assert(amtDv.location.contains("test%dv%prefix-")) assert(amtDv.offset == 8L) assert(amtDv.cardinality == 3L) // size_in_bytes is the total on-disk size: raw bitmap plus framing. assert(amtDv.size_in_bytes == 20L + dvFraming) - } - test("DeletionVector round-trips a UUID-relative DV (matching uniqueId)") { - // Cover both a bare and a random-prefixed UUID-relative DV. - Seq("", "abc123").foreach { prefix => - val dv = DeletionVectorDescriptor.onDiskWithRelativePath( - id = UUID.randomUUID(), randomPrefix = prefix, - sizeInBytes = 34, cardinality = 5L, offset = Some(16)) - val roundTripped = DeletionVector.toDescriptor( - DeletionVector.fromDescriptor(dv, tableRoot), tableRoot) - assert(roundTripped.storageType == DeletionVectorDescriptor.UUID_DV_MARKER, - s"prefix='$prefix': expected a UUID-relative DV, got ${roundTripped.storageType}.") - // uniqueId (storageType + pathOrInlineDv + @offset) must survive the round-trip -- it is the - // (path, dv) dedup key state reconstruction relies on. - assert(roundTripped.uniqueId == dv.uniqueId, - s"prefix='$prefix': ${roundTripped.uniqueId} != ${dv.uniqueId}.") - assert(roundTripped.sizeInBytes == dv.sizeInBytes) - assert(roundTripped.cardinality == dv.cardinality) - assert(roundTripped.offset == dv.offset) - } + val roundTripped = DeletionVector.toDescriptor(amtDv, tableRoot) + assert(roundTripped.storageType == DeletionVectorDescriptor.RELATIVE_DV_MARKER) + assert(roundTripped.pathOrInlineDv.contains("test%dv%prefix-")) + assert(roundTripped.absolutePath(tableRoot) == dv.absolutePath(tableRoot)) + assert(roundTripped.normalizedTableRelativeObjectId(tableRoot) == + dv.normalizedTableRelativeObjectId(tableRoot)) } test("DeletionVector round-trips an absolute-path DV outside the table root") { - // An absolute DV whose path is not a Delta DV file under the table root stays `p`. + // An absolute DV whose path is outside the table root stays `p`. val dv = DeletionVectorDescriptor.onDiskWithAbsolutePath( path = "s3://other-bucket/dvs/custom.bin", sizeInBytes = 12, cardinality = 1L, offset = Some(4)) val roundTripped = DeletionVector.toDescriptor( DeletionVector.fromDescriptor(dv, tableRoot), tableRoot) assert(roundTripped.storageType == DeletionVectorDescriptor.PATH_DV_MARKER) - assert(roundTripped.uniqueId == dv.uniqueId) + assert(roundTripped.legacyUniqueId == dv.legacyUniqueId) assert(roundTripped.sizeInBytes == dv.sizeInBytes) assert(roundTripped.offset == dv.offset) } @@ -358,9 +402,253 @@ class AMTSingleActionSerializerSuite extends QueryTest with SharedSparkSession { } test("DeletionVector.fromDescriptor rejects an on-disk DV with no offset") { - val dv = DeletionVectorDescriptor.onDiskWithRelativePath( + val dv = DeletionVectorDescriptor.onDiskWithUuidRelativePath( id = UUID.randomUUID(), sizeInBytes = 10, cardinality = 1L, offset = None) val ex = intercept[IllegalArgumentException](DeletionVector.fromDescriptor(dv, tableRoot)) assert(ex.getMessage.contains("missing an offset")) } + + /** A DATA entry carrying the AMT-native fields that have no Delta equivalent. */ + private def entryWithPassthrough: DataEntry = DataEntry( + location = "f.parquet", + file_format = AMTSingleAction.FileFormatParquet, + tracking = addedTracking, + record_count = 10L, + file_size_in_bytes = 100L, + spec_id = Some(7), + sort_order_id = Some(3), + key_metadata = Some(Array[Byte](4, 5, 6)), + split_offsets = Some(Seq(0L, 64L, 128L))) + + test("toAddFile carries no passthrough for the default parquet/v4 entry") { + val add = DataEntry( + location = "f.parquet", file_format = AMTSingleAction.FileFormatParquet, + tracking = addedTracking, record_count = 10L, file_size_in_bytes = 100L).toAddFile(tableRoot) + assert(add.amtPassthrough.isEmpty, "a default entry must add no per-file passthrough") + } + + test("DataEntry round-trips toAddFile -> fromAddFile") { + val add = entryWithPassthrough.toAddFile(tableRoot) + assert(add.amtPassthrough.isDefined, "a genuine passthrough must be carried") + val restored = DataEntry.fromAddFile(add, addedTracking, tableRoot) + assert(restored.spec_id.contains(7)) + assert(restored.sort_order_id.contains(3)) + assert(restored.key_metadata.exists(_.sameElements(Array[Byte](4, 5, 6)))) + assert(restored.split_offsets.contains(Seq(0L, 64L, 128L))) + // file_format / format_version are not carried; they are reconstructed at the M1 defaults. + assert(restored.file_format == AMTSingleAction.FileFormatParquet) + assert(restored.format_version == AMTSingleAction.FormatVersionV4) + } + + test("fromAddFile falls back to defaults with no passthrough") { + val restored = DataEntry.fromAddFile(sampleAddFile, addedTracking, tableRoot) + assert(restored.file_format == AMTSingleAction.FileFormatParquet) + assert(restored.spec_id.isEmpty) + assert(restored.format_version == AMTSingleAction.FormatVersionV4) + } + + test("fromAddFile maps defaultRowCommitVersion to both sequence numbers") { + val entry = DataEntry.fromAddFile( + sampleAddFile.copy(defaultRowCommitVersion = Some(10L)), + addedTracking, + tableRoot) + assert(entry.tracking.sequence_number.contains(10L)) + assert(entry.tracking.file_sequence_number.contains(10L)) + } + + test("toAddFile maps file_sequence_number to defaultRowCommitVersion") { + val add = DataEntry( + location = "f.parquet", + file_format = AMTSingleAction.FileFormatParquet, + tracking = addedTracking.copy( + sequence_number = Some(20L), + file_sequence_number = Some(10L)), + record_count = 10L, + file_size_in_bytes = 100L).toAddFile(tableRoot) + assert(add.defaultRowCommitVersion.contains(10L)) + } + + private val sampleTags: Map[String, String] = + Map(AddFile.Tags.INSERTION_TIME.name -> "123", "custom" -> "value") + + test("validate rejects tags on a non-data entry") { + assertRejected("tags must be null")( + mkEntry( + content_type = AMTSingleAction.ContentType.Type.DataManifest, + manifest_info = Some(sampleManifestInfo), + tags = Some(sampleTags))) + } + + test("fromAddFile preserves AddFile tags and drops an empty map") { + val withTags = DataEntry.fromAddFile(sampleAddFile.copy(tags = sampleTags), addedTracking, + tableRoot) + assert(withTags.tags.contains(sampleTags)) + // A null or empty tag map is indistinguishable from no tags. + assert(DataEntry.fromAddFile(sampleAddFile, addedTracking, tableRoot).tags.isEmpty) + assert(DataEntry.fromAddFile(sampleAddFile.copy(tags = Map.empty[String, String]), + addedTracking, tableRoot).tags.isEmpty) + } + + test("DataEntry round-trips tags through toAddFile -> fromAddFile") { + val entry = DataEntry( + location = "f.parquet", file_format = AMTSingleAction.FileFormatParquet, + tracking = addedTracking, record_count = 10L, file_size_in_bytes = 100L, + tags = Some(sampleTags)) + val add = entry.toAddFile(tableRoot) + assert(add.tags != null && add.tags == sampleTags, "tags must be carried onto the AddFile") + val restored = DataEntry.fromAddFile(add, addedTracking, tableRoot) + assert(restored.tags.contains(sampleTags)) + } + + test("parquet round-trip preserves tags") { + withTempDir { dir => + val entry = AMTSingleAction.fromAddFile( + sampleAddFile.copy(tags = sampleTags), addedTracking, tableRoot) + val path = new java.io.File(dir, "tags").getCanonicalPath + spark.createDataset(Seq(entry)).write.parquet(path) + val read = spark.read.parquet(path).as[AMTSingleAction].collect() + assert(read.length == 1) + assert(read.head.tags.contains(sampleTags), "tags did not round-trip through parquet.") + } + } + + test("amtPassthrough round-trips through the commit JSON") { + val add = entryWithPassthrough.toAddFile(tableRoot) + assert(add.amtPassthrough.isDefined) + val json = add.json + assert(json.contains("amtPassthrough"), s"amtPassthrough must be serialized; got $json") + Seq("spec_id", "sort_order_id", "key_metadata", "split_offsets").foreach { f => + assert(json.contains(f), s"$f must be serialized; got $json") + } + + val roundTripped = Action.fromJson(json) match { + case a: AddFile => a + case other => fail(s"expected an AddFile, got $other") + } + assert(roundTripped.amtPassthrough == add.amtPassthrough) + val passthrough = roundTripped.amtPassthrough.getOrElse(fail("passthrough must round-trip")) + assert(passthrough.spec_id.contains(7)) + assert(passthrough.sort_order_id.contains(3)) + assert(passthrough.key_metadata.exists(_.sameElements(Array[Byte](4, 5, 6)))) + assert(passthrough.split_offsets.contains(Seq(0L, 64L, 128L))) + } + + test("an AddFile with no amtPassthrough keeps it out of the commit JSON") { + // Include.NON_ABSENT: non-AMT tables must not gain a new key in their commit JSON. + assert(sampleAddFile.amtPassthrough.isEmpty) + assert(!sampleAddFile.json.contains("amtPassthrough")) + } + + test("amtPassthrough survives copy and InMemoryLogReplay state reconstruction") { + val add = entryWithPassthrough.toAddFile(tableRoot).copy(path = "f.parquet") + val passthrough = add.amtPassthrough + assert(passthrough.isDefined) + // Below simulates the real write path in [[writeIncrementalMaterialization]] + assert(add.copy(dataChange = false).amtPassthrough == passthrough) + val replay = new InMemoryLogReplay( + minFileRetentionTimestamp = None, + minSetTransactionRetentionTimestamp = None, + tableRoot = tableRoot, + useDeletionVectorObjectIdentity = true) + replay.append(0, Iterator(add)) + val reconstructed = replay.allFiles + assert(reconstructed.length == 1) + // The passthrough survives replay and is still usable end-to-end. + assert(reconstructed.head.amtPassthrough == passthrough) + val restored = DataEntry.fromAddFile(reconstructed.head, addedTracking, tableRoot) + assert(restored.spec_id.contains(7)) + assert(restored.sort_order_id.contains(3)) + assert(restored.key_metadata.exists(_.sameElements(Array[Byte](4, 5, 6)))) + assert(restored.split_offsets.contains(Seq(0L, 64L, 128L))) + } + + test("amtPassthrough participates in AddFile equality") { + val base = sampleAddFile + // Distinct `key_metadata` array instances with identical content. + def createAddFile(specId: Int, keyMetadata: Array[Byte]): AddFile = base.copy( + amtPassthrough = + Some(AMTPassthrough(spec_id = Some(specId), key_metadata = Some(keyMetadata)))) + val withA = createAddFile(7, Array[Byte](1, 2, 3)) + val withB = createAddFile(7, Array[Byte](1, 2, 3)) + assert(withA == withB && withA.hashCode == withB.hashCode, + "equal-content passthrough must compare equal") + assert( + withA != createAddFile(9, Array[Byte](1, 2, 3)), + "different spec_id must compare unequal" + ) + assert( + withA != createAddFile(7, Array[Byte](1, 2, 4)), + "different key_metadata must compare unequal" + ) + // Present vs absent -> unequal. + assert(withA != base && base != withA) + } + + test("RowIndices.resolve returns None when the schema does not project amtPassthrough") { + val schema = StructType(Seq(StructField("path", StringType))) + assert(AMTPassthrough.RowIndices.resolve(schema).isEmpty) + } + + test("RowIndices.resolve and fromRow read the passthrough positionally") { + val addSchema = Action.addFileSchema + val indices = AMTPassthrough.RowIndices.resolve(addSchema) + .getOrElse(fail("the AddFile schema must project amtPassthrough")) + assert(addSchema.fieldNames(indices.structIndex) == AMTPassthrough.FIELD_NAME) + assert(indices.numFields == AMTPassthrough.STRUCT_TYPE.fields.length) + + def structRow( + specId: Any = null, + sortOrderId: Any = null, + keyMetadata: Any = null, + splitOffsets: Any = null): GenericInternalRow = { + val fields = new Array[Any](indices.numFields) + fields(indices.specId) = specId + fields(indices.sortOrderId) = sortOrderId + fields(indices.keyMetadata) = keyMetadata + fields(indices.splitOffsets) = splitOffsets + new GenericInternalRow(fields) + } + def rowWith(struct: GenericInternalRow): GenericInternalRow = { + val row = new GenericInternalRow(addSchema.fields.length) + row.update(indices.structIndex, struct) + row + } + + // A row carrying every field -> read back structurally. + val full = rowWith(structRow( + specId = 7, + sortOrderId = 3, + keyMetadata = Array[Byte](4, 5, 6), + splitOffsets = new GenericArrayData(Array[Long](0L, 64L, 128L)))) + assert(AMTPassthrough.fromRow(full, indices).contains(AMTPassthrough( + spec_id = Some(7), + sort_order_id = Some(3), + key_metadata = Some(Array[Byte](4, 5, 6)), + split_offsets = Some(Seq(0L, 64L, 128L))))) + + // A null struct -> None. + val withoutPassthrough = new GenericInternalRow(addSchema.fields.length) + assert(AMTPassthrough.fromRow(withoutPassthrough, indices).isEmpty) + + // A present struct whose fields are all null -> present but empty. + assert(AMTPassthrough.fromRow(rowWith(structRow()), indices).contains(AMTPassthrough())) + } + + test("toAddFile keeps numRecords physical for a file with a deletion vector") { + // Iceberg's record_count and Delta's numRecords are both physical counts, so neither + // fromAddFile nor toAddFile may adjust for the DV -- AddFile subtracts the cardinality itself + // when it derives numLogicalRecords. Adjusting in either place would double-count. + // AMT only supports on-disk deletion vectors. + val dv = DeletionVectorDescriptor.onDiskWithUuidRelativePath( + id = UUID.randomUUID(), sizeInBytes = 20, cardinality = 4L, offset = Some(8)) + val add = sampleAddFile.copy(stats = """{"numRecords":10}""", deletionVector = dv) + assert(add.numPhysicalRecords.contains(10L)) + assert(add.numLogicalRecords.contains(6L)) + + val entry = DataEntry.fromAddFile(add, addedTracking, tableRoot) + assert(entry.record_count == 10L, "record_count is the physical count") + val roundTripped = entry.toAddFile(tableRoot) + assert(roundTripped.numPhysicalRecords.contains(10L)) + assert(roundTripped.numLogicalRecords.contains(6L)) + } } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTSingleCommitSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTSingleCommitSuite.scala new file mode 100644 index 00000000000..6c99b2dfcac --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTSingleCommitSuite.scala @@ -0,0 +1,249 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.amt + +import org.apache.spark.sql.delta.{DeltaFileProviderUtils, DeltaLog, DeltaOperations, SingleCommit} +import org.apache.spark.sql.delta.DeltaTestUtils.createTestAddFile +import org.apache.spark.sql.delta.actions.{Action, AddFile, RemoveFile} +import org.apache.spark.sql.delta.util.DeltaCommitFileProvider + +/** + * Tests for SingleCommit related handling. + */ +class AMTSingleCommitSuite extends AMTCheckpointTestBase { + + /** + * Create an AMT-featured table with `numVersions` plain INSERT commits that all stay classic log + * commits (checkpoint interval far away, huge size threshold so no manifest is emitted); return + * its [[DeltaLog]]. + */ + private def createLogCommitTable(name: String, numVersions: Int): DeltaLog = { + createAMTTable(name, checkpointInterval = 1000) + (1 to numVersions).foreach(i => sql(s"INSERT INTO $name VALUES ($i)")) + deltaLogForName(name) + } + + /** A [[SingleCommit]] handle for exactly `version`. */ + private def commitAt(deltaLog: DeltaLog, version: Long): SingleCommit = + deltaLog.getChangesIterator(startVersion = version) + .find(_.version == version) + .getOrElse(fail(s"no commit at version $version")) + + test("getActionsIterator returns the commit's actions and matches getChanges") { + withTable("amt_getactions") { + val deltaLog = createLogCommitTable("amt_getactions", numVersions = 3) + val viaHandle = deltaLog.getChangesIterator(startVersion = 0).map { commit => + (commit.version, commit.getActionsIterator().processAndClose(_.toList)) + }.toList + val viaGetChanges = deltaLog.getChanges(startVersion = 0).map { + case (version, actions) => (version, actions.toList) + }.toList + assert(viaHandle === viaGetChanges) + } + } + + test("getActionsIterator is rewindable and replays the same actions") { + withTable("amt_rewind") { + val deltaLog = createLogCommitTable("amt_rewind", numVersions = 1) + // Version 1 is the single INSERT; open its actions and read twice via rewind(). + val iter = commitAt(deltaLog, 1L).getActionsIterator() + try { + val firstPass = iter.toList + assert(firstPass.nonEmpty) + iter.rewind() + assert(iter.toList === firstPass) + } finally { + iter.close() + } + } + } + + test("getActionsIterator returns all actions of a multi-action commit") { + withTable("amt_multi_action") { + val name = "amt_multi_action" + // v1 adds "old"; v2 packs several data actions -- three new AddFiles and a RemoveFile of + // "old" -- into one manual commit so the commit has more than one action. + val deltaLog = createLogCommitTable(name, numVersions = 0) + val oldFile = createTestAddFile(encodedPath = "old") + deltaLog.startTransaction() + .commit(Seq(oldFile), DeltaOperations.ManualUpdate) + val dataActions: Seq[Action] = Seq( + createTestAddFile(encodedPath = "a"), + createTestAddFile(encodedPath = "b"), + createTestAddFile(encodedPath = "c"), + oldFile.remove) + deltaLog.startTransaction() + .commit(dataActions, DeltaOperations.ManualUpdate) + + // Read commit 2 (the multi-action one) back through the SingleCommit API. + val commit = commitAt(deltaLog, 2L) + val actions = commit.getActionsIterator().processAndClose(_.toList) + + // It contains more than one action, and every data action we committed is present (order and + // exact identity of synthesized CommitInfo/Protocol are not asserted -- commit() adds those). + assert(actions.size > 1, s"expected a multi-action commit, got: $actions") + val addedPaths = actions.collect { case a: AddFile => a.path }.toSet + assert(addedPaths === Set("a", "b", "c"), s"missing AddFiles, got: $addedPaths") + val removedPaths = actions.collect { case r: RemoveFile => r.path }.toSet + assert(removedPaths === Set("old"), s"missing RemoveFile, got: $removedPaths") + + // The authoritative invariant: getActionsIterator returns exactly what getChanges returns for + // this same multi-action commit. + val viaGetChanges = deltaLog.getChanges(startVersion = 2).next()._2.toList + assert(actions === viaGetChanges) + } + } + + test("getChangesIterator yields one SingleCommit per commit with the right version") { + withTable("amt_versions") { + val deltaLog = createLogCommitTable("amt_versions", numVersions = 3) + // CREATE is version 0, the three INSERTs are versions 1..3. + val commits = deltaLog.getChangesIterator(startVersion = 0).toList + assert(commits.map(_.version) === (0L to 3L).toList) + + // The handle exposes the commit file's modification time (not the in-commit timestamp), so it + // matches what getChangeLogFiles reports for the same commit. + val fileModTimes = + deltaLog.getChangeLogFiles(startVersion = 0).map(_._2.getModificationTime).toList + assert(commits.map(_.fileModificationTimestamp) === fileModTimes) + } + } + + test("getChangesIterator endVersion override bounds the range (both inclusive)") { + withTable("amt_range") { + val deltaLog = createLogCommitTable("amt_range", numVersions = 5) + val commits = deltaLog.getChangesIterator( + startVersion = 1, endVersion = 3, catalogTableOpt = None, failOnDataLoss = true).toList + assert(commits.map(_.version) === List(1L, 2L, 3L)) + } + } + + private val insertTwoRows: String => Unit = { name => + sql(s"INSERT INTO $name VALUES (1)") + sql(s"INSERT INTO $name VALUES (2)") + } + + private val insertThirdRow: Option[AMTCheckpointTrigger] = + Some(name => Right(s"INSERT INTO $name VALUES (3)")) + + testAcrossAMTCheckpointScenarios( + "getLogCommitActionsIteratorUnsafe reads a log commit like getActionsIterator", + tableName = "amt_log_read")( + setup = insertTwoRows, + inlineCheckpointTriggerActionsOrSQL = insertThirdRow + ) { context => + val deltaLog = deltaLogForName(context.tableName) + val logCommits = + (0L to context.manifestCommitVersion).filter(checkpointAt(deltaLog, _).isEmpty) + assert(logCommits.nonEmpty, "expected at least one log commit before the manifest commit") + logCommits.foreach { v => + val commit = commitAt(deltaLog, v) + val viaUnsafe = commit.getLogCommitActionsIteratorUnsafe().processAndClose(_.toList) + val viaPlain = commit.getActionsIterator().processAndClose(_.toList) + assert(viaUnsafe === viaPlain, s"v$v disagreed between the unsafe and plain readers") + } + } + + testAcrossAMTCheckpointScenarios( + "getLogCommitActionsIteratorUnsafe throws on a manifest commit", + tableName = "amt_manifest_read")( + setup = insertTwoRows, + inlineCheckpointTriggerActionsOrSQL = insertThirdRow + ) { context => + val deltaLog = deltaLogForName(context.tableName) + val manifestVersion = context.manifestCommitVersion + // TODO: Restore this call once `CoordinatedCommitsUtils.commitFilesIterator` discovers + // unbackfilled manifest commits. + /* + val commit = commitAt(deltaLogForName(context.tableName), manifestVersion) + */ + val path = DeltaCommitFileProvider(context.postCheckpointSnapshot).deltaFile(manifestVersion) + val status = path.getFileSystem(deltaLog.newDeltaHadoopConf()).getFileStatus(path) + val commit = SingleCommit(deltaLog, manifestVersion, status) + val e = intercept[IllegalStateException] { + commit.getLogCommitActionsIteratorUnsafe().processAndClose(_.toList) + } + assert(e.getMessage.contains(s"commit $manifestVersion")) + assert(e.getMessage.contains("cannot be read as a log commit")) + } + + testAcrossAMTCheckpointScenarios( + "parallelReadAndParseLogCommitsAsSeqUnsafe throws if any commit is a manifest commit", + tableName = "amt_par_manifest")( + setup = insertTwoRows, + inlineCheckpointTriggerActionsOrSQL = insertThirdRow + ) { context => + val deltaLog = deltaLogForName(context.tableName) + val firstLogCommit = context.postSetupSnapshot.version + assert(checkpointAt(deltaLog, firstLogCommit).isEmpty, s"v$firstLogCommit must be a log commit") + // TODO: Restore this call once `CoordinatedCommitsUtils.commitFilesIterator` discovers + // unbackfilled manifest commits. + /* + val commits = DeltaFileProviderUtils.getCommitsInVersionRange( + spark, + deltaLog, + startVersion = firstLogCommit, + endVersion = context.manifestCommitVersion, + catalogTableOpt = None) + */ + val fileProvider = DeltaCommitFileProvider(context.postCheckpointSnapshot) + val hadoopConf = deltaLog.newDeltaHadoopConf() + val commits = (firstLogCommit to context.manifestCommitVersion).map { version => + val path = fileProvider.deltaFile(version) + val status = path.getFileSystem(hadoopConf).getFileStatus(path) + SingleCommit(deltaLog, version, status) + } + val e = intercept[Exception] { + DeltaFileProviderUtils.parallelReadAndParseLogCommitsAsSeqUnsafe(spark, commits) + } + val causes = Iterator.iterate[Throwable](e)(_.getCause).takeWhile(_ != null).toList + assert( + causes.exists { c => + c.isInstanceOf[IllegalStateException] && + c.getMessage != null && c.getMessage.contains("cannot be read as a log commit") + }, + s"expected an IllegalStateException about a manifest commit in the cause chain, got: " + + causes.map(c => s"${c.getClass.getName}: ${c.getMessage}").mkString(" -> ")) + } + + testAcrossAMTCheckpointScenarios( + "parallelReadAndParseLogCommitsAsSeqUnsafe reads a range after the manifest commit", + tableName = "amt_par_after_manifest")( + setup = insertTwoRows, + inlineCheckpointTriggerActionsOrSQL = insertThirdRow + ) { context => + val deltaLog = deltaLogForName(context.tableName) + // Three log commits land after the manifest commit, so the table looks like + // [.., log, manifest, log, log, log] and the range below starts past the manifest commit. + (4 to 6).foreach(i => sql(s"INSERT INTO ${context.tableName} VALUES ($i)")) + val firstLogCommit = context.manifestCommitVersion + 1 + val lastLogCommit = context.manifestCommitVersion + 3 + assert((firstLogCommit to lastLogCommit).forall(checkpointAt(deltaLog, _).isEmpty), + s"v$firstLogCommit..v$lastLogCommit must all be log commits") + + val commits = DeltaFileProviderUtils.getCommitsInVersionRange( + spark, + deltaLog, + startVersion = firstLogCommit, + endVersion = lastLogCommit, + catalogTableOpt = None) + // Does not throw. + val viaUnsafe = DeltaFileProviderUtils.parallelReadAndParseLogCommitsAsSeqUnsafe(spark, commits) + val viaPlain = commits.map(_.getActionsIterator().processAndClose(_.toSeq)) + assert(viaUnsafe === viaPlain) + } +} diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTSnapshotDiscoverySuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTSnapshotDiscoverySuite.scala index 8abeb3afee7..9ae0e447f00 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTSnapshotDiscoverySuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTSnapshotDiscoverySuite.scala @@ -16,18 +16,634 @@ package org.apache.spark.sql.delta.amt -import org.apache.spark.sql.delta.util.FileNames +import org.apache.spark.sql.delta.{DeltaLog, DeltaMinorCompactionTestUtils, Snapshot} +import org.apache.spark.sql.delta.actions.{Action, CommitInfo, LastManifestCommit} +import org.apache.spark.sql.delta.sources.DeltaSQLConf +import org.apache.spark.sql.delta.util.{DeltaCommitFileProvider, FileNames} +import org.apache.commons.io.IOUtils +import org.apache.hadoop.fs.{FileSystem, Path} -class AMTSnapshotDiscoverySuite extends AMTCheckpointTestBase { +import org.apache.spark.SparkConf +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.catalog.CatalogTable + +class AMTSnapshotDiscoverySuite + extends AMTCheckpointTestBase + with DeltaMinorCompactionTestUtils { + + /** Whether this suite runs with `.crc` files enabled, read from the effective conf. */ + protected def writeChecksumEnabled: Boolean = + spark.conf.get(DeltaSQLConf.DELTA_WRITE_CHECKSUM_ENABLED) + + test("[snapshot init] rejects an AMT checkpoint without deltaAtCheckpointVersionOpt") { + val name = "amt_missing_delta_at_checkpoint" + withTable(name) { + createAMTTable(name, checkpointInterval = 2) + (1 to 2).foreach(i => sql(s"INSERT INTO $name VALUES ($i)")) + val snapshot = deltaLogForName(name).update() + assert(amtProvider(snapshot).isDefined) + assert(snapshot.logSegment.deltaAtCheckpointVersionOpt.isDefined) + // Reuse the real checkpoint and remove only the required commit-file reference. + val incompleteSegment = snapshot.logSegment.copy(deltaAtCheckpointVersionOpt = None) + val error = intercept[IllegalStateException] { + new Snapshot( + path = snapshot.path, + version = snapshot.version, + logSegment = incompleteSegment, + deltaLog = snapshot.deltaLog, + checksumOpt = snapshot.checksumOpt + ) + } + assert(error.getMessage.startsWith( + "An AMT-enabled snapshot must define deltaAtCheckpointVersionOpt, got None.")) + } + } //////////////////////////// // Cold snapshot discovery //////////////////////////// + /** Loads a genuinely cold deltaLog + snapshot (cache cleared) for the given table. */ + protected def coldLoad(tableName: String): (DeltaLog, Snapshot) = { + DeltaLog.clearCache() + DeltaLog.forTableWithSnapshot(spark, new TableIdentifier(tableName)) + } + + /** Asserts that the cold snapshot is installed with the expected state. */ + private def assertColdSnapshotStates( + tableName: String, + version: Long, + amtCheckpointVersion: Option[Long], + trailingDeltas: Seq[Long], + lastManifestCommit: Option[LastManifestCommit]): Unit = { + val (_, snapshot) = coldLoad(tableName) + assert(snapshot.version == version) + assert(snapshot.logSegment.version == version) + assert(snapshot.logSegment.deltas.map(FileNames.deltaVersion) == trailingDeltas) + assert(snapshot.lastManifestCommitOpt == lastManifestCommit) + assert(amtProvider(snapshot).map(_.version) == amtCheckpointVersion) + // The AMT checkpoint is trusted once the manifest-commit reference corroborates it. With a CRC + // that reference comes from the checksum; without one it is recovered from the latest commit's + // CommitInfo, and no checksum is resolved. + if (writeChecksumEnabled) { + val checksum = snapshot.checksumOpt.getOrElse( + fail(s"v$version: a cold AMT read with CRC enabled must resolve a checksum.")) + assert(checksum.lastManifestCommit == lastManifestCommit, + s"v$version: the CRC's manifest-commit reference must match the snapshot's.") + } else { + assert(snapshot.checksumOpt.isEmpty, + s"v$version: no checksum should be resolved when CRC writes are disabled.") + } + } + + testInline("[cold init] installs the correct provider from up-to-date _last_checkpoint: inline") { + withTable("amt_cold_discovery_inline") { + val name = "amt_cold_discovery_inline" + createAMTTable(name, checkpointInterval = 2) + // Inline mode, interval 2. Inline-incremental extends an existing manifest tree, so it cannot + // bootstrap the first one: the first (full) AMT is always a deferred OPTIMIZE CHECKPOINT. + // Only once that tree exists does each subsequent business commit write its manifest inline. + // v0: CREATE (genesis, no manifest) + // v1: INSERT #1 (no full AMT yet -> cannot inline) + // v2: INSERT #2 (reaches boundary) (still cannot inline) + // v3: OPTIMIZE CHECKPOINT (state@v2) (the first, full AMT; deferred) + // v4: INSERT #3 + inline AMT (state@v4) + // v5: INSERT #4 + inline AMT (state@v5) + + // v0: table just created. + assertColdSnapshotStates( + tableName = name, + version = 0, + amtCheckpointVersion = None, + trailingDeltas = Seq(0L), + lastManifestCommit = None) + + // v1: no full AMT exists yet, so this commit cannot inline one. + sql(s"INSERT INTO $name VALUES (1)") + assert(rootFiles(tablePath(name)).isEmpty) + assert(leafFiles(tablePath(name)).isEmpty) + assertColdSnapshotStates( + tableName = name, + version = 1, + amtCheckpointVersion = None, + trailingDeltas = Seq(0L, 1L), + lastManifestCommit = None) + + // v2 reaches the interval boundary, but the first AMT can never ride inline: it lands as the + // deferred follow-up OPTIMIZE CHECKPOINT at v3, describing state as of v2. + sql(s"INSERT INTO $name VALUES (2)") + val lmcAtV3 = LastManifestCommit(version = 3, contentRootVersion = 2) + assertColdSnapshotStates( + tableName = name, + version = 3, + amtCheckpointVersion = Some(2), + trailingDeltas = Seq(3L), + lastManifestCommit = Some(lmcAtV3)) + + // v4: a manifest tree now exists, so this business commit writes its AMT inline. Version and + // content-root version coincide, and the segment trims to no trailing deltas. + sql(s"INSERT INTO $name VALUES (3)") + val lmcAtV4 = LastManifestCommit(version = 4, contentRootVersion = 4) + assertColdSnapshotStates( + tableName = name, + version = 4, + amtCheckpointVersion = Some(4), + trailingDeltas = Seq.empty, + lastManifestCommit = Some(lmcAtV4)) + + // v5: each subsequent commit keeps inlining its own manifest. + sql(s"INSERT INTO $name VALUES (4)") + val lmcAtV5 = LastManifestCommit(version = 5, contentRootVersion = 5) + assertColdSnapshotStates( + tableName = name, + version = 5, + amtCheckpointVersion = Some(5), + trailingDeltas = Seq.empty, + lastManifestCommit = Some(lmcAtV5)) + } + } + + test("[cold init] installs the correct provider from up-to-date _last_checkpoint: deferred") { + withTable("amt_cold_discovery_deferred") { + val name = "amt_cold_discovery_deferred" + createAMTTable(name, checkpointInterval = 3) + // Deferred mode, interval 3. A business INSERT that reaches a boundary is followed + // synchronously by an OPTIMIZE CHECKPOINT commit landing one version later. + // v0: CREATE + // v1: INSERT #1 (before the first checkpoint) + // v2: INSERT #2 (before the first checkpoint) + // v3: INSERT #3 (reaches boundary) + // v4: OPTIMIZE CHECKPOINT (state@v3) + // v5: INSERT #4 (non-manifest, carries the v3 reference forward) + // v6: INSERT #5 (reaches next boundary) + // v7: OPTIMIZE CHECKPOINT (state@v6) + + // v0: table just created. + assertColdSnapshotStates( + tableName = name, + version = 0, + amtCheckpointVersion = None, + trailingDeltas = Seq(0L), + lastManifestCommit = None) + + // v1: before the first checkpoint. No provider, no reference yet. + sql(s"INSERT INTO $name VALUES (1)") + assertColdSnapshotStates( + tableName = name, + version = 1, + amtCheckpointVersion = None, + trailingDeltas = Seq(0L, 1L), + lastManifestCommit = None) + + // v2: still before the first checkpoint. + sql(s"INSERT INTO $name VALUES (2)") + assertColdSnapshotStates( + tableName = name, + version = 2, + amtCheckpointVersion = None, + trailingDeltas = Seq(0L, 1L, 2L), + lastManifestCommit = None) + + // v3 INSERT reaches the boundary; the follow-up OPTIMIZE CHECKPOINT lands at v4 describing + // state as of v3. Discovery installs the provider at v3 and trims to the checkpoint commit. + sql(s"INSERT INTO $name VALUES (3)") + val lmcAtV4 = LastManifestCommit(version = 4, contentRootVersion = 3) + assertColdSnapshotStates( + tableName = name, + version = 4, + amtCheckpointVersion = Some(3), + trailingDeltas = Seq(4L), + lastManifestCommit = Some(lmcAtV4)) + + // v5: a non-manifest INSERT. The v3 checkpoint stays latest, the segment carries the v4 + // checkpoint commit plus the v5 delta, and the reference is carried forward. + sql(s"INSERT INTO $name VALUES (4)") + assertColdSnapshotStates( + tableName = name, + version = 5, + amtCheckpointVersion = Some(3), + trailingDeltas = Seq(4L, 5L), + lastManifestCommit = Some(lmcAtV4)) + + // v6 INSERT reaches the next boundary; the follow-up OPTIMIZE CHECKPOINT lands at v7 + // describing state as of v6. Discovery installs the provider at v6 and trims accordingly. + sql(s"INSERT INTO $name VALUES (5)") + val lmcAtV7 = LastManifestCommit(version = 7, contentRootVersion = 6) + assertColdSnapshotStates( + tableName = name, + version = 7, + amtCheckpointVersion = Some(6), + trailingDeltas = Seq(7L), + lastManifestCommit = Some(lmcAtV7)) + } + } + + /** The `_last_checkpoint` file system and path for a table accessed by name. */ + private def lastCheckpointFsAndPath(tableName: String): (FileSystem, Path) = { + val log = deltaLogForName(tableName) + val path = log.LAST_CHECKPOINT + (path.getFileSystem(log.newDeltaHadoopConf()), path) + } + + /** The raw bytes of the current `_last_checkpoint` file. */ + private def readLastCheckpointBytes(tableName: String): Array[Byte] = { + val (fs, path) = lastCheckpointFsAndPath(tableName) + val in = fs.open(path) + try IOUtils.toByteArray(in) finally in.close() + } + + /** Overwrites `_last_checkpoint` with `bytes` (used to plant a stale hint). */ + private def overwriteLastCheckpoint(tableName: String, bytes: Array[Byte]): Unit = { + val (fs, path) = lastCheckpointFsAndPath(tableName) + val out = fs.create(path, true) + try out.write(bytes) finally out.close() + } + + /** Deletes the `_last_checkpoint` file (used to simulate a missing hint). */ + private def deleteLastCheckpoint(tableName: String): Unit = { + val (fs, path) = lastCheckpointFsAndPath(tableName) + assert(fs.delete(path, false), s"failed to delete $path") + } + + test("[cold init] updates to the correct provider when _last_checkpoint is stale") { + val name = "amt_stale_last_checkpoint" + withTable(name) { + createAMTTable(name, checkpointInterval = 2) + // Reach the first deferred checkpoint (content root 2, recorded at v3) and capture its hint. + (1 to 2).foreach(i => sql(s"INSERT INTO $name VALUES ($i)")) + assert(deltaLogForName(name).unsafeVolatileSnapshot.version == 3, + "The first deferred AMT must land at v3.") + val staleHint = readLastCheckpointBytes(name) + + // Advance to the second deferred checkpoint (content root 4, recorded at v5). + sql(s"INSERT INTO $name VALUES (3)") + assert(deltaLogForName(name).unsafeVolatileSnapshot.version == 5, + "The second deferred AMT must land at v5.") + + // Plant the stale hint: it points to content root 2 while the table is at content root 4. + overwriteLastCheckpoint(name, staleHint) + val lmcAtV5 = LastManifestCommit(version = 5, contentRootVersion = 4) + assertColdSnapshotStates( + tableName = name, + version = 5, + amtCheckpointVersion = Some(4), + trailingDeltas = Seq(5L), + lastManifestCommit = Some(lmcAtV5)) + } + } + + test("[cold init] builds the correct provider when _last_checkpoint is absent") { + withTable("amt_absent_last_checkpoint") { + val name = "amt_absent_last_checkpoint" + createAMTTable(name, checkpointInterval = 2) + // Same timeline as the deferred cold test: 2 INSERTs land the v2 tree, recorded at v3. + (1 to 2).foreach(i => sql(s"INSERT INTO $name VALUES ($i)")) + val lmcAtV3 = LastManifestCommit(version = 3, contentRootVersion = 2) + + // Baseline: the fresh cold read resolves the v2 tree via the hint. + assertColdSnapshotStates( + tableName = name, + version = 3, + amtCheckpointVersion = Some(2), + trailingDeltas = Seq(3L), + lastManifestCommit = Some(lmcAtV3)) + + // Delete the hint; the cold read must resolve the same state, rebuilt from the CRC. + deleteLastCheckpoint(name) + assertColdSnapshotStates( + tableName = name, + version = 3, + amtCheckpointVersion = Some(2), + trailingDeltas = Seq(3L), + lastManifestCommit = Some(lmcAtV3)) + } + } + /////////////////////////// // deltaLog.update() /////////////////////////// + /** The catalog table for a catalog-managed AMT table accessed by name. */ + private def catalogTableFor(tableName: String): CatalogTable = + spark.sessionState.catalog.getTableMetadata(new TableIdentifier(tableName)) + + /** The delta versions of the deltas kept in the snapshot's log segment. */ + private def segmentDeltaVersions(snapshot: Snapshot): Seq[Long] = + snapshot.logSegment.deltas.map(FileNames.deltaVersion) + + /** + * The master equivalence oracle for the warm-update path. It checks two independent things: + * + * 1. Absolute expectations on the warm snapshot (version, AMT provider checkpoint version, + * trailing delta versions). These do NOT use cold as the oracle, so a discovery bug that + * corrupts BOTH the warm and the cold path identically is still caught here. + * 2. That the warm snapshot is otherwise indistinguishable from a fresh cold load: structural + * fields, the manifest-commit reference, the installed CRC's file count, protocol/metadata, + * and -- the strongest check -- that state reconstructed from each yields identical contents. + */ + private def assertWarmMatchesCold( + warmSnapshot: Snapshot, + tableName: String, + expectedVersion: Int, + expectedProviderVersion: Option[Int], + expectedTrailingDeltas: Seq[Int]): Unit = { + // (1) Absolute expectations -- pinned literals, independent of the cold path. + assert(warmSnapshot.version == expectedVersion, + s"warm v${warmSnapshot.version} != expected v$expectedVersion.") + assert(amtProvider(warmSnapshot).map(_.checkpointVersion) == expectedProviderVersion, + s"warm provider ${amtProvider(warmSnapshot).map(_.checkpointVersion)} != " + + s"expected $expectedProviderVersion.") + assert(segmentDeltaVersions(warmSnapshot) == expectedTrailingDeltas, + s"warm deltas ${segmentDeltaVersions(warmSnapshot)} != expected $expectedTrailingDeltas.") + + // (2) Warm must additionally match a fresh cold load in every observable field. + val (coldDeltaLog, coldSnapshot) = coldLoad(tableName) + assert(warmSnapshot.version == coldSnapshot.version, + s"warm v${warmSnapshot.version} != cold v${coldSnapshot.version}.") + assert(warmSnapshot.logSegment.equals(coldSnapshot.logSegment), + "log segment must be equal between warm and cold.") + assert(warmSnapshot.lastManifestCommitOpt == coldSnapshot.lastManifestCommitOpt, + s"warm ${warmSnapshot.lastManifestCommitOpt} != cold ${coldSnapshot.lastManifestCommitOpt}.") + if (writeChecksumEnabled) { + assert(warmSnapshot.checksumOpt == coldSnapshot.checksumOpt, + "checksum must be equal between warm and cold.") + } + assert(warmSnapshot.protocol == coldSnapshot.protocol, + "protocol must be equal between warm and cold.") + assert(warmSnapshot.metadata == coldSnapshot.metadata, + "metadata must be equal between warm and cold.") + // Data-level: reconstructed contents must match, which catches a wrong log segment even when + // the scalar fields above happen to agree. + checkAnswer( + warmSnapshot.deltaLog.createDataFrame( + warmSnapshot, warmSnapshot.allFilesViaStateReconstruction.collect().toSeq), + coldDeltaLog.createDataFrame( + coldSnapshot, coldSnapshot.allFilesViaStateReconstruction.collect().toSeq).collect().toSeq) + } + + /** + * Context for the warm update test harness. + * + * @param label The test name. + * @param staleVersion The version of the stale DeltaLog handle. + * @param cpProviderVersions The versions of all the checkpoint providers in the table. + * @param expectedVersion The expected snapshot version after update(). + * @param expectedCpVersion The expected checkpoint provider version installed by update(). + * @param expectedTrailingDeltas The expected trailing deltas installed by update(). + */ + private case class WarmUpdateContext( + label: String, + staleVersion: Int, + cpProviderVersions: Seq[Int], + latestCommitVersion: Int, + expectedCpVersion: Option[Int], + expectedTrailingDeltas: Seq[Int]) { + // For simplicity in versioning, the test harness always uses inline manifest commits. But since + // the first checkpoint cannot be inline, we trigger a full checkpoint at v1, defer its emission + // to v2, and bump up the checkpoint interval to avoid emitting more deferred checkpoints in v3. + // The test harness then starts from v3. + require(staleVersion >= 3) + require(cpProviderVersions.head == 1) + require(staleVersion <= latestCommitVersion) + require(cpProviderVersions.forall(_ <= latestCommitVersion)) + require(expectedCpVersion.forall(_ <= latestCommitVersion)) + require(expectedTrailingDeltas.forall(_ <= latestCommitVersion)) + } + + private def runWarmUpdateContext(ctx: WarmUpdateContext): Unit = { + val name = s"amt_warm_matrix" + withTable(name) { + createAMTTable(name, checkpointInterval = 1) + sql(s"INSERT INTO $name VALUES (1)") // v1: triggers a deferred checkpoint + // v2: OPTIMIZE CHECKPOINT (state@v1) + sql(s"ALTER TABLE $name SET TBLPROPERTIES ('delta.checkpointInterval' = '1000')") + // v3: bump up the interval for maneuverability + assert(deltaLogForName(name).unsafeVolatileSnapshot.version == 3) + + (4 to ctx.staleVersion).foreach { i => + if (ctx.cpProviderVersions.contains(i)) { + withInline { + sql(s"INSERT INTO $name VALUES ($i)") + } + } else { + sql(s"INSERT INTO $name VALUES ($i)") + } + } + + val staleLog = deltaLogForName(name) + assert(staleLog.unsafeVolatileSnapshot.version == ctx.staleVersion, + s"expected stale v${ctx.staleVersion}, got v${staleLog.unsafeVolatileSnapshot.version}.") + DeltaLog.clearCache() + + // Advance the true table past the pin through fresh instances (cache cleared above). + ((ctx.staleVersion + 1) to ctx.latestCommitVersion).foreach { i => + if (ctx.cpProviderVersions.contains(i)) { + withInline { + sql(s"INSERT INTO $name VALUES ($i)") + } + } else { + sql(s"INSERT INTO $name VALUES ($i)") + } + } + + // The handle must still be stale at the pin before update(). + assert(staleLog.unsafeVolatileSnapshot.version == ctx.staleVersion, + s"handle must remain stale at v${ctx.staleVersion} before update(), but was at " + + s"v${staleLog.unsafeVolatileSnapshot.version}.") + + val snapshotAfterUpdate = staleLog.update(catalogTableOpt = Some(catalogTableFor(name))) + assertWarmMatchesCold( + snapshotAfterUpdate, + name, + ctx.latestCommitVersion, + ctx.expectedCpVersion, + ctx.expectedTrailingDeltas) + } + } + + Seq( + WarmUpdateContext( + label = "after a checkpoint", + staleVersion = 3, + cpProviderVersions = Seq(1), + latestCommitVersion = 3, + expectedCpVersion = Some(1), + expectedTrailingDeltas = Seq(2, 3) + ), + WarmUpdateContext( + label = "on a checkpoint", + staleVersion = 4, + cpProviderVersions = Seq(1, 4), + latestCommitVersion = 4, + expectedCpVersion = Some(4), + expectedTrailingDeltas = Seq.empty + ), + WarmUpdateContext( + label = "long trailing deltas", + staleVersion = 10, + cpProviderVersions = Seq(1, 4), + latestCommitVersion = 10, + expectedCpVersion = Some(4), + expectedTrailingDeltas = Seq(5, 6, 7, 8, 9, 10) + ) + ).foreach { ctx => + test(s"[warm update] no-op on the same version: ${ctx.label}") { + runWarmUpdateContext(ctx) + } + } + + Seq( + WarmUpdateContext( + label = "no new checkpoints", + staleVersion = 3, + cpProviderVersions = Seq(1), + latestCommitVersion = 4, + expectedCpVersion = Some(1), + expectedTrailingDeltas = Seq(2, 3, 4) + ), + WarmUpdateContext( + label = "one new checkpoint", + staleVersion = 3, + cpProviderVersions = Seq(1, 4), + latestCommitVersion = 5, + expectedCpVersion = Some(4), + expectedTrailingDeltas = Seq(5) + ), + WarmUpdateContext( + label = "multiple new checkpoints", + staleVersion = 3, + cpProviderVersions = Seq(1, 4, 7, 10), + latestCommitVersion = 12, + expectedCpVersion = Some(10), + expectedTrailingDeltas = Seq(11, 12) + ) + ).foreach { ctx => + test( + s"[warm update] builds the correct latest snapshot: some=>some trailing deltas,${ctx.label}" + ) { + runWarmUpdateContext(ctx) + } + } + + Seq( + WarmUpdateContext( + label = "one new checkpoint", + staleVersion = 3, + cpProviderVersions = Seq(1, 4), + latestCommitVersion = 4, + expectedCpVersion = Some(4), + expectedTrailingDeltas = Seq.empty + ), + WarmUpdateContext( + label = "multiple new checkpoints", + staleVersion = 3, + cpProviderVersions = Seq(1, 4, 7, 10), + latestCommitVersion = 10, + expectedCpVersion = Some(10), + expectedTrailingDeltas = Seq.empty + ) + ).foreach { ctx => + test( + s"[warm update] builds the correct latest snapshot: some=>none trailing deltas, ${ctx.label}" + ) { + runWarmUpdateContext(ctx) + } + } + + Seq( + WarmUpdateContext( + label = "no new checkpoints", + staleVersion = 4, + cpProviderVersions = Seq(1, 4), + latestCommitVersion = 5, + expectedCpVersion = Some(4), + expectedTrailingDeltas = Seq(5) + ), + WarmUpdateContext( + label = "one new checkpoint", + staleVersion = 4, + cpProviderVersions = Seq(1, 4, 7), + latestCommitVersion = 8, + expectedCpVersion = Some(7), + expectedTrailingDeltas = Seq(8) + ), + WarmUpdateContext( + label = "multiple new checkpoints", + staleVersion = 4, + cpProviderVersions = Seq(1, 4, 7, 10), + latestCommitVersion = 12, + expectedCpVersion = Some(10), + expectedTrailingDeltas = Seq(11, 12) + ) + ).foreach { ctx => + test( + s"[warm update] builds the correct latest snapshot: none=>some trailing deltas, ${ctx.label}" + ) { + runWarmUpdateContext(ctx) + } + } + + Seq( + WarmUpdateContext( + label = "one new checkpoint", + staleVersion = 4, + cpProviderVersions = Seq(1, 4, 7), + latestCommitVersion = 7, + expectedCpVersion = Some(7), + expectedTrailingDeltas = Seq.empty + ), + WarmUpdateContext( + label = "multiple new checkpoints", + staleVersion = 4, + cpProviderVersions = Seq(1, 4, 7, 10), + latestCommitVersion = 10, + expectedCpVersion = Some(10), + expectedTrailingDeltas = Seq.empty + ) + ).foreach { ctx => + test( + s"[warm update] builds the correct latest snapshot: none=>none trailing deltas, ${ctx.label}" + ) { + runWarmUpdateContext(ctx) + } + } + + // The above test harness doesn't cover the case of a checkpoint-less stale handle acquiring its + // first AMT checkpoint through update(), for the sake of versioning simplicity. + test("[warm update] a checkpoint-less stale handle acquires its first AMT checkpoint") { + val name = "amt_warm_no_cp_to_amt" + withTable(name) { + createAMTTable(name, checkpointInterval = 3) + sql(s"INSERT INTO $name VALUES (1)") // v1: before the first checkpoint -- no provider + sql(s"INSERT INTO $name VALUES (2)") // v2: still before the first checkpoint -- no provider + + // Pin a stale handle at v2, which precedes the first checkpoint and has no AMT provider. + val staleLog = deltaLogForName(name) + assert(staleLog.unsafeVolatileSnapshot.version == 2, + s"expected stale v2, got v${staleLog.unsafeVolatileSnapshot.version}.") + assert(amtProvider(staleLog.unsafeVolatileSnapshot).isEmpty, + "the stale handle must have no AMT checkpoint provider before update().") + DeltaLog.clearCache() + + sql(s"INSERT INTO $name VALUES (3)") // v3: reaches the boundary + // v4: OPTIMIZE CHECKPOINT (state@v3) -- the first AMT + assert(deltaLogForName(name).update().version == 4, "the first deferred AMT must land at v4.") + + // The handle must still be stale at v2 before update(). + assert(staleLog.unsafeVolatileSnapshot.version == 2, + s"handle must remain stale at v2 before update(), but was at " + + s"v${staleLog.unsafeVolatileSnapshot.version}.") + + val warm = staleLog.update(catalogTableOpt = Some(catalogTableFor(name))) + assertWarmMatchesCold( + warm, + tableName = name, + expectedVersion = 4, + expectedProviderVersion = Some(3), + expectedTrailingDeltas = Seq(4)) + } + } + /////////////////////////// // Post commit snapshot /////////////////////////// @@ -65,4 +681,249 @@ class AMTSnapshotDiscoverySuite extends AMTCheckpointTestBase { assert(segmentDeltaVersions.forall(_ > context.checkpoint.version), s"Log segment must trim deltas up to the checkpoint version; got $segmentDeltaVersions.") } + + ////////////////////////////////// + // Minor compaction compatibility + ////////////////////////////////// + + private def buildDeltaLogWithCompactedDeltasAndStaleLastCheckpointHint(name: String) + : (DeltaLog, Snapshot, DeltaLog) = { + createAMTTable(name, checkpointInterval = 2) + // v1: INSERT (1) + // v2: INSERT (2) + // v3: OPTIMIZE CHECKPOINT (content root @v2) + (1 to 2).foreach(i => sql(s"INSERT INTO $name VALUES ($i)")) + assert(deltaLogForName(name).unsafeVolatileSnapshot.version == 3) + + val staleHintAtV2 = readLastCheckpointBytes(name) + val staleLogAtV3 = deltaLogForName(name) + assert(staleLogAtV3.unsafeVolatileSnapshot.version == 3) + DeltaLog.clearCache() + + // v4: INSERT (3) + // v5: OPTIMIZE CHECKPOINT (content root @v4) + sql(s"INSERT INTO $name VALUES (3)") + assert(deltaLogForName(name).unsafeVolatileSnapshot.version == 5) + + // v6: Bump up checkpoint interval + // v7: INSERT (4) + // v8: INSERT (5) + sql(s"ALTER TABLE $name SET TBLPROPERTIES ('delta.checkpointInterval' = '1000')") + (4 to 5).foreach(i => sql(s"INSERT INTO $name VALUES ($i)")) + + val latestDeltaLogAtV8 = deltaLogForName(name) + val latestSnapshotAtV8 = latestDeltaLogAtV8.update() + assert(latestSnapshotAtV8.version == 8) + assert(amtProvider(latestSnapshotAtV8).map(_.version).contains(4L)) + assert(staleLogAtV3.unsafeVolatileSnapshot.version == 3) + + // Deltas layout: 1, 2, [3, 4, 5], [6, 7, 8] + // Stale CP hint: <2> + // Stale deltas: [3, 4, 5], [6, 7, 8]) + // Latest CP: <4> + // Trimmed deltas: 5, [6, 7, 8] + // (Stale means that this is the initial LogSegment built by `getLogSegmentForVersion`, + // which is not yet trimmed to the accurate checkpoint version by AMT reconciliation.) + Seq((3, 5), (6, 8)).foreach { case (start, end) => + minorCompactDeltaLog( + tablePath = latestDeltaLogAtV8.dataPath.toString, + startVersion = start, + endVersion = end, + tableName = Some(name)) + } + overwriteLastCheckpoint(name, staleHintAtV2) + + (latestDeltaLogAtV8, latestSnapshotAtV8, staleLogAtV3) + } + + private def assertSnapshotTrimmed( + snapshot: Snapshot, + expectedVersion: Long, + expectedAMTContentRootVersion: Long, + expectedIndividualDeltas: Seq[Long], + expectedCompactedDeltas: Seq[(Long, Long)], + expectedNonCompactedDeltas: Seq[Long], + expectedData: Set[Int]): Unit = { + assert(snapshot.version == expectedVersion) + assert(amtProvider(snapshot).map(_.version).contains(expectedAMTContentRootVersion)) + val actualIndividualDeltas = snapshot.logSegment.deltas + .filterNot(FileNames.isCompactedDeltaFile) + .map(FileNames.deltaVersion) + assert(actualIndividualDeltas == expectedIndividualDeltas) + val actualCompactedDeltas = snapshot.logSegment.deltas + .filter(FileNames.isCompactedDeltaFile) + .map(FileNames.compactedDeltaVersions) + assert(actualCompactedDeltas == expectedCompactedDeltas) + val actualNonCompactedDeltas = snapshot.logSegment.nonCompactedDeltasOpt + .map(n => n.map(FileNames.deltaVersion)) + assert(actualNonCompactedDeltas.contains(expectedNonCompactedDeltas)) + val actualDeltaAtCheckpointVersion = snapshot.logSegment.deltaAtCheckpointVersionOpt + .map(FileNames.deltaVersion) + assert(actualDeltaAtCheckpointVersion.contains(expectedAMTContentRootVersion)) + val reconstructedData = snapshot.deltaLog + .createDataFrame(snapshot, snapshot.allFilesViaStateReconstruction.collect().toSeq) + .collect().map(_.getInt(0)).toSet + assert(reconstructedData == expectedData) + } + + test("[minor compaction] a straddling compacted delta is dropped and its gap refilled: cold") { + val name = "amt_compaction_straddle_cold" + withTable(name) { + val (latestDeltaLogAtV8, latestSnapshotAtV8, _) = + buildDeltaLogWithCompactedDeltasAndStaleLastCheckpointHint(name) + // Cold path: a cold load builds the segment at content root 2 (stale hint) first, and then + // reconciles to trim to content root 4. + val (coldDeltaLog, coldSnapshot) = coldLoad(name) + assertSnapshotTrimmed( + coldSnapshot, + expectedVersion = 8, + expectedAMTContentRootVersion = 4, + expectedIndividualDeltas = Seq(5L), + expectedCompactedDeltas = Seq((6L, 8L)), + expectedNonCompactedDeltas = Seq(5L, 6L, 7L, 8L), + expectedData = Set(1, 2, 3, 4, 5)) + } + } + + test("[minor compaction] a straddling compacted delta is dropped and its gap refilled: warm") { + val name = "amt_compaction_straddle_warm" + withTable(name) { + val (latestDeltaLogAtV8, latestSnapshotAtV8, staleLogAtV3) = + buildDeltaLogWithCompactedDeltasAndStaleLastCheckpointHint(name) + // Warm path: a warm update builds the segment at content root 2 (stale hint + old checkpoint + // provider reuse) first, and then reconciles to trim to content root 4. + val warmSnapshot = staleLogAtV3.update(catalogTableOpt = Some(catalogTableFor(name))) + assert(warmSnapshot.version == 8, s"expected warm v8, got v${warmSnapshot.version}.") + assertSnapshotTrimmed( + warmSnapshot, + expectedVersion = 8, + expectedAMTContentRootVersion = 4, + expectedIndividualDeltas = Seq(5L), + expectedCompactedDeltas = Seq((6L, 8L)), + expectedNonCompactedDeltas = Seq(5L, 6L, 7L, 8L), + expectedData = Set(1, 2, 3, 4, 5)) + } + } + + test("[minor compaction] the post-commit fast path trims a compacted pre-commit segment") { + val name = "amt_compaction_post_commit" + withTable(name) { + val (latestDeltaLog, latestSnapshotAtV8, _) = + buildDeltaLogWithCompactedDeltasAndStaleLastCheckpointHint(name) + // Emit a deferred full checkpoint. All the previous deltas (compacted or not) will be trimmed + // as the checkpoint is installed. A compacted delta will never straddle on this content root, + // unless there has been concurrent updates on the table. + commitCheckpoint(latestDeltaLog, incremental = false) // V9: OPTIMIZE CHECKPOINT (AMT @V8) + val postCommitSnapshot = latestDeltaLog.unsafeVolatileSnapshot + assertSnapshotTrimmed( + postCommitSnapshot, + expectedVersion = 9L, + expectedAMTContentRootVersion = 8L, + expectedIndividualDeltas = Seq(9L), + expectedCompactedDeltas = Seq.empty, + expectedNonCompactedDeltas = Seq(9L), + expectedData = Set(1, 2, 3, 4, 5)) + } + } + + test("[minor compaction] an incremental checkpoint over a compacted pre-commit segment throws") { + val name = "amt_compaction_incremental" + withTable(name) { + val (_, _, _) = + buildDeltaLogWithCompactedDeltasAndStaleLastCheckpointHint(name) + // With compacted deltas in the pre-commit segment, Incremental AMT writer appends + // non-contiguous versions to InMemoryLogReplay, which fails the contiguity assertion. + // Cold load the delta log to incorporate the compacted deltas into the snapshot. + val (coldDeltaLog, _) = coldLoad(name) + val e = intercept[Throwable] { + commitCheckpoint(coldDeltaLog, incremental = true) + } + assert(e.getMessage().contains("Attempted to replay version")) + } + } +} + +/** + * AMT snapshot discovery must work identically via the CommitInfo fallback when CRC is missing. + */ +class AMTSnapshotDiscoveryWithoutCRCSuite extends AMTSnapshotDiscoverySuite { + + override protected def sparkConf: SparkConf = + super.sparkConf.set(DeltaSQLConf.DELTA_WRITE_CHECKSUM_ENABLED.key, "false") + + test("[cold init] refuses an AMT provider when neither CRC nor CommitInfo carries a reference") { + val name = "amt_no_reference_refused" + withTable(name) { + createAMTTable(name, checkpointInterval = 2) + // 2 INSERTs land the v2 tree recorded at v3, so the hint references an AMT checkpoint + // that a cold read installs as the provider. + (1 to 2).foreach(i => sql(s"INSERT INTO $name VALUES ($i)")) + + // Strip the reference from the recording commit's CommitInfo. With no CRC and no CommitInfo + // reference, nothing corroborates the installed AMT provider, so the cold read is refused. + val (deltaLog, snapshot) = coldLoad(name) + val commitPath = DeltaCommitFileProvider(snapshot).deltaFile(3) + val hadoopConf = deltaLog.newDeltaHadoopConf() + val stripped = deltaLog.store.readAsIterator(commitPath, hadoopConf).toList + .map(Action.fromJson) + .map { + case ci: CommitInfo => ci.copy(lastManifestCommit = None).json + case other => other.json + } + deltaLog.store.write(commitPath, stripped.toIterator, overwrite = true, hadoopConf) + + val e = intercept[IllegalStateException](coldLoad(name)) + assert(e.getMessage.contains("no lastManifestCommit is available from either the CRC"), + s"expected the no-reference refusal, got: ${e.getMessage}") + } + } + + test("[cold init] CommitInfo read is not kicked off when config is disabled: AMT table") { + withSQLConf( + DeltaSQLConf.AMT_SNAPSHOT_DISCOVERY_ASYNC_COMMIT_INFO_READ_ENABLED.key -> "false") { + val name = "amt_slow_commit_info_read" + withTable(name) { + createAMTTable(name, checkpointInterval = 2) + // 2 INSERTs land the v2 tree recorded at v3, so the hint references an AMT checkpoint + // that a cold read installs as the provider. + (1 to 2).foreach(i => sql(s"INSERT INTO $name VALUES ($i)")) + + // With CommitInfo read not kicked off, the installed AMT provider is refused. + val e = intercept[IllegalStateException](coldLoad(name)) + assert(e.getMessage.contains("no lastManifestCommit is available from either the CRC")) + } + } + } + + test("[cold init] CommitInfo read is not kicked off when config is disabled: non-AMT table") { + withSQLConf( + DeltaSQLConf.AMT_SNAPSHOT_DISCOVERY_ASYNC_COMMIT_INFO_READ_ENABLED.key -> "false") { + val name = "non_amt_slow_commit_info_read" + withTable(name) { + sql(s"CREATE TABLE $name (id INT) USING delta") + (1 to 2).foreach(i => sql(s"INSERT INTO $name VALUES ($i)")) + + // With CommitInfo read not kicked off, non-AMT tables can be loaded unaffected. + val (_, snapshot) = coldLoad(name) + assert(snapshot.version == 2, "the snapshot must be at v2.") + assert(amtProvider(snapshot).isEmpty, "the AMT provider must be absent.") + } + } + } +} + +/** + * With batch size 1, all commits are backfilled to a standard NNN.json immediately. + */ +class AMTSnapshotDiscoveryBackfillBatch1Suite extends AMTSnapshotDiscoverySuite { + override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(1) +} + +/** + * With a large backfill batch size, no commits are automatically backfilled. Only before an AMT + * checkpoint lands in a manifest commit, the previous commits are backfilled. This suite exercises + * that code path and ensures AMT snapshot discovery works correctly. + */ +class AMTSnapshotDiscoveryBackfillBatch100Suite extends AMTSnapshotDiscoverySuite { + override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(100) } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTSnapshotSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTSnapshotSuite.scala index 6967c0fc8eb..29871bc409d 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTSnapshotSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTSnapshotSuite.scala @@ -16,13 +16,17 @@ package org.apache.spark.sql.delta.amt -import org.apache.spark.sql.delta.{Checkpoints, DeletionVectorsTestUtils, DeltaLog, DeltaOperations} -import org.apache.spark.sql.delta.actions.{AddFile, Checkpoint, ContentRoot, DeletionVectorDescriptor} +import org.apache.spark.sql.delta.{Checkpoints, DeletionVectorsTestUtils, DeltaLog, DeltaOperations, Snapshot} +import org.apache.spark.sql.delta.DeltaTestUtils.createTestAddFile +import org.apache.spark.sql.delta.actions.{AddFile, Checkpoint, ContentRoot} +import org.apache.spark.sql.delta.actions.DeletionVectorDescriptor +import org.apache.spark.sql.delta.deletionvectors.ManifestBitmap import org.apache.spark.sql.delta.deletionvectors.RoaringBitmapArray import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util.FileNames import org.apache.hadoop.fs.Path +import org.apache.spark.paths.SparkPath import org.apache.spark.sql.Row import org.apache.spark.sql.execution.datasources.LogicalRelation import org.apache.spark.sql.functions.col @@ -34,15 +38,6 @@ class AMTSnapshotSuite extends AMTCheckpointTestBase with DeletionVectorsTestUti import testImplicits._ - /** - * A manifest deletion vector addresses entries by their position inside a leaf, so these tests - * need the checkpointed entries to live in leaves rather than stay resident in the root. The - * incremental writer only spills live adds into a leaf once the root would exceed - * `entriesPerLeaf`, and the default (50000) never trips for these small tables. A cap of one - * entry per leaf forces every live add into a leaf under every scenario. - */ - private val mdvLeafResidentConfs = Seq(DeltaSQLConf.AMT_ENTRIES_PER_LEAF.key -> "1") - /////////////////////////// // Post commit snapshot /////////////////////////// @@ -57,7 +52,22 @@ class AMTSnapshotSuite extends AMTCheckpointTestBase with DeletionVectorsTestUti checkAnswer(spark.read.table(context.tableName), Seq(Row(2), Row(3))) // The manifest tree captures exactly the post-DELETE live files (computePostCommitState applied // the RemoveFile), i.e. it matches snapshot.allFiles. - assertReconstructsLiveFileSet(context) + assertLiveAddFilesRoundTrip(context) + } + + testAcrossAMTCheckpointScenarios( + "plain AMT tables reconstruct AddFiles with no amtPassthrough", + "amt_passthrough_roundtrip")( + setup = name => sql(s"INSERT INTO $name VALUES (1)"), + inlineCheckpointTriggerActionsOrSQL = Some(name => Right( + s"INSERT INTO $name VALUES (2)"))) { context => + // These are plain parquet/v4 files with no Iceberg passthrough, so `toAddFile` carries + // nothing (avoids per-file overhead). + val files = context.postCheckpointSnapshot.allFiles.collect() + assert(files.nonEmpty) + assert(files.forall(_.amtPassthrough.isEmpty), + s"plain AMT files must carry no passthrough; got " + + s"${files.map(_.amtPassthrough).mkString(", ")}") } testAcrossAMTCheckpointScenarios( @@ -75,7 +85,603 @@ class AMTSnapshotSuite extends AMTCheckpointTestBase with DeletionVectorsTestUti assert(context.postCheckpointSnapshot.allFiles.count() == 1, "Only the surviving overwrite file should be live.") // The tree must capture exactly the surviving live files after overwrite + delete. - assertReconstructsLiveFileSet(context) + assertLiveAddFilesRoundTrip(context) + } + + // AMT inline or not should be irrelevant to test result. + testAcrossAMTCheckpointScenarios( + "amtPassthrough survives from one AMT into the next", + "amt_passthrough_amt_to_amt", + deferredScenarios = Seq.empty + )( + setup = name => { + sql(s"INSERT INTO $name VALUES (1)") + withSQLConf( + DeltaSQLConf.AMT_LARGE_COMMIT_ACTIONS_COUNT_THRESHOLD_FOR_INLINE_MANIFEST_COMMIT.key + -> "1" + ) { + deltaLogForName(name) + .startTransaction() + .commit( + Seq( + createTestAddFile(encodedPath = "passthrough-file") + .copy(amtPassthrough = Some(fullPassthrough)) + ), + DeltaOperations.ManualUpdate + ) + } + // At this point tree should be materialized. + val seeded = + amtFilesInTree(deltaLogForName(name).update(), Some("passthrough-file")) + assert( + seeded.length == 1, + s"expected one tree entry for passthrough-file, got: $seeded." + ) + assert( + seeded.head.amtPassthrough.contains(fullPassthrough), + "sanity: passthrough must be in the seeded AMT before the checkpoint." + ) + }, + inlineCheckpointTriggerActionsOrSQL = + Some(name => Right(s"INSERT INTO $name VALUES (100)")) + ) { context => + val seededVersion = amtProvider(context.postSetupSnapshot) + .getOrElse(fail("the seed commit must have emitted an AMT")) + .version + assert( + context.provider.version > seededVersion, + s"a new AMT must have re-materialized past the seeded one at v$seededVersion; " + + s"got v${context.provider.version}." + ) + + // The passthrough survives into the re-materialized AMT and reconstructs on read. + val snapshot = context.postCheckpointSnapshot + val inTree = amtFilesInTree(snapshot, Some("passthrough-file")) + assert( + inTree.length == 1, + s"expected one tree entry for passthrough-file, got: $inTree." + ) + assert( + inTree.head.amtPassthrough.contains(fullPassthrough), + s"passthrough must propagate into the re-materialized AMT; got " + + s"${inTree.head.amtPassthrough}." + ) + val liveFile = + snapshot.allFiles.collect().filter(_.path == "passthrough-file") + assert( + liveFile.length == 1, + s"expected exactly one seeded file, got: ${liveFile.toSeq}." + ) + assert( + liveFile.head.amtPassthrough.contains(fullPassthrough), + s"passthrough must reconstruct from the re-materialized AMT; got " + + s"${liveFile.head.amtPassthrough}." + ) + } + + // AMT inline or not should be irrelevant to test result. + testAcrossAMTCheckpointScenarios( + "amtPassthrough survives from one AMT, with in-place rewrite, into the next", + "amt_passthrough_inplace", + deferredScenarios = Seq.empty + )( + setup = name => { + sql(s"INSERT INTO $name VALUES (1)") + val log = deltaLogForName(name) + withSQLConf( + DeltaSQLConf.AMT_LARGE_COMMIT_ACTIONS_COUNT_THRESHOLD_FOR_INLINE_MANIFEST_COMMIT.key + -> "1" + ) { + log + .startTransaction() + .commit( + Seq( + createTestAddFile(encodedPath = "passthrough-file") + .copy( + amtPassthrough = Some(fullPassthrough), + stats = """{"numRecords":1}""" + ) + ), + DeltaOperations.ManualUpdate + ) + } + val seedAMTVersion = amtProvider(log.update()).map(_.version) + // AMTPassthrough before the rewrite. + val seeded = amtFilesInTree(log.update(), Some("passthrough-file")) + assert( + seeded.length == 1, + s"expected one tree entry for passthrough-file, got: $seeded." + ) + assert( + seeded.head.amtPassthrough.contains(fullPassthrough), + "passthrough seeded into the AMT." + ) + val current = log + .update() + .allFiles + .collect() + .find(_.path == "passthrough-file") + .getOrElse(fail("passthrough-file must be live.")) + assert( + current.amtPassthrough.contains(fullPassthrough), + s"the live file carries passthrough before the rewrite; got " + + s"${current.amtPassthrough}." + ) + + // The rewrite must land as a plain LOG commit. + withSQLConf( + DeltaSQLConf.AMT_LARGE_COMMIT_ACTIONS_COUNT_THRESHOLD_FOR_INLINE_MANIFEST_COMMIT.key + -> Long.MaxValue.toString + ) { + log + .startTransaction() + .commit( + Seq( + current.copy( + stats = """{"numRecords":1,"probe":true}""", + dataChange = false + ) + ), + DeltaOperations.ComputeStats(predicate = Nil) + ) + } + // Verify that AMT checkpoint version stays the same. + val afterRewriteAMTVersion = amtProvider(log.update()).map(_.version) + assert( + afterRewriteAMTVersion == seedAMTVersion, + s"the in-place rewrite must NOT be committed into an AMT; the tree moved from " + + s"$seedAMTVersion to $afterRewriteAMTVersion." + ) + + // The reconstructed live file keeps its passthrough, and the stats update lands. + val afterRewrite = + log.update().allFiles.collect().filter(_.path == "passthrough-file") + assert(afterRewrite.length == 1) + assert( + afterRewrite.head.stats == """{"numRecords":1,"probe":true}""", + "the in-place stats update itself must land." + ) + assert( + afterRewrite.head.amtPassthrough.contains(fullPassthrough), + s"passthrough must survive the in-place rewrite; got " + + s"${afterRewrite.head.amtPassthrough}." + ) + }, + inlineCheckpointTriggerActionsOrSQL = + Some(name => Right(s"INSERT INTO $name VALUES (100)")) + ) { context => + // A new AMT re-materialized past the rewrite and still carries the passthrough. + val seededVersion = amtProvider(context.postSetupSnapshot) + .getOrElse(fail("the seed commit must have emitted an AMT")) + .version + assert( + context.provider.version > seededVersion, + s"a new AMT must have re-materialized past the rewrite; the tree is still at " + + s"v${context.provider.version} (seeded at v$seededVersion)." + ) + val snapshot = context.postCheckpointSnapshot + val survived = amtFilesInTree(snapshot, Some("passthrough-file")) + assert( + survived.length == 1, + s"expected one tree entry for passthrough-file, got: $survived." + ) + assert( + survived.head.amtPassthrough.contains(fullPassthrough), + s"passthrough must survive into the re-materialized AMT; got " + + s"${survived.head.amtPassthrough}." + ) + val finalLive = + snapshot.allFiles.collect().filter(_.path == "passthrough-file") + assert( + finalLive.length == 1 && + finalLive.head.amtPassthrough.contains(fullPassthrough), + s"passthrough must survive the rewrite + re-materialization; got " + + s"${finalLive.map(_.amtPassthrough).toSeq}." + ) + } + + // Bypasses `testAcrossAMTCheckpointScenarios`: this test needs the table to have NO AMT at all + // while the passthrough is seeded. + test("amtPassthrough from a log commit is carried into the next AMT") { + withSQLConf( + DeltaSQLConf.AMT_LARGE_COMMIT_ACTIONS_COUNT_THRESHOLD_FOR_INLINE_MANIFEST_COMMIT.key + -> "1" + ) { + withTable("amt_passthrough_from_log_commit") { + val name = "amt_passthrough_from_log_commit" + createAMTTable(name, checkpointInterval = 10000) + val log = deltaLogForName(name) + assert( + amtProvider(log.update()).isEmpty, + "no AMT may exist before the first trigger." + ) + + // V1 is a LOG commit: the file's passthrough lives only in the commit JSON at this point. + val passthrough = fullPassthrough + val fileWithPassthrough = createTestAddFile(encodedPath = "passthrough-file") + .copy(amtPassthrough = Some(passthrough), stats = """{"numRecords":1}""") + log + .startTransaction() + .commit(Seq(fileWithPassthrough), DeltaOperations.ManualUpdate) + assert( + amtProvider(deltaLogForName(name).update()).isEmpty, + "the seeding commit must stay a LOG commit (no AMT yet)." + ) + val fromLog = deltaLogForName(name) + .update() + .allFiles + .collect() + .filter(_.path == "passthrough-file") + assert( + fromLog.length == 1, + s"expected the seeded file to be live, got: ${fromLog.toSeq}." + ) + assert( + fromLog.head.amtPassthrough.contains(passthrough), + s"the log-commit file must carry its passthrough; got " + + s"${fromLog.head.amtPassthrough}." + ) + + // Now trigger the table's first AMT. + sql(s"ALTER TABLE $name SET TBLPROPERTIES ('delta.checkpointInterval' = '2')") + sql(s"INSERT INTO $name VALUES (100)") + sql(s"INSERT INTO $name VALUES (101)") + val snapshot = deltaLogForName(name).update() + val amtVersion = amtProvider(snapshot).map(_.version) + assert( + amtVersion.contains(5L), + s"the first AMT must describe content version 5; got $amtVersion." + ) + + val inTree = amtFilesInTree(snapshot, Some("passthrough-file")) + assert( + inTree.length == 1, + s"expected one tree entry for passthrough-file, got: $inTree." + ) + assert( + inTree.head.amtPassthrough.contains(passthrough), + s"the log commit's passthrough must be carried into the AMT; got " + + s"${inTree.head.amtPassthrough}." + ) + val live = snapshot.allFiles.collect().filter(_.path == "passthrough-file") + assert( + live.length == 1 && live.head.amtPassthrough.contains(passthrough), + s"passthrough must reconstruct from the AMT; got " + + s"${live.map(_.amtPassthrough).toSeq}." + ) + } + } + } + + /** + * A passthrough with every [[AMTPassthrough]] field set, so tests use the whole carrier. + */ + private def fullPassthrough: AMTPassthrough = AMTPassthrough( + spec_id = Some(42), + sort_order_id = Some(7), + key_metadata = Some(Array[Byte](1, 2, 3)), + split_offsets = Some(Seq(0L, 128L, 256L))) + + /** + * The AMT-derived live [[AddFile]]s in the snapshot's current tree, optionally restricted to + * `path`. + */ + private def amtFilesInTree(snapshot: Snapshot, path: Option[String] = None): Seq[AddFile] = { + val provider = amtProvider(snapshot).getOrElse(fail("Snapshot has no AMTCheckpointProvider.")) + val live = provider.loadActionsForStateReconstruction(spark, snapshot.deltaLog) + .getOrElse(fail("AMT provider must contribute tree-derived file actions.")) + .where("add is not null") + .select(col("add").as[AddFile]) + .collect() + .toSeq + path.map(p => live.filter(_.path == p)).getOrElse(live) + } + + /** + * Returns the latest snapshot after the caller adds trailing LOG commits. + * This asserts that the tail does not emit a new AMT. + */ + private def latestSnapshotAfterLogTail(context: AMTCheckpointScenarioContext): Snapshot = { + val log = context.postCheckpointSnapshot.deltaLog + val snapshot = log.update() + assert(snapshot.version > context.postCheckpointSnapshot.version) + assert(checkpointAt(log, snapshot.version).isEmpty) + val provider = amtProvider(snapshot).getOrElse( + fail("The latest snapshot must still use the AMT checkpoint provider.")) + assert(provider.checkpointAction == context.provider.checkpointAction) + snapshot + } + + testAcrossAMTCheckpointScenarios( + "snapshot replays trailing append after an AMT checkpoint", + "amt_tail_append")( + setup = name => sql(s"INSERT INTO $name VALUES (1)"), + inlineCheckpointTriggerActionsOrSQL = Some(name => Right( + s"INSERT INTO $name VALUES (2)"))) { context => + val addFilesAtCheckpointVersion = context.postCheckpointSnapshot.allFiles.collect() + val addFilePathsAtCheckpointVersion = addFilesAtCheckpointVersion.map(_.path).toSet + assert(addFilePathsAtCheckpointVersion.size == 2) + + sql(s"INSERT INTO ${context.tableName} VALUES (3)") + val finalSnapshot = latestSnapshotAfterLogTail(context) + + assert(amtFilesInTree(finalSnapshot).map(_.path).toSet == addFilePathsAtCheckpointVersion) + assert(finalSnapshot.allFiles.count() == addFilesAtCheckpointVersion.length + 1) + checkAnswer(spark.read.table(context.tableName), Seq(Row(1), Row(2), Row(3))) + } + + testAcrossAMTCheckpointScenarios( + "snapshot replays trailing remove after an AMT checkpoint", + "amt_tail_remove")( + setup = name => sql(s"INSERT INTO $name VALUES (1)"), + inlineCheckpointTriggerActionsOrSQL = Some(name => Right( + s"INSERT INTO $name VALUES (2)"))) { context => + val log = context.postCheckpointSnapshot.deltaLog + val addFilesAtCheckpointVersion = context.postCheckpointSnapshot.allFiles.collect() + assert(addFilesAtCheckpointVersion.length == 2) + val setupFile = addFilesAtCheckpointVersion.head + + log.startTransaction().commit( + Seq(setupFile.removeWithTimestamp()), + DeltaOperations.ManualUpdate) + val finalSnapshot = latestSnapshotAfterLogTail(context) + + assert( + amtFilesInTree(finalSnapshot).map(_.path).toSet == addFilesAtCheckpointVersion + .map(_.path) + .toSet + ) + val livePaths = finalSnapshot.allFiles.collect().map(_.path).toSet + assert(!livePaths.contains(setupFile.path)) + assert(livePaths.size == 1) + checkAnswer( + spark.read.table(context.tableName).groupBy().count(), + Seq(Row(1L))) + } + + testAcrossAMTCheckpointScenarios( + "snapshot replays trailing overwrite after an AMT checkpoint", + "amt_tail_overwrite")( + setup = name => sql(s"INSERT INTO $name VALUES (1)"), + inlineCheckpointTriggerActionsOrSQL = Some(name => Right( + s"INSERT INTO $name VALUES (2)"))) { context => + val addFilePathsAtCheckpointVersion = + context.postCheckpointSnapshot.allFiles.collect().map(_.path).toSet + assert(addFilePathsAtCheckpointVersion.size == 2) + + sql(s"INSERT OVERWRITE ${context.tableName} VALUES (10), (20)") + val finalSnapshot = latestSnapshotAfterLogTail(context) + + val livePaths = finalSnapshot.allFiles.collect().map(_.path).toSet + assert(livePaths.nonEmpty) + assert((livePaths & addFilePathsAtCheckpointVersion).isEmpty) + checkAnswer(spark.read.table(context.tableName), Seq(Row(10), Row(20))) + } + + testAcrossAMTCheckpointScenarios( + "snapshot replays trailing DV update after an AMT checkpoint", + "amt_tail_dv", + sqlConfs = Seq(DeltaSQLConf.DELTA_HISTORY_METRICS_ENABLED.key -> "false"))( + setup = name => { + Seq(10, 20, 30, 40, 50).toDF("id").coalesce(1) + .write.mode("append").insertInto(name) + }, + inlineCheckpointTriggerActionsOrSQL = Some(name => Right( + s"INSERT INTO $name VALUES (60)"))) { context => + val log = context.postCheckpointSnapshot.deltaLog + val addFilesAtCheckpointVersion = + context.postCheckpointSnapshot.allFiles.collect() + val addFilesAtSetupVersion = + context.postSetupSnapshot.allFiles.collect() + assert(addFilesAtSetupVersion.length == 1, "The five setup rows must land in a single file.") + val setupFileAtSetupVersion = addFilesAtSetupVersion.head + + // Write a DV to the setup file. + val dvActions = writeFileWithDVOnDisk( + log, + setupFileAtSetupVersion, + RoaringBitmapArray(0L, 2L, 4L) + ) + log.startTransaction().commit(dvActions, DeltaOperations.Delete(predicate = Seq.empty)) + val finalSnapshot = latestSnapshotAfterLogTail(context) + + // Tree correctness test + val addFilesInAmt = amtFilesInTree(finalSnapshot, Some(setupFileAtSetupVersion.path)) + assert(addFilesInAmt.length == 1) + assert( + addFilesInAmt.head.deletionVector == null, + "The AMT provider should still expose the pre-DV checkpointed file.") + checkAnswer(spark.read.table(context.tableName), Seq(Row(20), Row(40), Row(60))) + + // The setup file now has a DV associated with 3 rows deleted. + val addFilesInFinalVersion = finalSnapshot.allFiles.collect() + assert(addFilesInFinalVersion.length == addFilesAtCheckpointVersion.length) + val setupFileAtFinalVersion = addFilesInFinalVersion + .find(_.path == setupFileAtSetupVersion.path) + .getOrElse( + fail( + s"The DV-updated file ${setupFileAtSetupVersion.path} must stay live." + ) + ) + assert(setupFileAtFinalVersion.deletionVector != null) + assert(setupFileAtFinalVersion.numPhysicalRecords.contains(5L)) + assert(setupFileAtFinalVersion.numLogicalRecords.contains(2L)) + } + + testAcrossAMTCheckpointScenarios( + "snapshot replays trailing DV that fully deletes a file after an AMT checkpoint", + "amt_tail_dv_full_delete", + sqlConfs = Seq(DeltaSQLConf.DELTA_HISTORY_METRICS_ENABLED.key -> "false"))( + setup = name => { + Seq(1, 2).toDF("id").coalesce(1).write.mode("append").insertInto(name) + }, + inlineCheckpointTriggerActionsOrSQL = Some(name => Right( + s"INSERT INTO $name VALUES (3)"))) { context => + val log = context.postCheckpointSnapshot.deltaLog + val addFilesAtCheckpointVersion = context.postCheckpointSnapshot.allFiles.collect() + val addFilesAtSetupVersion = context.postSetupSnapshot.allFiles.collect() + assert(addFilesAtSetupVersion.length == 1, "The two setup rows must land in a single file.") + val setupFileAtSetupVersion = addFilesAtSetupVersion.head + + // Fully delete the setup file. + val dvActions = writeFileWithDVOnDisk(log, setupFileAtSetupVersion, RoaringBitmapArray(0L, 1L)) + log.startTransaction().commit(dvActions, DeltaOperations.Delete(predicate = Seq.empty)) + val finalSnapshot = latestSnapshotAfterLogTail(context) + + checkAnswer(spark.read.table(context.tableName), Seq(Row(3))) + + // The setup file now has a DV associated with all rows deleted. + val addFilesInFinalVersion = finalSnapshot.allFiles.collect() + assert(addFilesInFinalVersion.length == addFilesAtCheckpointVersion.length) + val setupFileAtFinalVersion = addFilesInFinalVersion + .find(_.path == setupFileAtSetupVersion.path) + .getOrElse(fail(s"The fully-deleted file ${setupFileAtSetupVersion.path} must stay live.")) + assert(setupFileAtFinalVersion.deletionVector != null) + assert(setupFileAtFinalVersion.numPhysicalRecords.contains(2L)) + assert(setupFileAtFinalVersion.numLogicalRecords.contains(0L)) + } + + testAcrossAMTCheckpointScenarios( + "snapshot replays trailing shared-DV update after an AMT checkpoint", + "amt_tail_dv_shared_blob", + sqlConfs = Seq(DeltaSQLConf.DELTA_HISTORY_METRICS_ENABLED.key -> "false"))( + setup = name => { + Seq(1, 2).toDF("id").coalesce(1).write.mode("append").insertInto(name) + Seq(3, 4).toDF("id").coalesce(1).write.mode("append").insertInto(name) + }, + inlineCheckpointTriggerActionsOrSQL = Some(name => Right( + s"INSERT INTO $name VALUES (5)"))) { context => + val log = context.postCheckpointSnapshot.deltaLog + val addFilesAtSetupVersion = context.postSetupSnapshot.allFiles.collect() + assert(addFilesAtSetupVersion.length == 2) + + // Attach DVs to both setup files. + val dvActions = writeFilesWithDVsOnDisk(log, Seq( + addFilesAtSetupVersion(0) -> RoaringBitmapArray(0L), + addFilesAtSetupVersion(1) -> RoaringBitmapArray(0L))) + log.startTransaction().commit(dvActions, DeltaOperations.Delete(predicate = Seq.empty)) + val finalSnapshot = latestSnapshotAfterLogTail(context) + + checkAnswer(spark.read.table(context.tableName), Seq(Row(2), Row(4), Row(5))) + val addFilesInFinalVersion = finalSnapshot.allFiles.collect() + assert(addFilesInFinalVersion.length == 3, + "Both DV'd files and the trigger file must remain live.") + val filesWithDvs = addFilesInFinalVersion.filter(_.deletionVector != null) + assert(filesWithDvs.length == 2) + assert(filesWithDvs.forall(_.numLogicalRecords.contains(1L))) + val blobPaths = filesWithDvs.map(_.deletionVector.absolutePath(log.dataPath)).toSet + assert(blobPaths.size == 1) + } + + testAcrossAMTCheckpointScenarios( + "snapshot replays trailing remove of a file with DV in an AMT", + "amt_tail_remove_dv_in_amt", + sqlConfs = Seq(DeltaSQLConf.DELTA_HISTORY_METRICS_ENABLED.key -> "false"))( + setup = name => { + Seq(10, 20, 30, 40, 50).toDF("id").coalesce(1) + .write.mode("append").insertInto(name) + // Attach a DV and checkpoint with AMT. + val log = deltaLogForName(name) + val fileToDv = log.update().allFiles.collect() + assert(fileToDv.length == 1, "The five setup rows must land in a single file.") + val dvActions = writeFileWithDVOnDisk(log, fileToDv.head, RoaringBitmapArray(0L, 2L, 4L)) + log.startTransaction().commit(dvActions, DeltaOperations.Delete(predicate = Seq.empty)) + }, + inlineCheckpointTriggerActionsOrSQL = Some(name => Right( + s"INSERT INTO $name VALUES (60)"))) { context => + val log = context.postCheckpointSnapshot.deltaLog + val addFilesAtCheckpointVersion = context.postCheckpointSnapshot.allFiles.collect() + assert(addFilesAtCheckpointVersion.length == 2) + + // AMT converts the DV type from u to r when reading it back. + val addFilesAtSetupVersion = context.postSetupSnapshot.allFiles.collect() + assert(addFilesAtSetupVersion.length == 1) + val setupFileAtSetupVersion = addFilesAtSetupVersion.head + assert(setupFileAtSetupVersion.deletionVector.storageType == + DeletionVectorDescriptor.UUID_DV_MARKER) + val addFilesInAmt = + amtFilesInTree(context.postCheckpointSnapshot, Some(setupFileAtSetupVersion.path)) + assert(addFilesInAmt.head.deletionVector.storageType == + DeletionVectorDescriptor.RELATIVE_DV_MARKER) + + // Delete the setup file + log.startTransaction().commit( + Seq(setupFileAtSetupVersion.removeWithTimestamp()), + DeltaOperations.ManualUpdate) + val finalSnapshot = latestSnapshotAfterLogTail(context) + + assert( + amtFilesInTree(finalSnapshot).map(_.path).toSet == + addFilesAtCheckpointVersion.map(_.path).toSet) + val livePaths = finalSnapshot.allFiles.collect().map(_.path).toSet + assert(!livePaths.contains(setupFileAtSetupVersion.path)) + assert(livePaths.size == 1) + checkAnswer(spark.read.table(context.tableName), Seq(Row(60))) + } + + testAcrossAMTCheckpointScenarios( + "snapshot replays trailing DV update of a file with DV in an AMT", + "amt_tail_update_dv_in_amt", + sqlConfs = Seq(DeltaSQLConf.DELTA_HISTORY_METRICS_ENABLED.key -> "false"))( + setup = name => { + Seq(10, 20, 30, 40, 50).toDF("id").coalesce(1) + .write.mode("append").insertInto(name) + // Attach a DV and checkpoint with AMT. + val log = deltaLogForName(name) + val fileToDv = log.update().allFiles.collect() + assert(fileToDv.length == 1, "The five setup rows must land in a single file.") + val dvActions = writeFileWithDVOnDisk(log, fileToDv.head, RoaringBitmapArray(0L, 2L, 4L)) + log.startTransaction().commit(dvActions, DeltaOperations.Delete(predicate = Seq.empty)) + }, + inlineCheckpointTriggerActionsOrSQL = Some(name => Right( + s"INSERT INTO $name VALUES (60)"))) { context => + val log = context.postCheckpointSnapshot.deltaLog + val addFilesAtCheckpointVersion = context.postCheckpointSnapshot.allFiles.collect() + assert(addFilesAtCheckpointVersion.length == 2) + + // The checkpointed file's DV is committed in u form but stored in the leaf in r form. + val addFilesAtSetupVersion = context.postSetupSnapshot.allFiles.collect() + assert(addFilesAtSetupVersion.length == 1) + val setupFileAtSetupVersion = addFilesAtSetupVersion.head + assert(setupFileAtSetupVersion.deletionVector.storageType == + DeletionVectorDescriptor.UUID_DV_MARKER) + + // Supersede the checkpointed DV with a wider one + val dvActions = + writeFileWithDVOnDisk(log, setupFileAtSetupVersion, RoaringBitmapArray(0L, 1L, 2L, 4L)) + log.startTransaction().commit(dvActions, DeltaOperations.Delete(predicate = Seq.empty)) + val finalSnapshot = latestSnapshotAfterLogTail(context) + + checkAnswer(spark.read.table(context.tableName), Seq(Row(40), Row(60))) + val addFilesInFinalVersion = finalSnapshot.allFiles.collect() + assert(addFilesInFinalVersion.length == addFilesAtCheckpointVersion.length) + val setupFileAtFinalVersion = addFilesInFinalVersion + .find(_.path == setupFileAtSetupVersion.path) + .getOrElse(fail(s"The DV-updated file ${setupFileAtSetupVersion.path} must stay live.")) + assert(setupFileAtFinalVersion.deletionVector != null) + assert(setupFileAtFinalVersion.deletionVector.cardinality == 4L) + assert(setupFileAtFinalVersion.numPhysicalRecords.contains(5L)) + assert(setupFileAtFinalVersion.numLogicalRecords.contains(1L)) + } + + testAcrossAMTCheckpointScenarios( + "snapshot exposes trailing AMTPassthrough add after an AMT checkpoint", + "amt_tail_passthrough")( + setup = name => sql(s"INSERT INTO $name VALUES (1)"), + inlineCheckpointTriggerActionsOrSQL = Some(name => Right( + s"INSERT INTO $name VALUES (2)"))) { context => + val log = context.postCheckpointSnapshot.deltaLog + val path = "trailing-passthrough-file" + val add = createTestAddFile(encodedPath = path) + .copy(amtPassthrough = Some(fullPassthrough), stats = """{"numRecords":1}""") + + log.startTransaction().commit(Seq(add), DeltaOperations.ManualUpdate) + val finalSnapshot = latestSnapshotAfterLogTail(context) + + assert(amtFilesInTree(finalSnapshot, Some(path)).isEmpty) + val live = finalSnapshot.allFiles.collect().filter(_.path == path) + assert(live.length == 1) + assert(live.head.amtPassthrough.contains(fullPassthrough)) } testAcrossAMTCheckpointScenarios( @@ -97,22 +703,22 @@ class AMTSnapshotSuite extends AMTCheckpointTestBase with DeletionVectorsTestUti } testAcrossAMTCheckpointScenarios( - "deletion vector round-trips through the leaves with a matching uniqueId", + "deletion vector round-trips through the leaves as a relative-path DV", "amt_dv", sqlConfs = Seq(DeltaSQLConf.DELTA_HISTORY_METRICS_ENABLED.key -> "false"))( setup = name => { - Seq(1, 2).toDF("id").coalesce(1) - .write.mode("append").insertInto(name) // one file, two rows. + Seq(10, 20, 30, 40, 50).toDF("id").coalesce(1) + .write.mode("append").insertInto(name) // Attach a persistent DV directly rather than relying on DELETE's rewrite heuristic. val log = deltaLogForName(name) val fileToDv = log.unsafeVolatileSnapshot.allFiles.collect() - assert(fileToDv.length == 1, "The two rows must land in a single file.") - val dvActions = writeFileWithDVOnDisk(log, fileToDv.head, RoaringBitmapArray(0L)) + assert(fileToDv.length == 1, "The five rows must land in a single file.") + val dvActions = writeFileWithDVOnDisk(log, fileToDv.head, RoaringBitmapArray(0L, 2L, 4L)) log.startTransaction().commit(dvActions, DeltaOperations.Delete(predicate = Seq.empty)) }) { context => val snapshot = context.postCheckpointSnapshot val provider = context.provider - checkAnswer(spark.read.table(context.tableName), Seq(Row(2))) + checkAnswer(spark.read.table(context.tableName), Seq(Row(20), Row(40))) // The one surviving live file must carry a deletion vector in committed state. val committed = snapshot.allFiles.collect() @@ -120,14 +726,10 @@ class AMTSnapshotSuite extends AMTCheckpointTestBase with DeletionVectorsTestUti val committedFile = committed.head val committedDv = committedFile.deletionVector assert(committedDv != null, "The committed file must carry a deletion vector.") - // Two physical rows, one deleted by the DV -> one logical row. - assert(committedFile.numPhysicalRecords.contains(2L)) - assert(committedFile.numLogicalRecords.contains(1L)) - - // The leaf-reconstructed AddFile must recover a DV with the SAME uniqueId, so the - // (path, deletionVectorUniqueId) dedup key matches the committed file exactly (no - // double-count in the reader path). uniqueId is - // storageType + pathOrInlineDv (+ "@offset"), so compare it from the reconstructed fields. + // Five physical rows, three deleted by the DV -> two logical rows. + assert(committedFile.numPhysicalRecords.contains(5L)) + assert(committedFile.numLogicalRecords.contains(2L)) + val dvRow = provider .loadActionsForStateReconstruction(spark, snapshot.deltaLog) .getOrElse(fail("AMT provider must contribute leaf-derived file actions.")) @@ -142,9 +744,10 @@ class AMTSnapshotSuite extends AMTCheckpointTestBase with DeletionVectorsTestUti offset = Option(dvRow.head.get(2)).map(_ => dvRow.head.getInt(2)), sizeInBytes = committedDv.sizeInBytes, cardinality = committedDv.cardinality) - assert(reconstructedDv.uniqueId == committedDv.uniqueId, - s"Reconstructed DV uniqueId ${reconstructedDv.uniqueId} must equal committed " + - s"${committedDv.uniqueId}.") + val tableRoot = snapshot.deltaLog.dataPath + assert(reconstructedDv.storageType == DeletionVectorDescriptor.RELATIVE_DV_MARKER) + assert(reconstructedDv.absolutePath(tableRoot) == committedDv.absolutePath(tableRoot)) + assert(reconstructedDv.offset == committedDv.offset) // The leaf-reconstructed AddFile must recover the same physical/logical record counts as the // committed file: `numRecords` in stats is the physical count, so it must round-trip without @@ -158,33 +761,245 @@ class AMTSnapshotSuite extends AMTCheckpointTestBase with DeletionVectorsTestUti .as[AddFile] .collect() assert(reconstructed.length == 1) - assert(reconstructed.head.numPhysicalRecords.contains(2L), + assert(reconstructed.head.numPhysicalRecords.contains(5L), "Reconstructed physical record count must match the committed file.") - assert(reconstructed.head.numLogicalRecords.contains(1L), + assert(reconstructed.head.numLogicalRecords.contains(2L), "Reconstructed logical record count must match the committed file.") } + testAcrossAMTCheckpointScenarios( + "DV that fully deletes a file", + "amt_dv_full_delete", + sqlConfs = Seq(DeltaSQLConf.DELTA_HISTORY_METRICS_ENABLED.key -> "false"))( + setup = name => { + Seq(1, 2).toDF("id").coalesce(1).write.mode("append").insertInto(name) + val log = deltaLogForName(name) + val fileToDv = log.unsafeVolatileSnapshot.allFiles.collect() + assert(fileToDv.length == 1) + val dvActions = writeFileWithDVOnDisk(log, fileToDv.head, RoaringBitmapArray(0L, 1L)) + log.startTransaction().commit(dvActions, DeltaOperations.Delete(predicate = Seq.empty)) + }) { context => + val snapshot = context.postCheckpointSnapshot + checkAnswer(spark.read.table(context.tableName), Seq.empty) + + val committed = snapshot.allFiles.collect() + assert(committed.length == 1, "The fully-deleted file must stay live.") + assert(committed.head.numPhysicalRecords.contains(2L)) + assert(committed.head.numLogicalRecords.contains(0L)) + + // The tree reconstructs the same file. + val reconstructed = amtFilesInTree(snapshot) + assert(reconstructed.length == 1, "The fully-deleted file must stay live.") + val dv = reconstructed.head.deletionVector + assert(dv != null) + assert(dv.cardinality == 2L) + assert(reconstructed.head.numPhysicalRecords.contains(2L)) + assert(reconstructed.head.numLogicalRecords.contains(0L)) + } + + testAcrossAMTCheckpointScenarios( + "two files sharing one DV file on disk", + "amt_dv_shared_blob", + sqlConfs = Seq(DeltaSQLConf.DELTA_HISTORY_METRICS_ENABLED.key -> "false"))( + setup = name => { + Seq(1, 2).toDF("id").coalesce(1).write.mode("append").insertInto(name) + Seq(3, 4).toDF("id").coalesce(1).write.mode("append").insertInto(name) + val log = deltaLogForName(name) + val files = log.unsafeVolatileSnapshot.allFiles.collect() + assert(files.length == 2) + val dvActions = writeFilesWithDVsOnDisk(log, Seq( + files(0) -> RoaringBitmapArray(0L), + files(1) -> RoaringBitmapArray(0L))) + log.startTransaction().commit(dvActions, DeltaOperations.Delete(predicate = Seq.empty)) + }) { context => + val snapshot = context.postCheckpointSnapshot + // Each file drops its row 0 and the row-1 values (2 and 4) survive. + checkAnswer(spark.read.table(context.tableName), Seq(Row(2), Row(4))) + + // Both DV'd files stay live, each with one logical row left. + val committed = snapshot.allFiles.collect() + assert(committed.length == 2, "Both DV'd files must remain live.") + assert(committed.forall(_.numLogicalRecords.contains(1L))) + + // The tree reconstructs the same files. + val tableRoot = snapshot.deltaLog.dataPath + val reconstructed = amtFilesInTree(snapshot) + assert(reconstructed.length == 2) + val dvs = reconstructed.map(_.deletionVector) + assert(dvs.forall(dv => dv != null)) + // Both AddFiles reference the same DV file on disk. + val blobPaths = dvs.map(_.absolutePath(tableRoot)).toSet + assert(blobPaths.size == 1) + // Offsets are still different. + val offsets = dvs.map(_.offset).toSet + assert(offsets.size == 2) + } + + testAcrossAMTCheckpointScenarios( + "snapshot replays trailing remove of a shared checkpointed DV file", + "amt_tail_remove_checkpointed_dv", + sqlConfs = Seq( + DeltaSQLConf.DELTA_HISTORY_METRICS_ENABLED.key -> "false" + ))( + setup = name => { + Seq(1, 2).toDF("id").coalesce(1).write.mode("append").insertInto(name) + Seq(3, 4).toDF("id").coalesce(1).write.mode("append").insertInto(name) + val log = deltaLogForName(name) + val files = log.unsafeVolatileSnapshot.allFiles.collect() + assert(files.length == 2) + + val dvActions = writeFilesWithDVsOnDisk(log, Seq( + files(0) -> RoaringBitmapArray(0L), + files(1) -> RoaringBitmapArray(0L))) + log.startTransaction().commit(dvActions, DeltaOperations.Delete(predicate = Seq.empty)) + }, + inlineCheckpointTriggerActionsOrSQL = Some(name => Right( + s"INSERT INTO $name VALUES (5)"))) { context => + val log = context.postCheckpointSnapshot.deltaLog + val addFilesAtSetupVersion = context.postSetupSnapshot.allFiles.collect() + .filter(_.deletionVector != null) + assert(addFilesAtSetupVersion.length == 2) + val addFilesAtCheckpointVersion = context.postCheckpointSnapshot.allFiles.collect() + assert(addFilesAtCheckpointVersion.count(_.deletionVector != null) == 2) + assert(addFilesAtCheckpointVersion.filter(_.deletionVector != null).forall( + _.deletionVector.storageType == DeletionVectorDescriptor.RELATIVE_DV_MARKER)) + val checkpointRowCount = spark.read.table(context.tableName).count() + + val firstFile = addFilesAtSetupVersion.head + log.startTransaction().commit( + Seq(firstFile.removeWithTimestamp()), + DeltaOperations.ManualUpdate) + val finalSnapshot = latestSnapshotAfterLogTail(context) + + val addFilesInAmt = amtFilesInTree(finalSnapshot) + assert(addFilesInAmt.length == addFilesAtCheckpointVersion.length, + "The AMT provider should still expose every checkpointed file.") + val addFilesInFinalVersion = finalSnapshot.allFiles.collect() + assert(addFilesInFinalVersion.length == addFilesAtCheckpointVersion.length - 1, + "Only one file left after the last remove.") + val liveFiles = addFilesInFinalVersion.filter(_.deletionVector != null) + assert(liveFiles.length == 1) + assert( + liveFiles.head.deletionVector.absolutePath(log.dataPath) == + firstFile.deletionVector.absolutePath(log.dataPath), + "The surviving file should share the same DV path as the removed file.") + assert(liveFiles.head.deletionVector.offset != firstFile.deletionVector.offset) + checkAnswer( + spark.read.table(context.tableName).groupBy().count(), + Seq(Row(checkpointRowCount - 1))) + } + + testAcrossAMTCheckpointScenarios( + "DVs on files across multiple leaf manifests", + "amt_dv_multi_leaf", + sqlConfs = Seq( + DeltaSQLConf.AMT_ENTRIES_PER_LEAF.key -> "2", + DeltaSQLConf.DELTA_HISTORY_METRICS_ENABLED.key -> "false"))( + setup = name => { + // Four two-row files + Seq((10, 11), (20, 21), (30, 31), (40, 41)).foreach { case (a, b) => + Seq(a, b).toDF("id").coalesce(1).write.mode("append").insertInto(name) + } + val log = deltaLogForName(name) + val files = log.unsafeVolatileSnapshot.allFiles.collect() + assert(files.length == 4) + val dvActions = writeFilesWithDVsOnDisk(log, files.map(_ -> RoaringBitmapArray(0L)).toSeq) + log.startTransaction().commit(dvActions, DeltaOperations.Delete(predicate = Seq.empty)) + }) { context => + val snapshot = context.postCheckpointSnapshot + checkAnswer(spark.read.table(context.tableName), Seq(Row(11), Row(21), Row(31), Row(41))) + + val committed = snapshot.allFiles.collect() + assert(committed.length == 4, "All four DV'd files remain live.") + assert(committed.forall(_.numLogicalRecords.contains(1L))) + + // The tree reconstructs the same files. + val reconstructed = amtFilesInTree(snapshot) + assert(reconstructed.length == 4) + val filesWithDvs = reconstructed.filter(_.deletionVector != null) + assert(filesWithDvs.length == 4) + } + + test("DV with shallow cloned table") { + withSQLConf(DeltaSQLConf.DELTA_HISTORY_METRICS_ENABLED.key -> "false") { + withTable("amt_dv_clone_src", "amt_dv_clone_tgt") { + val src = "amt_dv_clone_src" + createAMTTable(src, checkpointInterval = 100) + Seq(1, 2).toDF("id").coalesce(1).write.mode("append").insertInto(src) + val srcLog = deltaLogForName(src) + val srcFile = srcLog.unsafeVolatileSnapshot.allFiles.collect() + assert(srcFile.length == 1, "The two rows must land in a single file.") + val dvActions = writeFileWithDVOnDisk(srcLog, srcFile.head, RoaringBitmapArray(0L)) + srcLog.startTransaction().commit(dvActions, DeltaOperations.Delete(predicate = Seq.empty)) + + val sourceJsonFiles = srcLog.update().allFiles.collect() + assert(sourceJsonFiles.length == 1) + assert(sourceJsonFiles.head.deletionVector != null) + assert(sourceJsonFiles.head.deletionVector.storageType == + DeletionVectorDescriptor.UUID_DV_MARKER) + + // Materialize the source's AMT before cloning. AMT stores an in-root DV location as a + // relative path, so source allFiles reconstructs the on-disk DV as an r descriptor. + commitCheckpoint(srcLog, incremental = false) + val sourceAMTSnapshot = srcLog.update() + amtProvider(sourceAMTSnapshot).getOrElse( + fail("the clone source must have an AMTCheckpointProvider.")) + val sourceAMTFiles = sourceAMTSnapshot.allFiles.collect() + assert(sourceAMTFiles.length == 1) + assert(sourceAMTFiles.head.deletionVector != null) + assert(sourceAMTFiles.head.deletionVector.storageType == + DeletionVectorDescriptor.RELATIVE_DV_MARKER) + + val tgt = "amt_dv_clone_tgt" + sql(s"CREATE TABLE $tgt SHALLOW CLONE $src") + + val tgtLog = deltaLogForName(tgt) + val clonedFiles = tgtLog.update().allFiles.collect() + assert(clonedFiles.length == 1) + val clonedDv = clonedFiles.head.deletionVector + assert(clonedDv != null) + assert(clonedDv.storageType == DeletionVectorDescriptor.PATH_DV_MARKER, + "A shallow-cloned AMT DV must be stored as an absolute p DV") + + // Materialize the target's AMT. + commitCheckpoint(tgtLog, incremental = false) + + val snapshot = tgtLog.update() + amtProvider(snapshot).getOrElse( + fail("the clone target must have an AMTCheckpointProvider.")) + checkAnswer(spark.read.table(tgt), Seq(Row(2))) + + val reconstructed = amtFilesInTree(snapshot) + assert(reconstructed.length == 1) + val dv = reconstructed.head.deletionVector + assert(dv != null) + assert(dv.storageType == DeletionVectorDescriptor.PATH_DV_MARKER, + "An outside-root DV must reconstruct as a p DV") + // The DV blob physically lives outside the target's root. + val tgtRoot = tgtLog.dataPath + val dvPath = dv.absolutePath(tgtRoot).toString + assert(!dvPath.startsWith(tgtRoot.toString)) + } + } + } + testAcrossAMTCheckpointScenarios( "distributed reconstruction returns every action exactly once", "amt_dist_reconstruction", - sqlConfs = Seq(DeltaSQLConf.AMT_ENTRIES_PER_LEAF.key -> "10"))( + sqlConfs = leafPackingConfs)( setup = name => { - appendRowsAsSeparateFiles(name, 30) // ids 0..29, one file each. - appendRowsAsSeparateFiles(name, 29, startId = 30) // ids 30..58, one file each. + appendRowsAsSeparateFiles(name, leafPackedFiles) + appendRowsAsSeparateFiles(name, leafPackedFiles - 1, startId = leafPackedFiles) }, inlineCheckpointTriggerActionsOrSQL = Some(name => Right( - s"INSERT INTO $name VALUES (59)"))) { context => + s"INSERT INTO $name VALUES (${2 * leafPackedFiles - 1})"))) { context => val snapshot = context.postCheckpointSnapshot val provider = context.provider - // `setup` accumulates ids 0..58 over two commits and the trigger commit adds id 59, so the - // checkpointed tree is built from files that arrived across several commits. - val expectedRows = 0 until 60 + val expectedRows = 0 until 2 * leafPackedFiles checkAnswer(spark.read.table(context.tableName), expectedRows.map(Row(_))) val committedPaths = snapshot.allFiles.select("path").as[String].collect().toSet assert(committedPaths.size == expectedRows.size) - // 60 live files at ten entries per leaf must pack into exactly six leaves. - assert(provider.leaves.size == 6, - s"60 files at entriesPerLeaf=10 must pack into 6 leaves; got ${provider.leaves.size}.") + assertLeafCount(provider.leaves, numFiles = 2 * leafPackedFiles) val df = provider.loadActionsForStateReconstruction(spark, snapshot.deltaLog) .getOrElse(fail("AMT provider must contribute leaf-derived actions.")) @@ -199,7 +1014,7 @@ class AMTSnapshotSuite extends AMTCheckpointTestBase with DeletionVectorsTestUti // Every leaf entry must surface exactly once across the leaves, plus the inline protocol and // metadata actions. - assertReconstructsLiveFileSet(context) + assertLiveAddFilesRoundTrip(context) assert(df.where("protocol.minReaderVersion is not null").count() == 1, "Reconstruction must carry the inline protocol action.") assert(df.where("metaData.id is not null").count() == 1, @@ -226,6 +1041,49 @@ class AMTSnapshotSuite extends AMTCheckpointTestBase with DeletionVectorsTestUti }.isDefined, "Reconstruction must read leaves through a distributed file scan.") } + test("RESTORE reconciles u / r DV of the same blob by object identity") { + withTable("amt_restore_dv") { + val t = "amt_restore_dv" + createAMTTable(t, checkpointInterval = 100) + Seq(1, 2).toDF("id").coalesce(1).write.mode("append").insertInto(t) + + val log = deltaLogForName(t) + val filesV1 = log.update().allFiles.collect() + assert(filesV1.length == 1, "the two rows must land in a single file") + + // Commit an u DV. + val dvActions = writeFileWithDVOnDisk(log, filesV1.head, RoaringBitmapArray(0L)) + withSQLConf(DeltaSQLConf.DELTA_HISTORY_METRICS_ENABLED.key -> "false") { + log.startTransaction().commit(dvActions, DeltaOperations.Delete(predicate = Nil)) + } + val dvVersion = log.update().version + val jsonDv = log.update().allFiles.collect().head.deletionVector + assert(jsonDv != null && jsonDv.storageType == DeletionVectorDescriptor.UUID_DV_MARKER, + "the DV committed to the log must be in `u` form") + + // Materialize an AMT full checkpoint. + commitCheckpoint(log, incremental = false) + val latest = log.unsafeVolatileSnapshot + amtProvider(latest).getOrElse( + fail("latest snapshot must be AMT-backed after the checkpoint")) + val amtDv = latest.allFiles.collect().head.deletionVector + assert(amtDv != null && amtDv.storageType == DeletionVectorDescriptor.RELATIVE_DV_MARKER, + "after the checkpoint the same blob must be read back in `r` form") + assert(dvVersion < latest.version, "there must be a newer version to restore from") + + // Restore back to the u DV. We assert the restore commit is empty. + locally { + val metrics = sql(s"RESTORE TABLE $t VERSION AS OF $dvVersion") + .select("num_restored_files", "num_removed_files") + .head() + assert(metrics.getLong(0) == 0L && metrics.getLong(1) == 0L) + + // The restored table still reads the single surviving logical row. + checkAnswer(spark.table(t), Seq(Row(2))) + } + } + } + test("reconstruction surfaces DATA entries that live directly in the root") { withTable("amt_root_data") { val name = "amt_root_data" @@ -250,7 +1108,8 @@ class AMTSnapshotSuite extends AMTCheckpointTestBase with DeletionVectorsTestUti val checkpoint = checkpointWithSyntheticRoot(deltaLog, provider.checkpointAction, rootDataRows) - val rootProvider = AMTCheckpointProvider.fromCheckpoint(spark, deltaLog, checkpoint) + val rootProvider = AMTCheckpointProvider.fromCheckpoint( + deltaLog, checkpoint, manifestCommitVersion = provider.manifestCommitVersion) assert(rootProvider.leaves.isEmpty, "A DATA-only root must yield no leaf pointers.") val expected = rootAdds.map(_.path).toSet @@ -269,14 +1128,14 @@ class AMTSnapshotSuite extends AMTCheckpointTestBase with DeletionVectorsTestUti testAcrossAMTCheckpointScenarios( "manifest deletion vector drops superseded leaf entries during reconstruction", "amt_mdv_drop", - sqlConfs = mdvLeafResidentConfs)( - setup = name => (1 to 3).foreach(i => sql(s"INSERT INTO $name VALUES ($i)")), + sqlConfs = leafPackingConfs)( + setup = name => appendRowsAsSeparateFiles(name, numFiles = leafPackedFiles - 1), inlineCheckpointTriggerActionsOrSQL = Some(name => Right( - s"INSERT INTO $name VALUES (4)"))) { context => + s"INSERT INTO $name VALUES (${leafPackedFiles - 1})"))) { context => val snapshot = context.postCheckpointSnapshot val base = context.provider val baseCount = snapshot.allFiles.count() - assert(baseCount == 4) + assert(baseCount == leafPackedFiles, "The seeded files plus the trigger's.") val leaf = base.leaves.head val posToLoc = leafPosToLoc(leaf, base.tableRoot) @@ -293,8 +1152,8 @@ class AMTSnapshotSuite extends AMTCheckpointTestBase with DeletionVectorsTestUti leaf.copy(manifest_info = leaf.manifest_info.copy(dv = Some(mdvBytesFor(deletedPos)), dv_cardinality = Some(1L))) +: base.leaves.tail - val provider = - new AMTCheckpointProvider(base.checkpointAction, patchedLeaves, base.tableRoot) + val provider = new AMTCheckpointProvider( + base.manifestCommitVersion, base.checkpointAction, patchedLeaves, base.tableRoot) val reconstructed = reconstructedPaths(provider, snapshot.deltaLog) assert(reconstructed.size == baseCount - 1, @@ -308,19 +1167,18 @@ class AMTSnapshotSuite extends AMTCheckpointTestBase with DeletionVectorsTestUti testAcrossAMTCheckpointScenarios( "manifest deletion vectors apply per leaf across a multi-leaf tree", "amt_mdv_multi", - sqlConfs = Seq(DeltaSQLConf.AMT_ENTRIES_PER_LEAF.key -> "10"))( + sqlConfs = leafPackingConfs)( setup = name => { - appendRowsAsSeparateFiles(name, 30) // ids 0..29, one file each. - appendRowsAsSeparateFiles(name, 29, startId = 30) // ids 30..58, one file each. + appendRowsAsSeparateFiles(name, leafPackedFiles) + appendRowsAsSeparateFiles(name, leafPackedFiles - 1, startId = leafPackedFiles) }, inlineCheckpointTriggerActionsOrSQL = Some(name => Right( - s"INSERT INTO $name VALUES (59)"))) { context => + s"INSERT INTO $name VALUES (${2 * leafPackedFiles - 1})"))) { context => val base = context.provider - assert(base.leaves.size == 6, - s"60 files with entriesPerLeaf=10 must pack into 6 leaves; got ${base.leaves.size}.") + assertLeafCount(base.leaves, numFiles = 2 * leafPackedFiles) val deltaLog = context.postCheckpointSnapshot.deltaLog val full = reconstructedPaths(base, deltaLog) - assert(full.size == 60) + assert(full.size == 2 * leafPackedFiles) // The leaf-local row-index -> entry-location map for each observed leaf. val locs = base.leaves.map(leafPosToLoc(_, base.tableRoot)) @@ -357,7 +1215,8 @@ class AMTSnapshotSuite extends AMTCheckpointTestBase with DeletionVectorsTestUti case _ => leaf } } - val provider = new AMTCheckpointProvider(base.checkpointAction, patched, base.tableRoot) + val provider = new AMTCheckpointProvider( + base.manifestCommitVersion, base.checkpointAction, patched, base.tableRoot) val dropped = twoPositions.map(locs(twoLeaf)).toSet ++ @@ -376,8 +1235,9 @@ class AMTSnapshotSuite extends AMTCheckpointTestBase with DeletionVectorsTestUti // the SQL operation metrics a real DELETE would populate are absent. History metrics would // call `Delete.transformMetrics`, which requires `numDeletedRows`; it is missing here and // would fail with `key not found: numDeletedRows`. Disable history metrics to skip that. - sqlConfs = mdvLeafResidentConfs :+ - (DeltaSQLConf.DELTA_HISTORY_METRICS_ENABLED.key -> "false"))( + sqlConfs = Seq( + DeltaSQLConf.AMT_ENTRIES_PER_LEAF.key -> entriesPerLeaf.toString, + DeltaSQLConf.DELTA_HISTORY_METRICS_ENABLED.key -> "false"))( setup = name => { // File A with two physical rows, then a persistent on-disk DV marking row 0 deleted. Seq(1, 2).toDF("id").coalesce(1).write.mode("append").insertInto(name) @@ -386,14 +1246,18 @@ class AMTSnapshotSuite extends AMTCheckpointTestBase with DeletionVectorsTestUti assert(fileA.length == 1, "The two rows must land in a single file.") val dvActions = writeFileWithDVOnDisk(log, fileA.head, RoaringBitmapArray(0L)) log.startTransaction().commit(dvActions, DeltaOperations.Delete(predicate = Seq.empty)) + // DV-less siblings, so the tree packs multi-entry leaves around file A. File A and the + // trigger's file make up the rest of the packed count. + appendRowsAsSeparateFiles(name, numFiles = leafPackedFiles - 2, startId = 10) }, inlineCheckpointTriggerActionsOrSQL = Some(name => Right( s"INSERT INTO $name VALUES (3)"))) { context => val snapshot = context.postCheckpointSnapshot val base = context.provider - // Classify the two reconstructed entries by DV presence, staying entirely within the - // reconstruction so path strings are consistent with the leaf `location` values. + // Classify the reconstructed entries by DV presence, staying entirely within the reconstruction + // so path strings are consistent with the leaf `location` values. Exactly one entry carries a + // DV (file A); the rest are the DV-less siblings the setup seeded. val fullAdds = base.loadActionsForStateReconstruction(spark, snapshot.deltaLog) .getOrElse(fail("AMT provider must contribute leaf-derived file actions.")) .where("add is not null") @@ -401,46 +1265,55 @@ class AMTSnapshotSuite extends AMTCheckpointTestBase with DeletionVectorsTestUti .collect() .map(r => r.getString(0) -> (Option(r.getString(1)), r.getString(2))) .toMap - assert(fullAdds.size == 2) - val aPath = fullAdds.collectFirst { case (p, (Some(_), _)) => p } - .getOrElse(fail("Expected one reconstructed entry to carry a DV.")) - val bPath = fullAdds.collectFirst { case (p, (None, _)) => p } - .getOrElse(fail("Expected one reconstructed entry without a DV.")) + assert(fullAdds.size == leafPackedFiles, + "One DV-bearing file plus the DV-less siblings.") + val withDv = fullAdds.filter(_._2._1.isDefined) + assert(withDv.size == 1, s"Exactly one reconstructed entry must carry a DV; got $withDv") + val aPath = withDv.head._1 val aStats = fullAdds(aPath)._2 - // Drop B (the DV-less sibling) via an MDV on whichever leaf holds it; A must survive untouched. - val bLeafAndPos = base.leaves.flatMap { leaf => - leafPosToLoc(leaf, base.tableRoot).collectFirst { case (pos, `bPath`) => leaf -> pos } - }.headOption.getOrElse(fail(s"No leaf holds the DV-less sibling $bPath.")) + // Drop one DV-less entry that shares file A's own leaf, so the MDV removes a true sibling from + // a multi-entry leaf while A sits beside it. + val aLeaf = base.leaves.find(leafPosToLoc(_, base.tableRoot).values.exists(_ == aPath)) + .getOrElse(fail(s"No leaf holds the DV-bearing file $aPath.")) + val aLeafEntries = leafPosToLoc(aLeaf, base.tableRoot) + val (bPos, bPath) = aLeafEntries.filter { case (_, loc) => loc != aPath }.head + val bLeafAndPos = (aLeaf, bPos) val patched = base.leaves.map { leaf => if (leaf eq bLeafAndPos._1) { leaf.copy(manifest_info = leaf.manifest_info.copy( dv = Some(mdvBytesFor(bLeafAndPos._2)), dv_cardinality = Some(1L))) } else leaf } - val provider = new AMTCheckpointProvider(base.checkpointAction, patched, base.tableRoot) + val provider = new AMTCheckpointProvider( + base.manifestCommitVersion, base.checkpointAction, patched, base.tableRoot) val survivors = provider.loadActionsForStateReconstruction(spark, snapshot.deltaLog) .getOrElse(fail("AMT provider must contribute leaf-derived file actions.")) .where("add is not null") .selectExpr("add.path", "add.deletionVector.storageType", "add.stats") .collect() - assert(survivors.length == 1, "Only the DV-bearing sibling must survive the MDV.") - assert(survivors.head.getString(0) == aPath, - "The surviving path must be the DV-bearing file.") - assert(survivors.head.getString(1) != null, + // Only the MDV-marked sibling is dropped; every other entry -- including file A, which shares + // its leaf -- survives with its deletion vector and stats intact. + assert(survivors.length == fullAdds.size - 1, + s"The MDV must drop exactly one entry; kept ${survivors.length} of ${fullAdds.size}.") + assert(!survivors.exists(_.getString(0) == bPath), + s"The MDV-marked sibling $bPath must not survive.") + val a = survivors.find(_.getString(0) == aPath) + .getOrElse(fail(s"The DV-bearing file $aPath must survive the MDV.")) + assert(a.getString(1) != null, "The surviving entry must retain its deletion vector after MDV filtering.") - assert(survivors.head.getString(2) == aStats, + assert(a.getString(2) == aStats, "The surviving entry's stats must be unchanged by MDV filtering.") } testAcrossAMTCheckpointScenarios( "a manifest DV with only one of dv/dv_cardinality set is rejected", "amt_mdv_malformed", - sqlConfs = mdvLeafResidentConfs)( - setup = name => (1 to 2).foreach(i => sql(s"INSERT INTO $name VALUES ($i)")), + sqlConfs = leafPackingConfs)( + setup = name => appendRowsAsSeparateFiles(name, numFiles = leafPackedFiles - 1), inlineCheckpointTriggerActionsOrSQL = Some(name => Right( - s"INSERT INTO $name VALUES (3)"))) { context => + s"INSERT INTO $name VALUES (${leafPackedFiles - 1})"))) { context => val base = context.provider // `dv` set but `dv_cardinality` missing: the AMT spec requires both or neither. @@ -448,7 +1321,8 @@ class AMTSnapshotSuite extends AMTCheckpointTestBase with DeletionVectorsTestUti base.leaves.head.copy(manifest_info = base.leaves.head.manifest_info.copy( dv = Some(mdvBytesFor(0L)), dv_cardinality = None)) +: base.leaves.tail - val provider = new AMTCheckpointProvider(base.checkpointAction, patched, base.tableRoot) + val provider = new AMTCheckpointProvider( + base.manifestCommitVersion, base.checkpointAction, patched, base.tableRoot) val e = intercept[IllegalStateException] { provider.loadActionsForStateReconstruction( @@ -465,17 +1339,21 @@ class AMTSnapshotSuite extends AMTCheckpointTestBase with DeletionVectorsTestUti .getOrElse(fail("AMT provider must contribute leaf-derived file actions.")) .where("add is not null").select("add.path").as[String].collect().toSet - /** Maps a leaf's parquet row positions to their entry `location`s (bypasses the format check). */ + /** Maps a leaf's parquet row positions to the encoded AddFile paths they reconstruct as. */ private def leafPosToLoc(leaf: DataManifestEntry, tableRoot: Path): Map[Long, String] = - { + allowReadWithinDeltaLog { spark.read.parquet(leaf.getAbsolutePath(tableRoot).toString) .select(col("_metadata.row_index").as("pos"), col("location")) - .as[(Long, String)].collect().toMap + .as[(Long, String)] + .collect() + .map { case (pos, location) => + pos -> SparkPath.fromPathString(location).urlEncoded + }.toMap } /** Serializes a manifest DV bitmap over the given leaf entry positions. */ private def mdvBytesFor(positions: Long*): Array[Byte] = - AMTUtils.serializeMdv(RoaringBitmapArray(positions: _*)) + AMTUtils.serializeMdv(ManifestBitmap.fromPositions(positions.map(_.toInt))) /** An ADDED tracking envelope with no lineage/sequence numbers, matching the AMT writer. */ private def addedTracking: Tracking = Tracking( @@ -499,9 +1377,21 @@ class AMTSnapshotSuite extends AMTCheckpointTestBase with DeletionVectorsTestUti val rootFile = FileNames.newAMTRootManifestFile(metadataDir) val useRename = deltaLog.store.isPartialWriteVisible(deltaLog.logPath, hadoopConf) val enc = org.apache.spark.sql.delta.implicits.amtSingleActionEncoder - val df = spark.createDataset(rows)(enc).toDF() - Checkpoints.writeAtomicCheckpointParquetFile(spark, df, rootFile, hadoopConf, useRename) + val metadata = base.metaData + val protocol = base.protocol + val withPartition = AMTPartitionValues.forWrite( + spark.createDataset(rows)(enc).toDF(), metadata.partitionSchema) + val df = AMTContentStats.forWrite(withPartition, metadata, protocol) + Checkpoints.writeAtomicCheckpointParquetFile( + spark, + df, + rootFile, + hadoopConf, + useRename, + outputSchema = Some(AMTSingleAction.persistedSchema(metadata, protocol)), + format = new AMTParquetFileFormat) val size = rootFile.getFileSystem(hadoopConf).getFileStatus(rootFile).getLen - base.copy(contentRoot = ContentRoot(path = rootFile.toString, sizeInBytes = size)) + base.copy(contentRoot = ContentRoot( + path = rootFile.toString, sizeInBytes = size, version = base.version)) } } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTUtilsSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTUtilsSuite.scala index 6cf79fe92b5..7edd6c1fb4a 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTUtilsSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTUtilsSuite.scala @@ -16,34 +16,95 @@ package org.apache.spark.sql.delta.amt -import org.apache.hadoop.conf.Configuration +import org.apache.spark.sql.delta.AdaptiveMetadataTableFeature +import org.apache.spark.sql.delta.actions.{Metadata, Protocol} import org.apache.hadoop.fs.Path -import org.apache.spark.SparkFunSuite +import org.apache.spark.sql.QueryTest +import org.apache.spark.sql.test.SharedSparkSession -class AMTUtilsSuite extends SparkFunSuite { +class AMTUtilsSuite extends QueryTest with SharedSparkSession { - private val tableRoot = new Path("file:/tables/t1") - private val fs = tableRoot.getFileSystem(new Configuration()) + test("invariantCheckWithLogging: succeeds when the invariant holds") { + AMTUtils.invariantCheckWithLogging( + checkInvariant = true, + opTypeSuffix = AMTUsageLogs.ALERT_MIXED_LEAF_CONTENT, + message = "valid AMT invariant") + } - test("relativizeManifestPathToTableRoot: a file under the table root becomes relative") { - val leaf = new Path("file:/tables/t1/metadata/leaf-1.parquet") - assert(AMTUtils.relativizeManifestPathToTableRoot(fs, tableRoot, leaf) === - "metadata/leaf-1.parquet") + + test("invariantCheckWithLogging: evaluates the check once") { + var checkCount = 0 + AMTUtils.invariantCheckWithLogging( + checkInvariant = { + checkCount += 1 + true + }, + opTypeSuffix = AMTUsageLogs.ALERT_MIXED_LEAF_CONTENT, + message = "valid AMT invariant") + assert(checkCount === 1) + } + + test("hasScheme: follows URI scheme grammar") { + assert(AMTUtils.hasScheme("s3://bucket/path")) + assert(AMTUtils.hasScheme("file:/tmp/table")) + assert(AMTUtils.hasScheme("git+ssh://host/repo")) + assert(AMTUtils.hasScheme("ab.c-1+2://host/repo")) + + assert(!AMTUtils.hasScheme("data/partition=key:value/file.parquet")) + assert(!AMTUtils.hasScheme("metadata/snap-123:456.avro")) + assert(!AMTUtils.hasScheme("3com://host")) + assert(!AMTUtils.hasScheme("+ssh://host")) + assert(!AMTUtils.hasScheme(".bar://host")) + } + + test("isAbsoluteLocation: detects absolute locations without constructing Path") { + assert(AMTUtils.isAbsoluteLocation("s3://bucket/path")) + assert(AMTUtils.isAbsoluteLocation("file:/tmp/table")) + assert(AMTUtils.isAbsoluteLocation("git+ssh://host/repo")) + assert(AMTUtils.isAbsoluteLocation("/tmp/table")) + + assert(!AMTUtils.isAbsoluteLocation("metadata/file.parquet")) + assert(!AMTUtils.isAbsoluteLocation("data/partition=key:value/file.parquet")) + assert(!AMTUtils.isAbsoluteLocation("metadata/snap-123:456.avro")) + assert(!AMTUtils.isAbsoluteLocation("3com://host")) } - test("relativizeManifestPathToTableRoot: a file outside the table root stays absolute") { - val outside = new Path("file:/other/metadata/leaf-1.parquet") - assert(AMTUtils.relativizeManifestPathToTableRoot(fs, tableRoot, outside) === - "file:/other/metadata/leaf-1.parquet") + test("relativizeLocation: child locations under the table location become relative") { + val root = "s3://bucket/db/table" + Seq( + "s3://bucket/db/table/metadata/file.parquet" -> "metadata/file.parquet", + "s3://bucket/db/table/data/00000-0.parquet" -> "data/00000-0.parquet" + ).foreach { case (location, expected) => + assert(AMTUtils.relativizeLocation(root, location) === expected) + } } - test("relativizeManifestPathToTableRoot: result is raw, not URL-encoded") { - val leaf = new Path("file:/tables/t1/metadata/leaf a.parquet") - val relative = AMTUtils.relativizeManifestPathToTableRoot(fs, tableRoot, leaf) - assert(relative === "metadata/leaf a.parquet", - s"space must stay raw, not percent-encoded; got $relative") - assert(!relative.contains("%20")) + test("relativizeLocation: non-child locations stay unchanged") { + val root = "s3://bucket/db/table" + Seq( + "s3://other-bucket/db/table/data/file.parquet", + "s3://bucket/db/other-table/data/file.parquet", + "s3://bucket/db/table_v2/data/00000-0.parquet", + root + ).foreach { location => + assert(AMTUtils.relativizeLocation(root, location) === location) + } + } + + test("relativizeLocation: mismatched file URI forms stay unchanged") { + assert(AMTUtils.relativizeLocation( + "file:/tmp/table", "file:///tmp/table/metadata/file.parquet") === + "file:///tmp/table/metadata/file.parquet") + assert(AMTUtils.relativizeLocation( + "file:///tmp/table", "file:/tmp/table/metadata/file.parquet") === + "file:/tmp/table/metadata/file.parquet") + } + + test("relativizeLocation: trailing slash table location matches child") { + assert(AMTUtils.relativizeLocation( + "s3://bucket/db/table/", "s3://bucket/db/table/data/00000-0.parquet") === + "data/00000-0.parquet") } test("absolutePathForManifestFile: a relative location joins raw onto the table root") { @@ -63,7 +124,16 @@ class AMTUtilsSuite extends SparkFunSuite { test("relativize then absolutize round-trips a file under the table root") { val root = new Path("file:/tables/t1") val leaf = new Path("file:/tables/t1/metadata/leaf-1.parquet") - val relative = AMTUtils.relativizeManifestPathToTableRoot(fs, root, leaf) + val relative = AMTUtils.relativizeLocation(root.toString, leaf.toString) assert(AMTUtils.absolutePathForManifestFile(root, relative) === leaf) } + + test("amtEnabled: true when the protocol supports the AMT feature") { + val protocol = Protocol.forTableFeature(AdaptiveMetadataTableFeature) + assert(AMTUtils.amtEnabled(Metadata(), protocol)) + } + + test("amtEnabled: false when the protocol lacks the AMT feature") { + assert(!AMTUtils.amtEnabled(Metadata(), Protocol())) + } } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTWriterManagerSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTWriterManagerSuite.scala index 8d2bbafe916..04658d02265 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTWriterManagerSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AMTWriterManagerSuite.scala @@ -16,8 +16,10 @@ package org.apache.spark.sql.delta.amt -import org.apache.spark.sql.delta.{CurrentTransactionInfo, DeltaOperations, Snapshot} -import io.delta.exceptions.ConcurrentWriteException +import org.apache.spark.sql.delta.{ConcurrentAMTCheckpointLandedException, CurrentTransactionInfo, DeltaOperations, FullAMTWriteFailedWithConflict, LogSegment, Snapshot, SnapshotManagement, WinningCommitMetrics, WinningCommitSummary} +import org.apache.spark.sql.delta.actions.{Action, AddFile, BackReference, Checkpoint, ContentRoot, RemoveFile} +import org.apache.spark.sql.delta.util.FileNames +import org.apache.hadoop.fs.{FileStatus, Path} /** * Tests for [[AMTWriterManager]]: the emission policy (checkpoint-interval and accumulated-size @@ -25,18 +27,30 @@ import io.delta.exceptions.ConcurrentWriteException */ class AMTWriterManagerSuite extends AMTCheckpointTestBase { + // Models conflict resolution advancing the read snapshot's segment by one log-only commit. + private def advanceSegmentByOneCommit(segment: LogSegment): LogSegment = { + val newVersion = segment.version + 1 + val commitFileStatus = + new FileStatus(1L, false, 1, 1L, 1L, FileNames.unsafeDeltaFile(segment.logPath, newVersion)) + SnapshotManagement.appendCommitToLogSegment(segment, commitFileStatus, newVersion) + } + // Reads the current snapshot and returns (manager, snapshot) for direct method-level tests. private def managerFor( tableName: String, operation: DeltaOperations.Operation = DeltaOperations.ManualUpdate): (AMTWriterManager, Snapshot) = { val snapshot = deltaLogForName(tableName).update() - (new AMTWriterManager(snapshot, operation), snapshot) + (new AMTWriterManager("txn", snapshot, operation), snapshot) } // A minimal transaction info over `snapshot` carrying `actions`, for direct writeAMT calls. + // `preCommitLatestAMTCheckpointOpt` models the base AMT the attempt would build on: on a rebase + // it is the tree the conflict fold advanced to (the winner's, if the winner wrote one). private def txnInfoFor( - snapshot: Snapshot, actions: Seq[org.apache.spark.sql.delta.actions.Action]) = + snapshot: Snapshot, + actions: Seq[Action], + preCommitLatestAMTCheckpointOpt: Option[Checkpoint] = None) = CurrentTransactionInfo( txnId = "txn", readPredicates = Vector.empty, @@ -51,48 +65,219 @@ class AMTWriterManagerSuite extends AMTCheckpointTestBase { readRowIdHighWatermark = 0L, catalogTable = None, domainMetadata = Seq.empty, - op = DeltaOperations.ManualUpdate) + op = DeltaOperations.ManualUpdate, + preCommitLatestAMTCheckpointOpt = preCommitLatestAMTCheckpointOpt) + + // A neutral single-winner metric for a one-version conflicted range, for direct writeAMT rebase + // tests that do not exercise the base-reuse decision (the loser is log-only, or the manager has + // no full base to reuse). + private def anyWinner: WinningCommitMetrics = WinningCommitMetrics( + isBlindAppend = true, + minDefaultRowCommitVersion = None, + numAdds = 0, + numRemoves = 0, + numAddFilesWithBackreferences = 0, + numRemoveFilesWithBackreferences = 0, + checkpointAction = None, + commitInfo = None) test("writeAMT performs a clustered full rewrite for an OPTIMIZE checkpoint operation") { withTable("amt_optimize_ckpt") { val name = "amt_optimize_ckpt" createAMTTable(name, checkpointInterval = 2) - sql(s"INSERT INTO $name VALUES (1)") + withSQLConf(leafPackingConfs: _*) { + appendRowsAsSeparateFiles(name, numFiles = leafPackedFiles) - val (manager, snapshot) = managerFor(name, DeltaOperations.OptimizeCheckpoint( - incremental = false, triggerName = AMTTriggerMode.CheckpointIntervalFull.name)) - val result = manager.writeAMT( - commitVersion = snapshot.version + 1, - currentTransactionInfo = txnInfoFor(snapshot, actions = Seq.empty), - preCommitLogSegment = snapshot.logSegment).getOrElse( - fail("OPTIMIZE checkpoint must materialize an AMT.")) - // A full rewrite flushes the live files into freshly clustered leaves via the distributed - // write path, so at least one leaf must be written. - assert(result.leaves.nonEmpty, "The clustered rewrite must write at least one leaf.") - // The commit carries no user actions, so the tree describes state as of the read version. - assert(result.contentRootVersion == snapshot.version) - // The metric records the trigger name carried on the operation. - assert(manager.metrics.attempts.head.trigger == AMTTriggerMode.CheckpointIntervalFull.name) + val (manager, snapshot) = managerFor(name, DeltaOperations.OptimizeCheckpoint( + incremental = false, triggerName = AMTTriggerMode.CheckpointIntervalFull.name)) + val result = manager.writeAMT( + nextAttemptVersion = snapshot.version + 1, + currentTransactionInfo = txnInfoFor(snapshot, actions = Seq.empty), + preCommitLogSegment = snapshot.logSegment, + winningCommitMetricsForConflictedRange = Seq.empty).getOrElse( + fail("OPTIMIZE checkpoint must materialize an AMT.")) + assertLeafCount(result.leaves) + // The commit carries no user actions, so the tree describes state as of the read version. + assert(result.contentRootVersion == snapshot.version) + } } } // End-to-end emission-policy scenarios (interval / full-rewrite cadence / size trigger / minor // compaction) live in AMTCheckpointPolicySuite. This suite covers writeAMT's direct behavior. - test("writeAMT hard-fails an AMT table on a conflict-resolution retry") { - withTable("amt_conflict_fail") { - val name = "amt_conflict_fail" + test("writeAMT lets a log-only commit rebase past a log-only winner on retry") { + withTable("amt_conflict_log_rebase") { + val name = "amt_conflict_log_rebase" createAMTTable(name, checkpointInterval = 2) - sql(s"INSERT INTO $name VALUES (1)") + commitCheckpoint(deltaLogForName(name), incremental = false) val (manager, snapshot) = managerFor(name) - // A retry: conflict resolution advanced the segment past the read snapshot's version. - val retrySegment = snapshot.logSegment.copy(version = snapshot.version + 1) - intercept[ConcurrentWriteException] { - manager.writeAMT( - commitVersion = snapshot.version + 2, + val baseTree = amtProvider(snapshot).map(_.checkpointAction) + assert(baseTree.isDefined, "the table must be AMT-backed for this case.") + // The winner wrote no tree, so the base AMT is unchanged (the folded pointer still equals the + // read snapshot's tree): a log-only commit rebases with no AMT write instead of hard-failing. + val retrySegment = advanceSegmentByOneCommit(snapshot.logSegment) + val result = manager.writeAMT( + nextAttemptVersion = snapshot.version + 2, + currentTransactionInfo = + txnInfoFor(snapshot, actions = Seq.empty, preCommitLatestAMTCheckpointOpt = baseTree), + preCommitLogSegment = retrySegment, + winningCommitMetricsForConflictedRange = Seq(anyWinner)) + assert(result.isEmpty, + "a log-only commit that lost to a log-only winner must rebase without an AMT write.") + } + } + + test("writeAMT writes no tree for a log-only commit rebasing past a tree-installing winner") { + withTable("amt_conflict_log_vs_tree") { + val name = "amt_conflict_log_vs_tree" + createAMTTable(name, checkpointInterval = 2) + commitCheckpoint(deltaLogForName(name), incremental = false) + + val (manager, snapshot) = managerFor(name) + val baseTree = amtProvider(snapshot).map(_.checkpointAction).getOrElse( + fail("the table must be AMT-backed for this case.")) + // A winner installed a newer tree than the read snapshot's. writeAMT for a log-only commit no + // longer hard-fails on this -- it writes no tree; re-deriving the file actions' back + // references against the winner tree happens in doCommit's rebaseBackReferences, exercised + // end-to-end in AMTConflictResolutionSuite. + val winnerTree = baseTree.copy(version = baseTree.version + 1) + val retrySegment = advanceSegmentByOneCommit(snapshot.logSegment) + val result = manager.writeAMT( + nextAttemptVersion = snapshot.version + 2, + currentTransactionInfo = txnInfoFor( + snapshot, actions = Seq.empty, preCommitLatestAMTCheckpointOpt = Some(winnerTree)), + preCommitLogSegment = retrySegment, + winningCommitMetricsForConflictedRange = Seq(anyWinner)) + assert(result.isEmpty, + "a log-only commit rebasing past a winner tree writes no AMT (re-derivation is elsewhere).") + } + } + + test("writeAMT skips a full checkpoint that lost to a full-rewrite winner") { + withTable("amt_full_ckpt_skips_full_winner") { + val name = "amt_full_ckpt_skips_full_winner" + createAMTTable(name, checkpointInterval = 2) + withSQLConf(leafPackingConfs: _*) { + appendRowsAsSeparateFiles(name, numFiles = leafPackedFiles) + + val (manager, snapshot) = managerFor(name, DeltaOperations.OptimizeCheckpoint( + incremental = false, triggerName = AMTTriggerMode.CheckpointIntervalFull.name)) + // First attempt: the full checkpoint materializes its own full tree (describing the read + // snapshot), caching a full last-write result on the manager. + val fullResult = manager.writeAMT( + nextAttemptVersion = snapshot.version + 1, currentTransactionInfo = txnInfoFor(snapshot, actions = Seq.empty), - preCommitLogSegment = retrySegment) + preCommitLogSegment = snapshot.logSegment, + winningCommitMetricsForConflictedRange = Seq.empty).getOrElse( + fail("the full OPTIMIZE checkpoint must materialize an AMT on its first attempt.")) + assert(fullResult.checkpoint.contentRoot.isIncremental.contains(false), + "precondition: the first attempt must write a full (non-incremental) tree.") + + // A concurrent winner installed its own full-rewrite tree at the target version. That tree + // already provides an up-to-date AMT, so this losing full checkpoint is redundant and + // skips. + val fullWinner = fullResult.checkpoint.copy(version = snapshot.version + 1) + val fullWinnerMetric = anyWinner.copy(checkpointAction = Some(fullWinner)) + val retrySegment = advanceSegmentByOneCommit(snapshot.logSegment) + val ex = intercept[ConcurrentAMTCheckpointLandedException] { + manager.writeAMT( + nextAttemptVersion = snapshot.version + 2, + currentTransactionInfo = txnInfoFor( + snapshot, actions = Seq.empty, preCommitLatestAMTCheckpointOpt = Some(fullWinner)), + preCommitLogSegment = retrySegment, + winningCommitMetricsForConflictedRange = Seq(fullWinnerMetric)) + } + assert(ex.contentRootVersion == fullWinner.contentRoot.version, + "the skip signal must carry the winner tree's content-root version.") + } + } + } + + test("writeAMT skips an incremental checkpoint that lost to a tree-installing winner") { + withTable("amt_incr_ckpt_skips_tree_winner") { + val name = "amt_incr_ckpt_skips_tree_winner" + createAMTTable(name, checkpointInterval = 2) + withSQLConf(leafPackingConfs: _*) { + appendRowsAsSeparateFiles(name, numFiles = leafPackedFiles) + // An incremental rewrite requires an existing full tree to build on. + commitCheckpoint(deltaLogForName(name), incremental = false) + + val (manager, snapshot) = managerFor(name, DeltaOperations.OptimizeCheckpoint( + incremental = true, triggerName = AMTTriggerMode.CheckpointIntervalIncremental.name)) + val baseTree = amtProvider(snapshot).map(_.checkpointAction).getOrElse( + fail("the table must be AMT-backed for this case.")) + // First attempt: the incremental checkpoint extends the base tree, caching an incremental + // last-write result on the manager. + val incrementalResult = manager.writeAMT( + nextAttemptVersion = snapshot.version + 1, + currentTransactionInfo = txnInfoFor( + snapshot, actions = Seq.empty, preCommitLatestAMTCheckpointOpt = Some(baseTree)), + preCommitLogSegment = snapshot.logSegment, + winningCommitMetricsForConflictedRange = Seq.empty).getOrElse( + fail("the incremental OPTIMIZE checkpoint must materialize an AMT on its first " + + "attempt.")) + assert(incrementalResult.checkpoint.contentRoot.isIncremental.contains(true), + "precondition: the first attempt must write an incremental tree.") + + // A concurrent winner installed a newer tree. Any winner tree supersedes a losing + // incremental checkpoint, so it is redundant and skips. + val winnerTree = baseTree.copy(version = baseTree.version + 1) + val retrySegment = advanceSegmentByOneCommit(snapshot.logSegment) + intercept[ConcurrentAMTCheckpointLandedException] { + manager.writeAMT( + nextAttemptVersion = snapshot.version + 2, + currentTransactionInfo = txnInfoFor( + snapshot, actions = Seq.empty, preCommitLatestAMTCheckpointOpt = Some(winnerTree)), + preCommitLogSegment = retrySegment, + winningCommitMetricsForConflictedRange = Seq(anyWinner)) + } + } + } + } + + test("writeAMT signals a full-AMT regenerate for a full checkpoint that lost to an " + + "incremental winner") { + withTable("amt_full_ckpt_regenerates_vs_incr_winner") { + val name = "amt_full_ckpt_regenerates_vs_incr_winner" + createAMTTable(name, checkpointInterval = 2) + withSQLConf(leafPackingConfs: _*) { + appendRowsAsSeparateFiles(name, numFiles = leafPackedFiles) + + val (manager, snapshot) = managerFor(name, DeltaOperations.OptimizeCheckpoint( + incremental = false, triggerName = AMTTriggerMode.CheckpointIntervalFull.name)) + val fullResult = manager.writeAMT( + nextAttemptVersion = snapshot.version + 1, + currentTransactionInfo = txnInfoFor(snapshot, actions = Seq.empty), + preCommitLogSegment = snapshot.logSegment, + winningCommitMetricsForConflictedRange = Seq.empty).getOrElse( + fail("the full OPTIMIZE checkpoint must materialize an AMT on its first attempt.")) + + // The winner installed an INCREMENTAL tree. It neither satisfies the requested full AMT nor + // serves as a full base to fold onto, so the full checkpoint signals a regenerate for the + // caller to refresh and retry rather than skipping. + val incrementalRoot = ContentRoot( + path = fullResult.checkpoint.contentRoot.path, + sizeInBytes = fullResult.checkpoint.contentRoot.sizeInBytes, + version = snapshot.version + 1, + isIncremental = true, + lastManifestCommitWithFullRewrite = 0L, + numLeaves = fullResult.checkpoint.contentRoot.numLeaves.getOrElse(0L)) + val incrementalWinner = + fullResult.checkpoint.copy(version = snapshot.version + 1, contentRoot = incrementalRoot) + val retrySegment = advanceSegmentByOneCommit(snapshot.logSegment) + val ex = intercept[FullAMTWriteFailedWithConflict] { + manager.writeAMT( + nextAttemptVersion = snapshot.version + 2, + currentTransactionInfo = txnInfoFor( + snapshot, actions = Seq.empty, + preCommitLatestAMTCheckpointOpt = Some(incrementalWinner)), + preCommitLogSegment = retrySegment, + winningCommitMetricsForConflictedRange = Seq(anyWinner)) + } + assert(ex.conflictingCommitVersion == incrementalWinner.version, + "the regenerate signal must carry the winner commit's version.") } } } @@ -105,12 +290,194 @@ class AMTWriterManagerSuite extends AMTCheckpointTestBase { sql(s"INSERT INTO $name VALUES (1)") val (manager, snapshot) = managerFor(name) - val retrySegment = snapshot.logSegment.copy(version = snapshot.version + 1) + val retrySegment = advanceSegmentByOneCommit(snapshot.logSegment) val result = manager.writeAMT( - commitVersion = snapshot.version + 2, + nextAttemptVersion = snapshot.version + 2, currentTransactionInfo = txnInfoFor(snapshot, actions = Seq.empty), - preCommitLogSegment = retrySegment) + preCommitLogSegment = retrySegment, + winningCommitMetricsForConflictedRange = Seq.empty) assert(result.isEmpty, "Non-AMT tables emit no AMT and are not hard-failed on a retry.") } } + + test("WinningCommitMetrics summarizes a winning commit and computes base-preservation") { + val logPath = new Path("/tmp/winning-commit-metrics/_delta_log") + def summaryOf(actions: Seq[Action], version: Long): WinningCommitSummary = { + val deltaFile = FileNames.unsafeDeltaFile(logPath, version) + val fileStatus = new FileStatus(1L, false, 1, 1L, 1L, deltaFile) + new WinningCommitSummary(actions, fileStatus, readTimeMs = 0L) + } + def add(path: String, drcv: Option[Long]): AddFile = + AddFile(path, Map.empty[String, String], size = 1L, modificationTime = 1L, + dataChange = true, defaultRowCommitVersion = drcv) + def remove(path: String, drcv: Option[Long]): RemoveFile = + AddFile(path, Map.empty[String, String], size = 1L, modificationTime = 1L, + dataChange = true, defaultRowCommitVersion = drcv).removeWithTimestamp(1L) + + // Every add/remove file was created at version 5 or later. + val allNew = WinningCommitMetrics.fromWinningCommitSummary( + summaryOf(Seq(add("a", Some(5L)), add("b", Some(7L)), remove("c", Some(6L))), version = 8L)) + assert(allNew.numAdds == 2 && allNew.numRemoves == 1) + assert(allNew.minDefaultRowCommitVersion.contains(5L)) + assert(allNew.allFileActionsHaveDefaultCommitVersionNewerThan(4L), + "min DRCV 5 > base 4 must preserve the base") + // A file created at the base version is a leaf of that base tree, so it does not preserve it. + assert(!allNew.allFileActionsHaveDefaultCommitVersionNewerThan(5L), + "min DRCV 5 is a leaf of base 5, so must not preserve it") + assert(!allNew.allFileActionsHaveDefaultCommitVersionNewerThan(6L), + "min DRCV 5 < base 6 must not preserve the base") + // Files without a back reference contribute nothing to the back-reference counts. + assert(allNew.numAddFilesWithBackreferences == 0 && + allNew.numRemoveFilesWithBackreferences == 0) + + // Only the file actions carrying a backReference into a base leaf are counted. + val backRef = Some(BackReference("leaf", 0)) + val withBackrefs = WinningCommitMetrics.fromWinningCommitSummary(summaryOf( + Seq(add("a", Some(5L)).copy(backReference = backRef), + add("b", Some(6L)), + remove("c", Some(7L)).copy(backReference = backRef)), + version = 8L)) + assert(withBackrefs.numAddFilesWithBackreferences == 1, + "only the one Add file carrying a back reference must be counted") + assert(withBackrefs.numRemoveFilesWithBackreferences == 1, + "the Remove file carrying a back reference must be counted") + + // A file missing a defaultRowCommitVersion leaves the minimum unknown, so the commit is + // conservatively treated as not base-preserving. + val missingDrcv = WinningCommitMetrics.fromWinningCommitSummary( + summaryOf(Seq(add("a", Some(9L)), add("b", None)), version = 9L)) + assert(missingDrcv.minDefaultRowCommitVersion.isEmpty) + assert(!missingDrcv.allFileActionsHaveDefaultCommitVersionNewerThan(0L), + "a missing DRCV is never base-preserving") + + // A commit with no file actions touches no leaf, so it preserves any base. + val noFiles = WinningCommitMetrics.fromWinningCommitSummary(summaryOf(Seq.empty, version = 3L)) + assert(noFiles.numAdds == 0 && noFiles.numRemoves == 0) + assert(noFiles.minDefaultRowCommitVersion.isEmpty) + assert(noFiles.allFileActionsHaveDefaultCommitVersionNewerThan(100L), + "a commit with no file actions preserves any base") + } + + test("writeAMT recommits the full base as-is when every winner is base-preserving") { + withTable("amt_full_ckpt_reuse_base_preserving") { + val name = "amt_full_ckpt_reuse_base_preserving" + createAMTTable(name, checkpointInterval = 2) + appendRowsAsSeparateFiles(name, numFiles = 2, startId = 1) + + val (manager, snapshot) = managerFor(name, DeltaOperations.OptimizeCheckpoint( + incremental = false, triggerName = AMTTriggerMode.CheckpointIntervalFull.name)) + + // First attempt materializes the full tree for the read snapshot (caches it for the retry). + val firstAttempt = manager.writeAMT( + nextAttemptVersion = snapshot.version + 1, + currentTransactionInfo = txnInfoFor(snapshot, actions = Seq.empty), + preCommitLogSegment = snapshot.logSegment, + winningCommitMetricsForConflictedRange = Seq.empty).getOrElse( + fail("an OPTIMIZE checkpoint must materialize a full AMT.")) + assert(firstAttempt.checkpoint.contentRoot.isIncremental.contains(false), + "the first attempt must write a full (non-incremental) tree.") + + // Retry: a log-only winner won the target version, and its files were all created at or after + // the full base's content-root version (min defaultRowCommitVersion >= base), so the base + // tree is still exact -- writeAMT recommits it as-is instead of folding the winner in. + val basePreservingWinner = WinningCommitMetrics( + isBlindAppend = false, + minDefaultRowCommitVersion = Some(firstAttempt.contentRootVersion + 1), + numAdds = 1, + numRemoves = 0, + numAddFilesWithBackreferences = 0, + numRemoveFilesWithBackreferences = 0, + checkpointAction = None, + commitInfo = None) + val retrySegment = advanceSegmentByOneCommit(snapshot.logSegment) + val reused = manager.writeAMT( + nextAttemptVersion = snapshot.version + 2, + currentTransactionInfo = txnInfoFor(snapshot, actions = Seq.empty), + preCommitLogSegment = retrySegment, + winningCommitMetricsForConflictedRange = Seq(basePreservingWinner)).getOrElse( + fail("a base-preserving rebase must reuse the full base.")) + assert(reused.contentRootVersion == firstAttempt.contentRootVersion, + "reuse must keep the full base's content-root version, not advance it by folding.") + assert(reused.checkpoint.contentRoot.isIncremental.contains(false), + "the reused checkpoint must remain the full base, not an incremental fold.") + } + } + + test("writeAMT asserts a base-preserving winner carries no back references") { + withTable("amt_full_ckpt_reuse_backref_guard") { + val name = "amt_full_ckpt_reuse_backref_guard" + createAMTTable(name, checkpointInterval = 2) + appendRowsAsSeparateFiles(name, numFiles = 2, startId = 1) + + val (manager, snapshot) = managerFor(name, DeltaOperations.OptimizeCheckpoint( + incremental = false, triggerName = AMTTriggerMode.CheckpointIntervalFull.name)) + val firstAttempt = manager.writeAMT( + nextAttemptVersion = snapshot.version + 1, + currentTransactionInfo = txnInfoFor(snapshot, actions = Seq.empty), + preCommitLogSegment = snapshot.logSegment, + winningCommitMetricsForConflictedRange = Seq.empty).getOrElse( + fail("an OPTIMIZE checkpoint must materialize a full AMT.")) + + // A contradictory winner: its min DRCV says base-preserving, yet a file carries a back + // reference into the base tree. The two signals disagree, so reuse must hard-fail. + val contradictoryWinner = WinningCommitMetrics( + isBlindAppend = false, + minDefaultRowCommitVersion = Some(firstAttempt.contentRootVersion + 1), + numAdds = 1, + numRemoves = 0, + numAddFilesWithBackreferences = 1, + numRemoveFilesWithBackreferences = 0, + checkpointAction = None, + commitInfo = None) + val retrySegment = advanceSegmentByOneCommit(snapshot.logSegment) + val ex = intercept[IllegalStateException] { + manager.writeAMT( + nextAttemptVersion = snapshot.version + 2, + currentTransactionInfo = txnInfoFor(snapshot, actions = Seq.empty), + preCommitLogSegment = retrySegment, + winningCommitMetricsForConflictedRange = Seq(contradictoryWinner)) + } + assert(ex.getMessage.contains("back references")) + } + } + + test("writeAMT signals a full-AMT regenerate for a non-base-preserving winner") { + withTable("amt_full_ckpt_regenerate") { + val name = "amt_full_ckpt_regenerate" + createAMTTable(name, checkpointInterval = 2) + appendRowsAsSeparateFiles(name, numFiles = 2, startId = 1) + + val (manager, snapshot) = managerFor(name, DeltaOperations.OptimizeCheckpoint( + incremental = false, triggerName = AMTTriggerMode.CheckpointIntervalFull.name)) + val firstAttempt = manager.writeAMT( + nextAttemptVersion = snapshot.version + 1, + currentTransactionInfo = txnInfoFor(snapshot, actions = Seq.empty), + preCommitLogSegment = snapshot.logSegment, + winningCommitMetricsForConflictedRange = Seq.empty).getOrElse( + fail("an OPTIMIZE checkpoint must materialize a full AMT.")) + + // A non-base-preserving winner (its min DRCV is the base version, so it changed a base leaf) + // cannot be reused: the writer signals a full-AMT regenerate for CheckpointHook to refresh + // and retry, rather than folding or deferring. + val nonBasePreservingWinner = WinningCommitMetrics( + isBlindAppend = false, + minDefaultRowCommitVersion = Some(firstAttempt.contentRootVersion), + numAdds = 1, + numRemoves = 0, + numAddFilesWithBackreferences = 0, + numRemoveFilesWithBackreferences = 0, + checkpointAction = None, + commitInfo = None) + val retrySegment = advanceSegmentByOneCommit(snapshot.logSegment) + val ex = intercept[FullAMTWriteFailedWithConflict] { + manager.writeAMT( + nextAttemptVersion = snapshot.version + 2, + currentTransactionInfo = txnInfoFor(snapshot, actions = Seq.empty), + preCommitLogSegment = retrySegment, + winningCommitMetricsForConflictedRange = Seq(nonBasePreservingWinner)) + } + assert(ex.conflictingCommitVersion == retrySegment.version, + "the regenerate signal must carry the winners' latest commit version.") + } + } } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AdaptiveMetadataActionsSerializerSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AdaptiveMetadataActionsSerializerSuite.scala index b7325abc2fa..06db55368d1 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AdaptiveMetadataActionsSerializerSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AdaptiveMetadataActionsSerializerSuite.scala @@ -31,7 +31,8 @@ class AdaptiveMetadataActionsSerializerSuite extends SparkFunSuite { private val sampleRoot = ContentRoot( path = "metadata/root-abc.parquet", - sizeInBytes = 4096L) + sizeInBytes = 4096L, + version = 1L) private val sampleProtocol = Protocol(minReaderVersion = 3, minWriterVersion = 7) private val sampleMetadata = Metadata(id = "metadata-id", name = "t") @@ -57,15 +58,17 @@ class AdaptiveMetadataActionsSerializerSuite extends SparkFunSuite { test("ContentRoot: ser-de") { val root = ContentRoot( path = "metadata/root-abc.parquet", - sizeInBytes = 4096L) + sizeInBytes = 4096L, + version = 1L) val json = JsonUtils.toJson(root) assert(json === - """{"path":"metadata/root-abc.parquet","sizeInBytes":4096}""") + """{"path":"metadata/root-abc.parquet","sizeInBytes":4096,"version":1}""") val pretty = JsonUtils.toPrettyJson(root) assert(pretty === """{ | "path" : "metadata/root-abc.parquet", - | "sizeInBytes" : 4096 + | "sizeInBytes" : 4096, + | "version" : 1 |}""".stripMargin) assert(JsonUtils.fromJson[ContentRoot](json) === root) assert(JsonUtils.fromJson[ContentRoot](pretty) === root) @@ -74,7 +77,8 @@ class AdaptiveMetadataActionsSerializerSuite extends SparkFunSuite { test("ContentRoot: serde handles Long.MaxValue size") { val root = ContentRoot( path = "metadata/root-big.parquet", - sizeInBytes = Long.MaxValue) + sizeInBytes = Long.MaxValue, + version = Long.MaxValue) val json = JsonUtils.toJson(root) val roundTripped = JsonUtils.fromJson[ContentRoot](json) assert(roundTripped === root) @@ -84,19 +88,26 @@ class AdaptiveMetadataActionsSerializerSuite extends SparkFunSuite { val root = ContentRoot( path = "metadata/root-abc.parquet", sizeInBytes = 4096L, + version = 7L, isIncremental = true, - lastManifestCommitWithFullRewrite = 7L) + lastManifestCommitWithFullRewrite = 7L, + numLeaves = 3L) + assert(root.version === 7L) assert(root.isIncremental === Some(true)) assert(root.lastManifestCommitWithFullRewrite === Some(7L)) + assert(root.numLeaves === Some(3L)) val roundTripped = JsonUtils.fromJson[ContentRoot](JsonUtils.toJson(root)) assert(roundTripped === root) + assert(roundTripped.version === 7L) assert(roundTripped.isIncremental === Some(true)) assert(roundTripped.lastManifestCommitWithFullRewrite === Some(7L)) + assert(roundTripped.numLeaves === Some(3L)) } test("ContentRoot: accessors are None when tags are absent") { assert(sampleRoot.isIncremental.isEmpty) assert(sampleRoot.lastManifestCommitWithFullRewrite.isEmpty) + assert(sampleRoot.numLeaves.isEmpty) } // ============================================================================================ @@ -206,7 +217,8 @@ class AdaptiveMetadataActionsSerializerSuite extends SparkFunSuite { | "version" : 1, | "contentRoot" : { | "path" : "metadata/root-abc.parquet", - | "sizeInBytes" : 4096 + | "sizeInBytes" : 4096, + | "version" : 1 | }, | "protocol" : { | "minReaderVersion" : 3, @@ -263,7 +275,7 @@ class AdaptiveMetadataActionsSerializerSuite extends SparkFunSuite { SetTransaction(appId = "app-2", version = 8L, lastUpdated = None)) val cp = Checkpoint( version = 41L, - contentRoot = sampleRoot, + contentRoot = sampleRoot.copy(version = 41L), protocol = protocol, metaData = sampleMetadata, domainMetadata = dm, @@ -275,7 +287,8 @@ class AdaptiveMetadataActionsSerializerSuite extends SparkFunSuite { | "version" : 41, | "contentRoot" : { | "path" : "metadata/root-abc.parquet", - | "sizeInBytes" : 4096 + | "sizeInBytes" : 4096, + | "version" : 41 | }, | "protocol" : { | "minReaderVersion" : 3, @@ -332,7 +345,8 @@ class AdaptiveMetadataActionsSerializerSuite extends SparkFunSuite { | "version" : 1, | "contentRoot" : { | "path" : "metadata/root-abc.parquet", - | "sizeInBytes" : 4096 + | "sizeInBytes" : 4096, + | "version" : 1 | }, | "protocol" : { | "minReaderVersion" : 3, @@ -416,4 +430,32 @@ class AdaptiveMetadataActionsSerializerSuite extends SparkFunSuite { assert(parsed.sidecars.length === 2) assert(parsed.sidecars.forall(_.sidecarType.contains(SidecarType.Type.DomainMetadata))) } + + test("Checkpoint: contentRoot.version may lag checkpoint.version") { + val cp = Checkpoint( + version = 10L, + contentRoot = sampleRoot.copy(version = 5L), + protocol = sampleProtocol, + metaData = sampleMetadata, + domainMetadata = Seq.empty, + txns = Seq.empty, + sidecars = Seq.empty) + val parsed = Action.fromJson(cp.json).asInstanceOf[Checkpoint] + assert(parsed.contentRoot.version === 5L) + assert(parsed.version === 10L) + } + + test("Checkpoint: rejects contentRoot.version ahead of checkpoint.version") { + val ex = intercept[IllegalArgumentException] { + Checkpoint( + version = 1L, + contentRoot = sampleRoot.copy(version = 2L), + protocol = sampleProtocol, + metaData = sampleMetadata, + domainMetadata = Seq.empty, + txns = Seq.empty, + sidecars = Seq.empty) + } + assert(ex.getMessage.contains("contentRoot.version")) + } } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AdaptiveMetadataTableFeatureSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AdaptiveMetadataTableFeatureSuite.scala index 59b795aadcc..14fdd12b723 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/amt/AdaptiveMetadataTableFeatureSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/AdaptiveMetadataTableFeatureSuite.scala @@ -64,9 +64,9 @@ class AdaptiveMetadataTableFeatureSuite private def protocolOf(tableName: String) = DeltaLog.forTableWithSnapshot(spark, new TableIdentifier(tableName))._2.protocol - test("feature is a ReaderWriterFeature and is NOT a RemovableFeature") { + test("feature is a ReaderWriterFeature and a RemovableFeature") { assert(AdaptiveMetadataTableFeature.isReaderWriterFeature) - assert(!AdaptiveMetadataTableFeature.isInstanceOf[RemovableFeature]) + assert(AdaptiveMetadataTableFeature.isInstanceOf[RemovableFeature]) } test("feature is not automatically enabled by metadata") { @@ -93,6 +93,8 @@ class AdaptiveMetadataTableFeatureSuite val protocol = protocolOf("amt_supported") assert(protocol.isFeatureSupported(AdaptiveMetadataTableFeature), s"Protocol must record adaptiveMetadata-preview as supported. Got: $protocol") + assert(AMTUtils.amtEnabled(metadataOf("amt_supported"), protocol), + s"AMTUtils.amtEnabled must report the table as AMT-enabled. Got: $protocol") // Every required feature (and their transitive dependencies) must be present. dependentFeatures(AdaptiveMetadataTableFeature).foreach { f => assert(protocol.readerAndWriterFeatureNames.contains(f.name), diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/MergeIntoAMTMixin.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/MergeIntoAMTMixin.scala new file mode 100644 index 00000000000..c9b47e5a1dd --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/MergeIntoAMTMixin.scala @@ -0,0 +1,147 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.amt + +import scala.util.control.NonFatal + +import org.apache.spark.sql.delta.{AdaptiveMetadataTableFeature, DeltaConfigs, DeltaLog, DeltaTestUtils, MergeIntoSQLTestUtils} +import org.apache.spark.sql.delta.actions.TableFeatureProtocolUtils._ +import org.apache.spark.sql.delta.coordinatedcommits.{CatalogOwnedCommitCoordinatorProvider, CatalogOwnedTableUtils, TrackingInMemoryCommitCoordinatorBuilder} + +import org.apache.spark.SparkConf + +/** + * Generates AMT (`adaptiveMetadata-preview`) variants of the MERGE INTO test suites. + */ +trait MergeIntoAMTMixin + extends AMTCheckpointTestBase + with MergeIntoSQLTestUtils { + + override protected def sparkConf: SparkConf = super.sparkConf + .set(defaultPropertyKey(AdaptiveMetadataTableFeature), FEATURE_PROP_SUPPORTED) + // Disable periodic checkpoints so the only AMT checkpoints are the two that + // withAMTCheckpointsAroundMerge brackets each MERGE with. + .set(DeltaConfigs.CHECKPOINT_INTERVAL.defaultTablePropertyKey, Int.MaxValue.toString) + + // Register the in-memory commit coordinator before super.beforeAll(). Some MERGE suites (e.g. + // RowTrackingMergeSuiteBase) create catalog-managed helper tables in their own beforeAll(), which + // runs before the beforeEach() that normally registers the coordinator; without it, creating + // those AMT tables fails with "Couldn't locate commit coordinator for ...". + override def beforeAll(): Unit = { + CatalogOwnedCommitCoordinatorProvider.clearBuilders() + catalogOwnedCoordinatorBackfillBatchSize.foreach { batchSize => + CatalogOwnedCommitCoordinatorProvider.registerBuilder( + catalogName = CatalogOwnedTableUtils.DEFAULT_CATALOG_NAME_FOR_TESTING, + commitCoordinatorBuilder = TrackingInMemoryCommitCoordinatorBuilder(batchSize)) + } + super.beforeAll() + } + + override def excluded: Seq[String] = super.excluded ++ Seq( + // scalastyle:off line.size.limit + // AMT tables are always catalog-managed, so the path-based (catalogManaged=false) analysis- + // snapshot-reuse variants are not applicable. + "merge SQL command reuses analysis snapshot in SQL environments (catalogManaged=false)", + "merge SQL command does not reuse analysis snapshot when config is disabled (catalogManaged=false)", + // This test strips record-count stats from the target files (AddFile.stats = null) to exercise + // Delta's graceful missing-stats handling. AMT cannot represent such files: its manifest + // requires a per-file physical record count (DataEntry.fromAddFile throws on a stats-less + // AddFile), so the post-commit AMT checkpoint fails before the assertion is reached. + "merge logs error if number of records are missing in stats", + // RowTrackingMerge: these tests create a table with delta.enableRowTracking = false (one to + // assert row tracking stays off, one to later enable it via backfill). AMT mandates row + // tracking, so table creation fails with DELTA_ADAPTIVE_METADATA_REQUIRES_DEPENDENT_FEATURE_ + // ENABLED. Structural AMT invariant (row tracking cannot be disabled), not a MERGE bug. + "Row tracking marked as not preserved when row tracking disabled", + "MERGE preserves Row Tracking on tables enabled using backfill" + // scalastyle:on line.size.limit + ) + + // Bracket each MERGE with two checkpoints: a full one before (so the MERGE reads an AMT-backed + // target, exercising AMTCheckpointProvider.fromCheckpoint) and an incremental one after. The + // invariant: the post-MERGE checkpoint is a pure metadata reorganization -- the live file set is + // identical immediately before and after it. Only applies when the target resolves to an + // AMT-backed Delta table; temp views, non-Delta / not-yet-created targets and analysis-error + // negative cases run the MERGE unchanged (and a throwing MERGE skips the post-checkpoint/assert). + private def withAMTCheckpointsAroundMerge(target: String)(runMerge: => Unit): Unit = { + resolveAMTDeltaLog(target) match { + case Some(deltaLog) => + commitCheckpoint(deltaLog, incremental = false) + runMerge + val liveBeforeCheckpoint = liveFileKeys(deltaLog) + commitCheckpoint(deltaLog, incremental = true) + val liveAfterCheckpoint = liveFileKeys(deltaLog) + assert(liveAfterCheckpoint == liveBeforeCheckpoint, + "AMT checkpoint after MERGE changed the live file set: only-before = " + + s"${liveBeforeCheckpoint -- liveAfterCheckpoint}, only-after = " + + s"${liveAfterCheckpoint -- liveBeforeCheckpoint}") + case None => + runMerge + } + } + + // Resolves `target` to its DeltaLog, or None if it is not a resolvable AMT-backed Delta table + // (temp view, non-Delta target, not-yet-created table, analysis-error negative case). + private def resolveAMTDeltaLog(target: String): Option[DeltaLog] = { + val deltaLogOpt = + try { + DeltaTestUtils.getTableIdentifierOrPath(target) match { + case DeltaTestUtils.TableIdentifierOrPath.Identifier(id, _) => + Some(DeltaLog.forTable(spark, id)) + case DeltaTestUtils.TableIdentifierOrPath.Path(path, _) => + Some(DeltaLog.forTable(spark, path)) + } + } catch { + case NonFatal(_) => None + } + deltaLogOpt.filter(_.update().protocol.isFeatureSupported(AdaptiveMetadataTableFeature)) + } + + // The set of live files, keyed the way AMT identifies them (path + deletion-vector object + // identity) so a DV re-encoded by the checkpoint still compares equal. + private def liveFileKeys(deltaLog: DeltaLog) = + deltaLog.update().allFiles.collect() + .map(_.toUniqueFileActionTuple(deltaLog.dataPath, useObjectIdentity = true)).toSet + + abstract override def executeMerge( + target: String, + source: String, + condition: String, + update: String, + insert: String): Unit = + withAMTCheckpointsAroundMerge(target) { + super.executeMerge(target, source, condition, update, insert) + } + + abstract override def executeMerge( + tgt: String, + src: String, + cond: String, + clauses: MergeClause*): Unit = + withAMTCheckpointsAroundMerge(tgt) { + super.executeMerge(tgt, src, cond, clauses: _*) + } + + abstract override def executeMergeWithSchemaEvolution( + tgt: String, + src: String, + cond: String, + clauses: MergeClause*): Unit = + withAMTCheckpointsAroundMerge(tgt) { + super.executeMergeWithSchemaEvolution(tgt, src, cond, clauses: _*) + } +} diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/amt/SnapshotLastManifestCommitSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/amt/SnapshotLastManifestCommitSuite.scala index 70d2f60c9d8..3fb0bac4a1e 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/amt/SnapshotLastManifestCommitSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/amt/SnapshotLastManifestCommitSuite.scala @@ -29,20 +29,23 @@ import org.apache.spark.SparkConf */ trait SnapshotLastManifestCommitSuiteBase extends AMTCheckpointTestBase { - override protected def sparkConf: SparkConf = super.sparkConf - // Inline every manifest commit instead of deferring to a follow-up OPTIMIZE CHECKPOINT commit. - // This simplifies the test suite as the deferring is not the primary focus of the suite. - .set(DeltaSQLConf.AMT_LARGE_COMMIT_ACTIONS_COUNT_THRESHOLD_FOR_INLINE_MANIFEST_COMMIT.key, "1") - /** Whether this suite runs with `.crc` files enabled, read from the effective conf. */ protected def writeChecksumEnabled: Boolean = spark.conf.get(DeltaSQLConf.DELTA_WRITE_CHECKSUM_ENABLED) + //////////////////////// + // Fake, Injected LMCs + //////////////////////// + /** * Seeds `lmc` onto every source that exists for this suite's mode. Always rewrites the delta-file * `CommitInfo`; the CRC-enabled suite additionally rewrites the `.crc`. */ - protected def injectLmc(deltaLog: DeltaLog, version: Long, lmc: Option[LastManifestCommit]): Unit + protected def injectLmc( + deltaLog: DeltaLog, version: Long, lmc: Option[LastManifestCommit]): Unit = { + injectLmcIntoCommitInfo(deltaLog, version, lmc) + injectLmcIntoCrc(deltaLog, version, lmc) + } /** Rewrites the `CommitInfo` at `version` to carry `lmc`, preserving all other actions. */ protected def injectLmcIntoCommitInfo( @@ -77,6 +80,31 @@ trait SnapshotLastManifestCommitSuiteBase extends AMTCheckpointTestBase { deltaLogForName(tableName).getSnapshotAt(version) } + /** + * Returns a snapshot at `version` suitable for asserting `lastManifestCommitOpt` resolution when + * the test has injected a synthetic reference (see [[injectLmc]]) onto a table with no matching + * AMT checkpoint. + */ + protected def snapshotForLmcResolution(deltaLog: DeltaLog, version: Long): Snapshot = { + require(version > 0, "Version must be greater than 0.") + val base = deltaLog.unsafeVolatileSnapshot + new Snapshot( + path = base.path, + version = version, + logSegment = base.logSegment.copy( + version = version, + deltas = base.logSegment.deltas.filter(f => FileNames.deltaVersion(f) == version), + nonCompactedDeltasOpt = base.logSegment.nonCompactedDeltasOpt + .map(_.filter(f => FileNames.deltaVersion(f) == version)), + deltaAtCheckpointVersionOpt = base.logSegment.deltas + .find(f => FileNames.deltaVersion(f) == version - 1), + checkpointProvider = fakeAMTProviderAt( + version - 1, protocol = base.protocol, metadata = base.metadata)), + deltaLog = deltaLog, + checksumOpt = deltaLog.readChecksum(version) + ) + } + /** * Asserts that `opType` was (or was not) emitted among the in-scope captured `usageLogs`. Wrap * the code under test in `implicit val usageLogs = Log4jUsageLogger.track { ... }` first. @@ -97,8 +125,8 @@ trait SnapshotLastManifestCommitSuiteBase extends AMTCheckpointTestBase { sql(s"INSERT INTO $name VALUES (3)") val deltaLog = deltaLogForName(name) - val lmcV1 = LastManifestCommit(version = 1, contentRootVersion = 1) - val lmcV3 = LastManifestCommit(version = 3, contentRootVersion = 3) + val lmcV1 = LastManifestCommit(version = 1, contentRootVersion = 0) + val lmcV3 = LastManifestCommit(version = 3, contentRootVersion = 2) injectLmc(deltaLog, version = 1, lmc = Some(lmcV1)) // v1: carries a reference. // v2: no reference. injectLmc(deltaLog, version = 3, lmc = Some(lmcV3)) // v3: carries a different reference. @@ -106,7 +134,7 @@ trait SnapshotLastManifestCommitSuiteBase extends AMTCheckpointTestBase { /** Assert each snapshot resolves to its own version's reference with the expected path. */ def assertResolves(version: Long, expected: Option[LastManifestCommit]): Unit = { implicit val usageLogs: Seq[UsageRecord] = Log4jUsageLogger.track { - assert(freshSnapshotAt(name, version).lastManifestCommitOpt == expected) + assert(snapshotForLmcResolution(deltaLog, version).lastManifestCommitOpt == expected) } // There are trailing deltas, so the CommitInfo fallback is never reached. assertUsageLog(AMTUsageLogs.LAST_MANIFEST_COMMIT_READ_FROM_COMMIT_INFO, isPresent = false) @@ -123,7 +151,6 @@ trait SnapshotLastManifestCommitSuiteBase extends AMTCheckpointTestBase { // The in-commit-timestamp and last-manifest-commit are sibling fields in CommitInfo, so a // single commit carries both on one reconstruction row; both must survive. Catalog-managed // tables have in-commit timestamps enabled by default, so no explicit enablement is needed. - val lmc = LastManifestCommit(version = 7, contentRootVersion = 5) withTable("amt_lmc_ict") { val name = "amt_lmc_ict" createAMTTable(name, checkpointInterval = 100) @@ -135,9 +162,10 @@ trait SnapshotLastManifestCommitSuiteBase extends AMTCheckpointTestBase { val expectedIct = deltaLog.unsafeVolatileSnapshot.getInCommitTimestampOpt.getOrElse { fail("Expected a non-None in-commit-timestamp.") } + val lmc = LastManifestCommit(version = version, contentRootVersion = version - 1) injectLmc(deltaLog, version, lmc = Some(lmc)) - val snapshot = freshSnapshotAt(name, version) + val snapshot = snapshotForLmcResolution(deltaLog, version) assert(snapshot.lastManifestCommitOpt.contains(lmc), "LMC must survive on a row that also carries an in-commit-timestamp.") assert(snapshot.getInCommitTimestampOpt.contains(expectedIct), @@ -154,34 +182,213 @@ trait SnapshotLastManifestCommitSuiteBase extends AMTCheckpointTestBase { assert(freshSnapshotAt(name, 1).lastManifestCommitOpt.isEmpty) } } + + //////////////////////////// + // Actually Populated LMCs + //////////////////////////// + + /** The `lastManifestCommit` recorded on the CommitInfo committed at `version`, if any. */ + private def lastManifestCommitFromCommitInfoAt( + deltaLog: DeltaLog, version: Long): Option[LastManifestCommit] = { + actionsAt(deltaLog, version) + .collectFirst { case ci: CommitInfo => ci } + .flatMap(_.lastManifestCommit) + } + + /** The `lastManifestCommit` persisted in the CRC at `version`, if the CRC exists. */ + private def lastManifestCommitFromCrcAt( + deltaLog: DeltaLog, version: Long): Option[LastManifestCommit] = + deltaLog.readChecksum(version).flatMap(_.lastManifestCommit) + + /** Asserts the commit at `version` has the expected manifest-commit states. */ + private def assertCommitStates( + deltaLog: DeltaLog, + version: Long, + emitsCheckpoint: Boolean, + expected: Option[LastManifestCommit]): Unit = { + assert(checkpointAt(deltaLog, version).nonEmpty == emitsCheckpoint, + s"v$version checkpoint emission: expected $emitsCheckpoint.") + // The CRC only carries the reference when `.crc` files are written for this suite; without + // them there is no CRC to inspect. + if (writeChecksumEnabled) { + assert(lastManifestCommitFromCrcAt(deltaLog, version) == expected, + s"v$version CRC must carry LMC $expected.") + } + assert(lastManifestCommitFromCommitInfoAt(deltaLog, version) == expected, + s"v$version CommitInfo must carry LMC $expected.") + // The snapshot resolves the reference from the CRC when present, and otherwise from the latest + // commit's CommitInfo fallback, so this holds in both CRC and no-CRC modes. + assert(deltaLog.getSnapshotAt(version).lastManifestCommitOpt == expected, + s"v$version snapshot must resolve LMC $expected.") + } + + test("manifest commits persist lastManifestCommit to the CRC and CommitInfo, " + + "while non-manifest commits carry the previous manifest-commit reference forward") { + withTable("amt_lmc_persist") { + val name = "amt_lmc_persist" + createAMTTable(name, checkpointInterval = 3) + + sql(s"INSERT INTO $name VALUES (1)") // v1: no checkpoint. + sql(s"INSERT INTO $name VALUES (2)") // v2: no checkpoint. + sql(s"INSERT INTO $name VALUES (3)") // v3: triggers a checkpoint, but not in this commit. + // v4: AMT checkpoint emitted via hook. + sql(s"INSERT INTO $name VALUES (4)") // v5: no checkpoint. + sql(s"INSERT INTO $name VALUES (5)") // v6: triggers another checkpoint, not in this commit. + // v7: AMT checkpoint emitted via hook. + + val deltaLog = deltaLogForName(name) + + // Before the first manifest checkpoint (first emit is v4), there is no reference to record. + assertCommitStates(deltaLog, 1, emitsCheckpoint = false, expected = None) + assertCommitStates(deltaLog, 2, emitsCheckpoint = false, expected = None) + + // The checkpoint-trigger commit (v3) does not create or store a reference yet. + assertCommitStates(deltaLog, 3, emitsCheckpoint = false, expected = None) + // The actual manifest commit stores the new reference to the CRC and CommitInfo. + // Note that the manifest commit version is 4, but the content root version is 3. + val expectedLmcAtV4 = LastManifestCommit(version = 4, contentRootVersion = 3) + assertCommitStates(deltaLog, 4, emitsCheckpoint = true, expected = Some(expectedLmcAtV4)) + + // The non-manifest commit carries the previous manifest-commit reference forward. + assertCommitStates(deltaLog, 5, emitsCheckpoint = false, expected = Some(expectedLmcAtV4)) + + // The next checkpoint-trigger commit (v6) does not create or store a new reference yet. + assertCommitStates(deltaLog, 6, emitsCheckpoint = false, expected = Some(expectedLmcAtV4)) + // The actual manifest commit stores the new reference to the CRC and CommitInfo. + // Note that the manifest commit version is 7, but the content root version is 6. + val expectedLmcAtV7 = LastManifestCommit(version = 7, contentRootVersion = 6) + assertCommitStates(deltaLog, 7, emitsCheckpoint = true, expected = Some(expectedLmcAtV7)) + } + } + + testInline("consecutive manifest commits persist lastManifestCommit to the CRC and CommitInfo") { + withTable("amt_lmc_persist_inline") { + val name = "amt_lmc_persist_inline" + createAMTTable(name, checkpointInterval = 2) + + sql(s"INSERT INTO $name VALUES (1)") // v1: no checkpoint (no existing tree to inline into). + sql(s"INSERT INTO $name VALUES (2)") // v2: reaches the interval boundary. + // v3: the FIRST AMT is emitted as a deferred OPTIMIZE + // CHECKPOINT (a full rewrite describing state@v2). + sql(s"INSERT INTO $name VALUES (3)") // v4: now a tree exists, so this commit emits inline. + sql(s"INSERT INTO $name VALUES (4)") // v5: another inline manifest commit. + + val deltaLog = deltaLogForName(name) + + // v1/v2: carry no manifest reference yet: the first AMT has not been written. + assertCommitStates(deltaLog, 1, emitsCheckpoint = false, expected = None) + assertCommitStates(deltaLog, 2, emitsCheckpoint = false, expected = None) + + // v3: the deferred first AMT. It is a full rewrite of state as of v2, so version=3 but + // contentRootVersion=2. + val expectedLmcAtV3 = LastManifestCommit(version = 3, contentRootVersion = 2) + assertCommitStates(deltaLog, 3, emitsCheckpoint = true, expected = Some(expectedLmcAtV3)) + + // v4: with a tree in place, the business commit writes its manifest inline, so version and + // contentRootVersion coincide. + val expectedLmcAtV4 = LastManifestCommit(version = 4, contentRootVersion = 4) + assertCommitStates(deltaLog, 4, emitsCheckpoint = true, expected = Some(expectedLmcAtV4)) + + // v5: another inline manifest commit. + val expectedLmcAtV5 = LastManifestCommit(version = 5, contentRootVersion = 5) + assertCommitStates(deltaLog, 5, emitsCheckpoint = true, expected = Some(expectedLmcAtV5)) + } + } + + test("commitLarge (RESTORE) always carries lastManifestCommit forward") { + withTable("amt_lmc_restore") { + val name = "amt_lmc_restore" + createAMTTable(name, checkpointInterval = 2) + sql(s"INSERT INTO $name VALUES (1)") // v1: no checkpoint + sql(s"INSERT INTO $name VALUES (2)") // v2: reaches the interval boundary + // v3: the FIRST AMT is always a deferred OPTIMIZE + // CHECKPOINT (describing state@v2). + val deltaLog = deltaLogForName(name) + sql(s"RESTORE TABLE $name VERSION AS OF 1") // v4: RESTORE commit (via commitLarge) + // v5: RESTORE's follow-up full AMT (state@v4). + assert(deltaLog.snapshot.version == 5, + s"RESTORE plus its follow-up AMT should reach v5, but got v${deltaLog.snapshot.version}.") + + // RESTORE commits via commitLarge. The RESTORE commit (v4) itself emits no inline checkpoint + // and carries the previous manifest reference forward; commitLarge then emits a full AMT as a + // follow-up OPTIMIZE CHECKPOINT commit (v5) describing the restored state as of v4. + + // v4: the RESTORE commit emits no checkpoint and carries the reference from v3 forward. + val carriedLmc = LastManifestCommit(version = 3, contentRootVersion = 2) + assert(checkpointAt(deltaLog, 4).isEmpty, s"v4 (RESTORE) must not emit a checkpoint.") + assert( + lastManifestCommitFromCommitInfoAt(deltaLog, 4).contains(carriedLmc), + s"v4 CommitInfo must carry LMC $carriedLmc.") + + // v5: the follow-up full AMT is written at v5 and describes the restored state as of v4, so + // the manifest-commit version is 5 but the content root version is 4. + val newLmc = LastManifestCommit(version = 5, contentRootVersion = 4) + assert(checkpointAt(deltaLog, 5).nonEmpty, s"v5 must emit the follow-up AMT checkpoint.") + assert( + lastManifestCommitFromCommitInfoAt(deltaLog, 5).contains(newLmc), + s"v5 CommitInfo must carry LMC $newLmc.") + } + } } -/** Runs the shared cases with `.crc` files enabled; the reference is seeded to CommitInfo + CRC. */ -class SnapshotLastManifestCommitSuite extends SnapshotLastManifestCommitSuiteBase { +/** + * With incremental CRC and verification both enabled, the commit discards the incrementally-derived + * checksum and verifies against a full reconstruction, so the final CRC file is actually sourced + * from reconstruction via [[Snapshot.computeChecksum]], not incremental derivation. + */ +class SnapshotLastManifestCommitIncrementalCRCWithVerificationSuite + extends SnapshotLastManifestCommitSuiteBase { + override protected def sparkConf: SparkConf = super.sparkConf .set(DeltaSQLConf.DELTA_WRITE_CHECKSUM_ENABLED.key, "true") + .set(DeltaSQLConf.INCREMENTAL_COMMIT_ENABLED.key, "true") + .set(DeltaSQLConf.INCREMENTAL_COMMIT_VERIFY.key, "true") + .set(DeltaSQLConf.INCREMENTAL_COMMIT_FORCE_VERIFY_IN_TESTS.key, "true") +} - override protected def injectLmc( - deltaLog: DeltaLog, version: Long, lmc: Option[LastManifestCommit]): Unit = { - injectLmcIntoCommitInfo(deltaLog, version, lmc) - injectLmcIntoCrc(deltaLog, version, lmc) - } +/** + * With incremental CRC enabled but verification off, the commit trusts and caches the + * incrementally-derived checksum, so the final CRC file is indeed sourced from incremental + * derivation via [[Checksum.computeNewChecksum]]. This matches the production hot path. + */ +class SnapshotLastManifestCommitIncrementalCRCWithoutVerificationSuite + extends SnapshotLastManifestCommitSuiteBase { + + override protected def sparkConf: SparkConf = super.sparkConf + .set(DeltaSQLConf.DELTA_WRITE_CHECKSUM_ENABLED.key, "true") + .set(DeltaSQLConf.INCREMENTAL_COMMIT_ENABLED.key, "true") + .set(DeltaSQLConf.INCREMENTAL_COMMIT_VERIFY.key, "false") + .set(DeltaSQLConf.INCREMENTAL_COMMIT_FORCE_VERIFY_IN_TESTS.key, "false") +} + +/** + * With incremental CRC disabled, the final CRC file is sourced directly from reconstruction via + * [[Snapshot.computeChecksum]]. No incremental derivation is ever attempted. + */ +class SnapshotLastManifestCommitNonIncrementalCRCSuite + extends SnapshotLastManifestCommitSuiteBase { + + override protected def sparkConf: SparkConf = super.sparkConf + .set(DeltaSQLConf.DELTA_WRITE_CHECKSUM_ENABLED.key, "true") + .set(DeltaSQLConf.INCREMENTAL_COMMIT_ENABLED.key, "false") + // When incremental CRC is disabled, the verify flags are irrelevant. } -/** Runs the shared cases with `.crc` files disabled; the reference is seeded to CommitInfo only. */ +/** No CRC files are ever written in this suite. Aims to test the fallback paths. */ class SnapshotLastManifestCommitWithoutCRCSuite extends SnapshotLastManifestCommitSuiteBase { override protected def sparkConf: SparkConf = super.sparkConf .set(DeltaSQLConf.DELTA_WRITE_CHECKSUM_ENABLED.key, "false") + // When CRC is disabled, the incremental/verify flags are all irrelevant. override protected def injectLmc( deltaLog: DeltaLog, version: Long, lmc: Option[LastManifestCommit]): Unit = { + // No CRC files are written, so the only inject lastManifestCommit into the CommitInfo. injectLmcIntoCommitInfo(deltaLog, version, lmc) } - test("lastManifestCommitOpt falls back to CommitInfo when no other source carries the ref") { + testInline("lastManifestCommitOpt falls back to reading CommitInfo with no other sources") { // With CRC unavailable, when the snapshot sits on an AMT checkpoint, there is no trailing delta // to provide the CommitInfo during P&M query, so we must fallback to a direct CommitInfo read. - val lmc = LastManifestCommit(version = 7, contentRootVersion = 5) withTable("amt_lmc_fallback") { val name = "amt_lmc_fallback" createAMTTable(name, checkpointInterval = 2) @@ -195,30 +402,16 @@ class SnapshotLastManifestCommitWithoutCRCSuite extends SnapshotLastManifestComm sql(s"INSERT INTO $name VALUES (2)") // v2: boundary -> deferred full AMT follow-up at v3. sql(s"INSERT INTO $name VALUES (3)") // v4: boundary -> inline AMT checkpoint action. - val deltaLog = deltaLogForName(name) - injectLmc(deltaLog, version = 4, lmc = Some(lmc)) + implicit val usageLogs: Seq[UsageRecord] = Log4jUsageLogger.track { + DeltaLog.clearCache() + val snapshot = deltaLogForName(name).unsafeVolatileSnapshot - // Build the AMT provider from v4's emitted inline checkpoint action and stub it into a fresh - // snapshot's log segment, trimming the version's delta as cold discovery eventually will. - val checkpoint = checkpointAt(deltaLog, 4).getOrElse { - fail("v4 must emit an inline AMT checkpoint action.") - } - val provider = AMTCheckpointProvider.fromCheckpoint(spark, deltaLog, checkpoint) - val coldSnapshot = freshSnapshotAt(name, 4) - val segment = coldSnapshot.logSegment.copy(checkpointProvider = provider, deltas = Nil) - val snapshot = new Snapshot( - path = coldSnapshot.path, - version = coldSnapshot.version, - logSegment = segment, - deltaLog = coldSnapshot.deltaLog, - checksumOpt = None // No CRC, so reconstruction has no source other than the CommitInfo. - ) - - assert(amtProvider(snapshot).isDefined, "Snapshot must carry the (stubbed) AMT provider.") - assert(snapshot.logSegment.deltas.isEmpty, "Segment must have no trailing deltas.") - assert(snapshot.checksumOpt.isEmpty, "Snapshot must have no CRC.") + assert(snapshot.version == 4, s"Expected volatile snapshot at v4, got ${snapshot.version}.") + assert(amtProvider(snapshot).isDefined, "Snapshot must carry the AMT provider.") + assert(snapshot.logSegment.deltas.isEmpty, "Segment must have no trailing deltas.") + assert(snapshot.checksumOpt.isEmpty, "Snapshot must have no CRC.") - implicit val usageLogs: Seq[UsageRecord] = Log4jUsageLogger.track { + val lmc = LastManifestCommit(version = 4, contentRootVersion = 4) assert(snapshot.lastManifestCommitOpt.contains(lmc), "The CommitInfo fallback must resolve the reference.") } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/catalog/AbstractDeltaCatalogClientRoutingSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/catalog/AbstractDeltaCatalogClientRoutingSuite.scala index 8cadc722e0c..23fd780e60a 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/catalog/AbstractDeltaCatalogClientRoutingSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/catalog/AbstractDeltaCatalogClientRoutingSuite.scala @@ -32,7 +32,7 @@ import io.delta.storage.commit.uniform.{IcebergMetadata, UniformMetadata} import org.apache.spark.sql.QueryTest import org.apache.spark.sql.catalyst.catalog.CatalogTableType import org.apache.spark.sql.connector.catalog.{Identifier, Table, TableCatalog, V1Table} -import org.apache.spark.sql.delta.IcebergConstants +import org.apache.spark.sql.delta.{CatalogManagedTableMaintenanceOperation, IcebergConstants} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.util.CaseInsensitiveStringMap @@ -446,6 +446,7 @@ class AbstractDeltaCatalogClientRoutingSuite extends QueryTest with DeltaSQLComm "s3://bucket/existing/tbl", metadata, storageProperties, + Collections.emptyList(), Optional.empty()) } @@ -713,6 +714,7 @@ class AbstractDeltaCatalogClientRoutingSuite extends QueryTest with DeltaSQLComm "s3://bucket/table", metadata, util.Map.of("fs.s3a.access.key", "key"), + Collections.emptyList(), Optional.empty()) val client = new UCDeltaCatalogClientImpl( @@ -736,6 +738,34 @@ class AbstractDeltaCatalogClientRoutingSuite extends QueryTest with DeltaSQLComm assert(merged.get("fs.s3a.access.key") === Some("key")) } + test("loadTable uses only the catalog maintenance operation list") { + val property = CatalogManagedTableMaintenanceOperation.ALLOWED_OPERATIONS_PROPERTY + val metadata = new TestMetadata(configuration = util.Map.of(property, "DATA_CLEANUP")) + + def loadTable(operations: util.List[String]): V1Table = { + val info = new TableInfo( + UUID.randomUUID(), + UCDeltaModels.TableType.MANAGED, + "s3://bucket/table", + metadata, + Collections.emptyMap(), + operations, + Optional.empty()) + new UCDeltaCatalogClientImpl( + catalogName = "main", + ucClient = new StubUCDeltaClient(info)) + .loadTable(Identifier.of(Array("sch"), "tbl")) + .asInstanceOf[V1Table] + } + + val withoutPermission = loadTable(Collections.emptyList()).catalogTable.storage.properties + assert(!withoutPermission.contains(property)) + + val withPermission = loadTable(util.List.of("DATA_REORGANIZATION")) + .catalogTable.storage.properties + assert(withPermission.get(property) === Some("DATA_REORGANIZATION")) + } + test("loadTable falls back to SSP on CredentialFetchFailedException when SSP is enabled") { val tableId = UUID.randomUUID() val metadata = new TestMetadata() // no credential properties @@ -745,6 +775,7 @@ class AbstractDeltaCatalogClientRoutingSuite extends QueryTest with DeltaSQLComm "s3://bucket/no-creds-table", metadata, Collections.emptyMap(), // no storage properties either + Collections.emptyList(), Optional.empty()) val credEx = new CredentialFetchFailedException( "creds exhausted", new RuntimeException("simulated"), tableInfoNoCreds) @@ -788,6 +819,7 @@ class AbstractDeltaCatalogClientRoutingSuite extends QueryTest with DeltaSQLComm "s3://bucket/table", metadata, Collections.emptyMap(), + Collections.emptyList(), Optional.of(uniform)) val client = new UCDeltaCatalogClientImpl( @@ -819,6 +851,7 @@ class AbstractDeltaCatalogClientRoutingSuite extends QueryTest with DeltaSQLComm "s3://bucket/table", metadata, Collections.emptyMap(), + Collections.emptyList(), Optional.empty()) val client = new UCDeltaCatalogClientImpl( diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/catalog/UCDeltaCatalogClientImplMetricsSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/catalog/UCDeltaCatalogClientImplMetricsSuite.scala index 00dabf24276..19b0df7850b 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/catalog/UCDeltaCatalogClientImplMetricsSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/catalog/UCDeltaCatalogClientImplMetricsSuite.scala @@ -191,6 +191,7 @@ class UCDeltaCatalogClientImplMetricsSuite extends QueryTest job = None, notebook = None, clusterId = None, readVersion = None, isolationLevel = None, isBlindAppend = Some(false), + dataChange = None, operationMetrics = operationMetrics, userMetadata = None, tags = None, engineInfo = None, txnId = None, lastManifestCommit = None) diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/cdc/CDCReaderSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/cdc/CDCReaderSuite.scala index f7d09732845..bde3bbb09d7 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/cdc/CDCReaderSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/cdc/CDCReaderSuite.scala @@ -22,7 +22,7 @@ import java.io.File import com.databricks.spark.util.{Log4jUsageLogger, UsageRecord} import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.DeltaOperations.Delete -import org.apache.spark.sql.delta.DeltaTestUtils.BOOLEAN_DOMAIN +import org.apache.spark.sql.delta.DeltaTestUtils.{recordDataChangeInCommitInfo, BOOLEAN_DOMAIN} import org.apache.spark.sql.delta.actions.{Action, AddCDCFile, AddFile} import org.apache.spark.sql.delta.commands.cdc.CDCReader import org.apache.spark.sql.delta.commands.cdc.CDCReader._ @@ -108,6 +108,34 @@ class CDCReaderSuite .withColumn(CDC_COMMIT_VERSION, lit(commitVersion)) } + test("a commit recording dataChange = false yields no CDC rows") { + // The CommitInfo and the file actions are made to disagree, which no write path produces. It + // is the only way to observe which of the two the reader consulted: from the commit's own + // summary it infers no change, from the file actions an insert per added file. + Seq(true, false).foreach { readFromCommitInfo => + withSQLConf( + DeltaSQLConf.DELTA_COMMIT_INFO_DATA_CHANGE_READ_ENABLED.key -> + readFromCommitInfo.toString) { + withTempDir { dir => + val path = dir.getAbsolutePath + val data = spark.range(10) + data.write.format("delta").save(path) + recordDataChangeInCommitInfo( + DeltaLog.forTable(spark, path), version = 0, dataChange = Some(false)) + + val log = DeltaLog.forTable(spark, path) + val inserted = if (readFromCommitInfo) spark.range(0) else data + checkCDCAnswer( + log, + CDCReader.changesToBatchDF(log, 0, 0, spark), + inserted + .withColumn(CDC_TYPE_COLUMN_NAME, lit("insert")) + .withColumn(CDC_COMMIT_VERSION, lit(0))) + } + } + } + } + test("simple CDC scan") { withTempDir { dir => val log = DeltaLog.forTable(spark, dir.getAbsolutePath) diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/columnmapping/DropColumnMappingFeatureSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/columnmapping/DropColumnMappingFeatureSuite.scala index 64f31e61e40..38d15aac8e4 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/columnmapping/DropColumnMappingFeatureSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/columnmapping/DropColumnMappingFeatureSuite.scala @@ -18,8 +18,14 @@ package org.apache.spark.sql.delta.columnmapping import java.util.concurrent.TimeUnit -import org.apache.spark.sql.delta._ +import org.apache.spark.sql.delta.ColumnMappingTableFeature +import org.apache.spark.sql.delta.DeltaAnalysisException +import org.apache.spark.sql.delta.DeltaColumnMappingUnsupportedException +import org.apache.spark.sql.delta.DeltaConfigs import org.apache.spark.sql.delta.DeltaConfigs._ +import org.apache.spark.sql.delta.DeltaErrors +import org.apache.spark.sql.delta.DeltaLog +import org.apache.spark.sql.delta.DeltaTableFeatureException import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.sources.DeltaSQLConf._ diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/columnmapping/RemoveColumnMappingSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/columnmapping/RemoveColumnMappingSuite.scala index f11a25f2c00..54a65a3f877 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/columnmapping/RemoveColumnMappingSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/columnmapping/RemoveColumnMappingSuite.scala @@ -18,7 +18,11 @@ package org.apache.spark.sql.delta.columnmapping import io.delta.tables.DeltaTable -import org.apache.spark.sql.delta._ +import org.apache.spark.sql.delta.DeltaAnalysisException +import org.apache.spark.sql.delta.DeltaColumnMappingUnsupportedException +import org.apache.spark.sql.delta.DeltaConfigs +import org.apache.spark.sql.delta.DeltaLog +import org.apache.spark.sql.delta.GeneratedColumn import org.apache.spark.sql.delta.schema.DeltaInvariantViolationException import org.apache.spark.sql.delta.sources.DeltaSQLConf._ diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/columnmapping/RemoveColumnMappingSuiteUtils.scala b/spark/src/test/scala/org/apache/spark/sql/delta/columnmapping/RemoveColumnMappingSuiteUtils.scala index dc9be3c805d..d6662f33f05 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/columnmapping/RemoveColumnMappingSuiteUtils.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/columnmapping/RemoveColumnMappingSuiteUtils.scala @@ -16,8 +16,13 @@ package org.apache.spark.sql.delta.columnmapping -import org.apache.spark.sql.delta._ +import org.apache.spark.sql.delta.DeltaColumnMapping +import org.apache.spark.sql.delta.DeltaColumnMappingSuiteUtils +import org.apache.spark.sql.delta.DeltaConfigs +import org.apache.spark.sql.delta.DeltaHistory +import org.apache.spark.sql.delta.DeltaLog import org.apache.spark.sql.delta.DeltaOperations.RemoveColumnMapping +import org.apache.spark.sql.delta.Snapshot import org.apache.spark.sql.delta.actions.AddFile import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.schema.SchemaMergingUtils @@ -59,10 +64,6 @@ trait RemoveColumnMappingSuiteUtils extends QueryTest with DeltaColumnMappingSui protected val testTableName: String = "test_table_" + this.getClass.getSimpleName protected def deltaLog = DeltaLog.forTable(spark, TableIdentifier(testTableName)) - // Hook for subclasses to route DDL/DML through a specific connector mode (e.g. V1 for V2 suites - // that require DDL to go through the V1 connector). Defaults to `sql`. - protected def executeDml(sqlText: String): Unit = sql(sqlText) - import testImplicits._ protected def testRemovingColumnMapping(unsetTableProperty: Boolean = false): Any = { diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/concurrency/TransactionExecutionTestMixin.scala b/spark/src/test/scala/org/apache/spark/sql/delta/concurrency/TransactionExecutionTestMixin.scala index 2b1a83e6850..f104dda1299 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/concurrency/TransactionExecutionTestMixin.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/concurrency/TransactionExecutionTestMixin.scala @@ -28,11 +28,13 @@ import org.apache.spark.sql.delta.fuzzer.{OptimisticTransactionPhases, PhaseLock import org.apache.spark.SparkException import org.apache.spark.internal.Logging import org.apache.spark.sql.Row -import org.apache.spark.sql.test.SharedSparkSession +import org.apache.spark.sql.SparkSession import org.apache.spark.util.ThreadUtils trait TransactionExecutionTestMixin { - self: PhaseLockingTestMixin with SharedSparkSession with Logging => + self: PhaseLockingTestMixin with Logging => + + protected def spark: SparkSession /** * Timeout used when waiting for individual phases of instrumented operations to complete. diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/coordinatedcommits/CatalogManagedStreamingSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/coordinatedcommits/CatalogManagedStreamingSuite.scala index 7bbad7c939c..5334064e2ba 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/coordinatedcommits/CatalogManagedStreamingSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/coordinatedcommits/CatalogManagedStreamingSuite.scala @@ -21,6 +21,7 @@ import org.apache.spark.sql.delta.test.{DeltaSQLCommandTest, DeltaSQLTestUtils} import org.apache.spark.sql.delta.test.shims.StreamingTestShims.MemoryStream import org.apache.spark.SparkConf +import org.apache.spark.sql.QueryTest import org.apache.spark.sql.streaming.StreamTest /** @@ -30,10 +31,11 @@ import org.apache.spark.sql.streaming.StreamTest * on the tracking client will not be deterministic. */ trait CatalogManagedStreamingSuiteBase - extends StreamTest - with DeltaSQLTestUtils + extends QueryTest with DeltaSQLCommandTest - with CatalogOwnedTestBaseSuite { + with DeltaSQLTestUtils + with CatalogOwnedTestBaseSuite + with StreamTest { import testImplicits._ @@ -64,37 +66,53 @@ trait CatalogManagedStreamingSuiteBase getTrackingClient.foreach(_.reset()) } + + /** Runs test-data setup through V1 when a subclass forces Delta V2. */ + protected def withV1Mode(f: => Unit): Unit = f + + /** + * Provides a checkpoint location for streaming tests. Defaults to a local temp dir; subclasses + * can override it to run the same test bodies against a different storage backend. + */ + protected def withCheckpointDir(f: String => Unit): Unit = + withTempDir(dir => f(dir.getAbsolutePath)) + protected override def beforeEach(): Unit = { super.beforeEach() resetTrackingClient() } test("stream from delta source") { - withTempTable(createTable = false) { sourceTableName => - sql(s"CREATE TABLE $sourceTableName (value INT) USING delta") - - val df = spark.readStream - .format("delta") - .table(sourceTableName) - - resetTrackingClient() - - testStream(df)( - Execute{ _ => - Seq(1, 2).toDF().write.format("delta").mode("append").saveAsTable(sourceTableName) - }, - ProcessAllAvailable(), - CheckAnswer(1, 2), - Execute { _ => assertNumCommitsCalled(1) }, - // At least one read from the commit and one from checking the result - Execute { _ => assertNumGetCommitsCalled(2) } - ) + withCheckpointDir { checkpointPath => + withTempTable(createTable = false) { sourceTableName => + sql(s"CREATE TABLE $sourceTableName (value INT) USING delta") + + val df = spark.readStream + .format("delta") + .table(sourceTableName) + + resetTrackingClient() + + testStream(df)( + StartStream(checkpointLocation = checkpointPath), + Execute{ _ => + withV1Mode { + Seq(1, 2).toDF().write.format("delta").mode("append").saveAsTable(sourceTableName) + } + }, + ProcessAllAvailable(), + CheckAnswer(1, 2), + Execute { _ => assertNumCommitsCalled(1) }, + // At least one read from the commit and one from checking the result + Execute { _ => assertNumGetCommitsCalled(2) } + ) + } } } test("stream to delta sink") { // The dir is only used as the checkpoint location and doesn't imply a path-based access. - withTempDir { tempDir => + withCheckpointDir { checkpointPath => withTempTable(createTable = false) { sinkTableName => var expectedNumCommits = 0 var expectedNumGetCommits = 0 @@ -104,7 +122,7 @@ trait CatalogManagedStreamingSuiteBase .toDF() .writeStream .format("delta") - .option("checkpointLocation", tempDir.getAbsolutePath) + .option("checkpointLocation", checkpointPath) .toTable(sinkTableName) query.processAllAvailable() @@ -138,7 +156,7 @@ trait CatalogManagedStreamingSuiteBase } test("stream from delta source to delta sink with shared commit coordinator") { - withTempDir { tempDir => + withCheckpointDir { checkpointPath => withTempTable(createTable = false) { sourceTableName => withTempTable(createTable = false) { sinkTableName => var expectedNumCommits = 0 @@ -154,7 +172,7 @@ trait CatalogManagedStreamingSuiteBase .toDF() .writeStream .format("delta") - .option("checkpointLocation", tempDir.getAbsolutePath) + .option("checkpointLocation", checkpointPath) .toTable(sinkTableName) query.processAllAvailable() expectedNumCommits += 1 @@ -163,7 +181,9 @@ trait CatalogManagedStreamingSuiteBase assertNumGetCommitsCalled(expectedNumGetCommits) try { - Seq(1).toDF().write.format("delta").mode("append").saveAsTable(sourceTableName) + withV1Mode { + Seq(1).toDF().write.format("delta").mode("append").saveAsTable(sourceTableName) + } query.processAllAvailable() // One commit to the source table and one commit to the sink table expectedNumCommits += 2 @@ -176,7 +196,9 @@ trait CatalogManagedStreamingSuiteBase assertNumCommitsCalled(expectedNumCommits) assertNumGetCommitsCalled(expectedNumGetCommits) - Seq(2).toDF().write.format("delta").mode("append").saveAsTable(sourceTableName) + withV1Mode { + Seq(2).toDF().write.format("delta").mode("append").saveAsTable(sourceTableName) + } query.processAllAvailable() // One commit to the source table and one commit to the sink table expectedNumCommits += 2 diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/coordinatedcommits/CommitCoordinatorClientImplSuiteBase.scala b/spark/src/test/scala/org/apache/spark/sql/delta/coordinatedcommits/CommitCoordinatorClientImplSuiteBase.scala index e0288d94ce8..61ccfec34a8 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/coordinatedcommits/CommitCoordinatorClientImplSuiteBase.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/coordinatedcommits/CommitCoordinatorClientImplSuiteBase.scala @@ -361,7 +361,7 @@ trait CommitCoordinatorClientImplSuiteBase extends QueryTest } } } - tasks.foreach(ThreadUtils.awaitResult(_, 150.seconds)) + tasks.foreach(ThreadUtils.awaitResult(_, 5.minutes)) } catch { case e: InterruptedException => fail("Test interrupted: " + e.getMessage) diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/coordinatedcommits/CoordinatedCommitsSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/coordinatedcommits/CoordinatedCommitsSuite.scala index 81a42271e1f..ce7e70dc30f 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/coordinatedcommits/CoordinatedCommitsSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/coordinatedcommits/CoordinatedCommitsSuite.scala @@ -1185,7 +1185,8 @@ abstract class CommitCoordinatorSuiteBase Some(newProtocol), DeltaOperations.TestOperation("TEST"), Map.empty, - Map.empty) + Map.empty, + dataChange = Some(false)) log = DeltaLog.forTable(spark, tablePath) assert(cs.numRegisterTableCalled.get === 1) assert(cs.numCommitsCalled.get === 0) diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/coordinatedcommits/CoordinatedCommitsTestUtils.scala b/spark/src/test/scala/org/apache/spark/sql/delta/coordinatedcommits/CoordinatedCommitsTestUtils.scala index 363dcf4cc7a..b9ef109f1c0 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/coordinatedcommits/CoordinatedCommitsTestUtils.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/coordinatedcommits/CoordinatedCommitsTestUtils.scala @@ -34,6 +34,8 @@ import io.delta.storage.commit.actions.{AbstractMetadata, AbstractProtocol} import io.delta.storage.commit.uccommitcoordinator.UCCommitCoordinatorClient import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path +import org.scalactic.source.Position +import org.scalatest.Tag import org.apache.spark.{SparkConf, SparkFunSuite} import org.apache.spark.sql.{QueryTest, Row, SparkSession} import org.apache.spark.sql.catalyst.{TableIdentifier => CatalystTableIdentifier} @@ -41,6 +43,9 @@ import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.functions.col import org.apache.spark.sql.test.SharedSparkSession +case object CatalogManagedMaintenanceIncompatible + extends Tag("CatalogManagedMaintenanceIncompatible") + // This trait is built to serve as a base trait for tests built for both CatalogOwned // and commit-coordinators table feature. trait CommitCoordinatorUtilBase { @@ -169,6 +174,17 @@ trait CatalogOwnedTestBaseSuite } } + override protected def test(testName: String, testTags: Tag*)(testFun: => Any)( + implicit pos: Position): Unit = { + super.test(testName, testTags: _*) { + if (catalogOwnedDefaultCreationEnabledInTests && + testTags.contains(CatalogManagedMaintenanceIncompatible)) { + cancel("Maintenance operation is not supported for catalog-managed tables") + } + testFun + } + } + override def testWithDifferentBackfillInterval(testName: String)(f: Int => Unit): Unit = { Seq(1, 2, 10).foreach { backfillBatchSize => test(s"$testName [Backfill batch size: $backfillBatchSize]") { diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/coordinatedcommits/StagedCommitListingSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/coordinatedcommits/StagedCommitListingSuite.scala new file mode 100644 index 00000000000..c45ab281f96 --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/coordinatedcommits/StagedCommitListingSuite.scala @@ -0,0 +1,301 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.spark.sql.delta.coordinatedcommits + +import java.io.File + +import org.apache.spark.sql.delta._ +import org.apache.spark.sql.delta.DeltaTestUtils.verifyUnbackfilled +import org.apache.spark.sql.delta.sources.DeltaSQLConf +import org.apache.spark.sql.delta.storage.LocalLogStore +import org.apache.spark.sql.delta.storage.LogStore.logStoreClassConfKey +import org.apache.spark.sql.delta.test.DeltaSQLCommandTest +import org.apache.spark.sql.delta.util.FileNames +import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.fs.{FileStatus, Path} + +import org.apache.spark.SparkConf +import org.apache.spark.sql.QueryTest + +/** + * On a Catalog Owned table the fs listing must not promote a staged-commit file + * (`_staged_commits/N..json`) to a commit. [[RecursiveListingLocalLogStore]] reproduces a + * recursive-listing LogStore hermetically (no real object store). + */ +class StagedCommitListingSuite extends QueryTest + with CatalogOwnedTestBaseSuite + with WithCatalogOwnedBatch100 + with DeltaSQLCommandTest { + + override protected def sparkConf: SparkConf = + super.sparkConf.set(logStoreClassConfKey, classOf[RecursiveListingLocalLogStore].getName) + + /** Runs `f` with Catalog Owned (CCv2) and staged-commit filtering enabled. */ + private def withCatalogOwnedCommits(f: => Unit): Unit = withDefaultCCTableFeature { + withSQLConf( + DeltaSQLConf.DELTA_SNAPSHOT_FILESYSTEM_LISTING_FILTER_STAGED_COMMITS_ENABLED.key -> + "true") { + f + } + } + + private def logDir(path: String): File = new File(DeltaLog.logPathFor(path).toString) + + private def stagedCommitsDir(path: String): File = + new File(logDir(path), FileNames.COMMIT_SUBDIR) + + private def stagedCommitFiles(path: String): Seq[File] = + Option(stagedCommitsDir(path).listFiles()).getOrElse(Array.empty) + .filter(_.getName.endsWith(".json")) + .sortBy(_.getName) + .toSeq + + private def assertStagedVersions(path: String, expectedVersions: Seq[Long]): Seq[File] = { + val staged = stagedCommitFiles(path) + assert(staged.map(f => FileNames.deltaVersion(new Path(f.getName))) === expectedVersions) + staged + } + + private def backfilledDeltaFile(path: String, version: Long): File = + new File(FileNames.unsafeDeltaFile(new Path(logDir(path).toString), version).toString) + + /** The [[InMemoryCommitCoordinator]] backing the builder, for manipulating its ledger. */ + private def getInMemoryCoordinator: InMemoryCommitCoordinator = { + val client = getCatalogOwnedCommitCoordinatorClient( + CatalogOwnedTableUtils.DEFAULT_CATALOG_NAME_FOR_TESTING) + client match { + case tracking: TrackingCommitCoordinatorClient => + tracking.delegatingCommitCoordinatorClient.asInstanceOf[InMemoryCommitCoordinator] + case inMemory: InMemoryCommitCoordinator => inMemory + case other => + throw new IllegalStateException( + s"Unexpected commit coordinator client type: ${other.getClass.getName}") + } + } + + // Sanity check: the harness actually surfaces staged files, else the tests below pass vacuously. + test("test harness: recursive listing surfaces _staged_commits files") { + withCatalogOwnedCommits { + withTempDir { tempDir => + val path = tempDir.getCanonicalPath + spark.range(0, 10).write.format("delta").save(path) + // Append v1; with batch size 100 it stays staged (un-backfilled). + spark.range(10, 20).write.format("delta").mode("append").save(path) + + assertStagedVersions(path, Seq(1L)) + + val deltaLog = DeltaLog.forTable(spark, path) + val logPath = deltaLog.logPath + val listed = deltaLog.store + .listFrom(FileNames.listingPrefix(logPath, 0L), deltaLog.newDeltaHadoopConf()) + .map(_.getPath) + .toSeq + val listedStagedVersions = listed + .filter(_.getParent.getName == FileNames.COMMIT_SUBDIR) + .map(FileNames.deltaVersion) + assert(listedStagedVersions === Seq(1L), + s"recursive listing should surface exactly staged version 1, but listed: $listed") + } + } + } + + test("filesystem listing keeps checkpoints and minor compactions") { + withCatalogOwnedCommits { + withTempDir { tempDir => + val path = tempDir.getCanonicalPath + spark.range(0, 10).write.format("delta").save(path) // v0 + spark.range(10, 20).write.format("delta").mode("append").save(path) // v1 (staged) + assertStagedVersions(path, Seq(1L)) + + val deltaLog = DeltaLog.forTable(spark, path) + val hadoopConf = deltaLog.newDeltaHadoopConf() + deltaLog.store.write( + FileNames.checkpointFileSingular(deltaLog.logPath, 0L), + Iterator("{}"), + overwrite = false, + hadoopConf) + deltaLog.store.write( + FileNames.compactedDeltaFile(deltaLog.logPath, 0L, 0L), + Iterator("{}"), + overwrite = false, + hadoopConf) + + def listedFileTypes( + includeMinorCompactions: Boolean): Set[(FileNames.FileType.Value, Long)] = { + deltaLog.listFromFileSystemInternal( + startVersion = 0L, + versionToLoad = None, + includeMinorCompactions = includeMinorCompactions) + ._1 + .getOrElse(Array.empty) + .map { case (_, fileType, version) => fileType -> version } + .toSet + } + + assert(listedFileTypes(includeMinorCompactions = false) === Set( + FileNames.FileType.DELTA -> 0L, + FileNames.FileType.CHECKPOINT -> 0L)) + assert(listedFileTypes(includeMinorCompactions = true) === Set( + FileNames.FileType.DELTA -> 0L, + FileNames.FileType.CHECKPOINT -> 0L, + FileNames.FileType.COMPACTED_DELTA -> 0L)) + } + } + } + + testWithCatalogOwned(backfillBatchSize = 1)( + "rollout flag controls staged-commit filtering for a backfilled version") { + withTempDir { tempDir => + val path = tempDir.getCanonicalPath + spark.range(0, 10).write.format("delta").save(path) // v0 + spark.range(10, 20).write.format("delta").mode("append").save(path) // v1 (backfilled) + + val stagedV1 = assertStagedVersions(path, Seq(1L)).head + val backfilledV1 = backfilledDeltaFile(path, 1L) + assert(backfilledV1.exists() && stagedV1.exists(), + "batch size 1 must leave both backfilled and staged copies of v1") + + DeltaLog.invalidateCache(spark, new Path(path)) + withSQLConf( + DeltaSQLConf.DELTA_SNAPSHOT_FILESYSTEM_LISTING_FILTER_STAGED_COMMITS_ENABLED.key -> + "false") { + val error = intercept[DeltaIllegalStateException] { + DeltaLog.forTable(spark, path).update() + } + assert(error.getErrorClass.startsWith("DELTA_VERSIONS_NOT_CONTIGUOUS")) + } + + DeltaLog.invalidateCache(spark, new Path(path)) + withSQLConf( + DeltaSQLConf.DELTA_SNAPSHOT_FILESYSTEM_LISTING_FILTER_STAGED_COMMITS_ENABLED.key -> + "true") { + val snapshot = DeltaLog.forTable(spark, path).update() + assert(snapshot.version == 1L) + checkAnswer( + spark.read.format("delta").load(path).orderBy("id"), + (0 until 20).map(i => org.apache.spark.sql.Row(i.toLong))) + } + } + } + + test("orphaned staged commit (not in getCommits) is not read") { + withCatalogOwnedCommits { + withTempDir { tempDir => + val path = tempDir.getCanonicalPath + spark.range(0, 10).write.format("delta").save(path) // v0 + // Write a genuine, well-formed v1 commit. With batch size 100 it stays staged. + spark.range(10, 20).write.format("delta").mode("append").save(path) // v1 (staged) + + val deltaLog = DeltaLog.forTable(spark, path) + val logPath = deltaLog.logPath + + val stagedV1 = assertStagedVersions(path, Seq(1L)).head + val backfilledV1 = backfilledDeltaFile(path, 1L) + assert(!backfilledV1.exists(), "batch size 100 must not backfill v1") + + // Orphan v1 by dropping it from the coordinator. Its staged file remains on disk. + val coordinator = getInMemoryCoordinator + coordinator.removeCommitTestOnly(logPath, commitVersion = 1L) + assert(stagedV1.exists(), "orphaning v1 must not delete its staged file") + assertStagedVersions(path, Seq(1L)) + assert(!backfilledV1.exists(), "orphaned v1 must not have a backfilled copy") + + DeltaLog.invalidateCache(spark, new Path(path)) + + val snapshot = DeltaLog.forTable(spark, path).update() + assert(snapshot.version == 0L, + s"orphaned staged commit must not advance the snapshot; got v${snapshot.version}") + checkAnswer( + spark.read.format("delta").load(path).orderBy("id"), + (0 until 10).map(i => org.apache.spark.sql.Row(i.toLong))) + } + } + } + + test("ratified, not-yet-backfilled staged commit is still read via the coordinator") { + withCatalogOwnedCommits { + withTempDir { tempDir => + val path = tempDir.getCanonicalPath + spark.range(0, 10).write.format("delta").save(path) // v0 + spark.range(10, 20).write.format("delta").mode("append").save(path) // v1 (ratified, staged) + + // v1 is ratified by the coordinator but, with batch size 100, still only exists staged. + assertStagedVersions(path, Seq(1L)) + assert(!backfilledDeltaFile(path, 1L).exists(), + "v1 should NOT be backfilled yet (batch size is large)") + + DeltaLog.invalidateCache(spark, new Path(path)) + val deltaLog = DeltaLog.forTable(spark, path) + + // Channel check: the raw fs listing must NOT surface v1 (else we wouldn't be testing the + // coordinator path). + val (fsListingOpt, _) = deltaLog.listFromFileSystemInternal( + startVersion = 0L, + versionToLoad = None, + includeMinorCompactions = false) + val fsDeltaVersions = fsListingOpt.getOrElse(Array.empty) + .collect { case (f, FileNames.FileType.DELTA, v) => v }.toSeq + assert(!fsDeltaVersions.contains(1L), + s"raw fs listing must not surface staged v1 as a delta; got $fsDeltaVersions") + + val snapshot = deltaLog.update() + assert(snapshot.version == 1L, + s"ratified staged v1 must be read via the coordinator; got v${snapshot.version}") + // The v1 delta is the staged (unbackfilled) file, i.e. it came from the coordinator. + val v1Delta = snapshot.logSegment.deltas + .find(f => FileNames.deltaVersion(f.getPath) == 1L) + .getOrElse(fail("v1 delta missing from the log segment")) + verifyUnbackfilled(v1Delta) + + checkAnswer( + spark.read.format("delta").load(path).orderBy("id"), + (0 until 20).map(i => org.apache.spark.sql.Row(i.toLong))) + } + } + } +} + +/** + * A [[LocalLogStore]] whose `listFrom` recurses into subdirectories, reproducing a recursive + * LogStore that surfaces `_staged_commits/` files (the default [[LocalLogStore]] lists only + * immediate children and masks the bug). Top-level so it can be set via the logStore class conf. + */ +class RecursiveListingLocalLogStore(sparkConf: SparkConf, hadoopConf: Configuration) + extends LocalLogStore(sparkConf, hadoopConf) { + + override def listFrom(path: Path, hadoopConf: Configuration): Iterator[FileStatus] = { + val fs = path.getFileSystem(hadoopConf) + // `path` is the first file name to return. List from its containing log directory. + val logDir = path.getParent + if (!fs.exists(logDir)) { + throw DeltaErrors.fileOrDirectoryNotFoundException(s"$logDir") + } + + // This mirrors an object-store prefix listing without a delimiter. + val collected = scala.collection.mutable.ArrayBuffer.empty[FileStatus] + val remoteFiles = fs.listFiles(logDir, true) + while (remoteFiles.hasNext) { + collected += remoteFiles.next() + } + // Return files with name >= the start name, sorted by leaf name (as the S3 log store does). + collected + .filter(_.getPath.getName >= path.getName) + .sortBy(_.getPath.getName) + .iterator + } +} diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/deletionvectors/DeletionVectorsSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/deletionvectors/DeletionVectorsSuite.scala index b5abd7a62c1..cf9f69ff626 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/deletionvectors/DeletionVectorsSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/deletionvectors/DeletionVectorsSuite.scala @@ -18,11 +18,12 @@ package org.apache.spark.sql.delta.deletionvectors import java.io.{File, FileNotFoundException} import java.net.URISyntaxException +import java.util.UUID import org.apache.spark.sql.delta.{DeletionVectorsTableFeature, DeletionVectorsTestUtils, DeltaChecksumException, DeltaConfigs, DeltaLog, DeltaMetricsUtils, DeltaTestUtilsForTempViews} import org.apache.spark.sql.delta.DeltaTestUtils.createTestAddFile import org.apache.spark.sql.delta.actions.{AddFile, DeletionVectorDescriptor, RemoveFile} -import org.apache.spark.sql.delta.actions.DeletionVectorDescriptor.EMPTY +import org.apache.spark.sql.delta.actions.DeletionVectorDescriptor._ import org.apache.spark.sql.delta.deletionvectors.DeletionVectorsSuite._ import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.{DeltaExceptionTestUtils, DeltaSQLCommandTest} @@ -38,6 +39,7 @@ import org.apache.parquet.hadoop.ParquetFileReader import org.apache.spark.SparkConf import org.apache.spark.SparkException +import org.apache.spark.paths.SparkPath import org.apache.spark.sql.{DataFrame, QueryTest, Row} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.plans.logical.{AppendData, Subquery} @@ -45,6 +47,7 @@ import org.apache.spark.sql.execution.FileSourceScanLike import org.apache.spark.sql.functions.col import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SharedSparkSession +import org.apache.spark.sql.types.{StringType, StructField, StructType} class DeletionVectorsSuite extends QueryTest with SharedSparkSession @@ -734,6 +737,56 @@ class DeletionVectorsSuite extends QueryTest } } + test("object uniqueIdExpression matches descriptor uniqueId") { + val tableRoot = new Path("s3a://table/test path") + val uuid = UUID.randomUUID() + val uDv = onDiskWithUuidRelativePath( + uuid, randomPrefix = "prefix + %space", sizeInBytes = 15, cardinality = 25, offset = Some(8)) + val rDv = rFormOf(uuid, "prefix + %space", offset = Some(8)) + val pDv = onDiskWithAbsolutePath( + SparkPath.fromPath(new Path(tableRoot, "data/a space + b.bin")).urlEncoded, + sizeInBytes = 15, + cardinality = 25, + offset = Some(8)) + val pElsewhere = onDiskWithAbsolutePath( + SparkPath.fromPath( + new Path(s"s3a://other-bucket/tbl/prefix/${DELETION_VECTOR_FILE_NAME_CORE}_$uuid.bin")) + .urlEncoded, + sizeInBytes = 15, + cardinality = 25, + offset = Some(8)) + + val testCases = Seq(uDv, rDv, pDv, pElsewhere).map { dv => + ( + dv, + dv.uniqueId(tableRoot, useObjectIdentity = true), + dv.uniqueId(tableRoot, useObjectIdentity = false)) + } + val rows = testCases.map { case (dv, expectedObjectId, expectedLegacyId) => + Row(dvStructRow(dv), expectedObjectId, expectedLegacyId) + } + val input = spark.createDataFrame( + spark.sparkContext.parallelize(rows), + dvExpressionTestSchema) + val actual = input.select( + DeletionVectorDescriptor + .uniqueIdExpression(col("dv"), tableRoot, useObjectIdentity = true) + .as("actualObjectId"), + col("expectedObjectId"), + DeletionVectorDescriptor + .uniqueIdExpression(col("dv"), tableRoot, useObjectIdentity = false) + .as("actualLegacyId"), + col("expectedLegacyId")) + + checkAnswer(actual, testCases.map { case (_, expectedObjectId, expectedLegacyId) => + Row( + expectedObjectId, + expectedObjectId, + expectedLegacyId, + expectedLegacyId) + }) + } + test("Check no resource leak when DV files are missing (table corrupted)") { withTempDir { tempDir => val source = new File(table2Path) @@ -837,6 +890,36 @@ class DeletionVectorsSuite extends QueryTest val optimizedPlan = queryDf.queryExecution.analyzed.toString() assert(optimizedPlan.contains(expected), s"Plan is missing `$expected`: $optimizedPlan") } + + /** The `r` form of the `u` DV that `uuid`/`prefix` describe. */ + private def rFormOf( + uuid: UUID, + prefix: String, + offset: Option[Int]): DeletionVectorDescriptor = { + val path = if (prefix.isEmpty) { + s"${DELETION_VECTOR_FILE_NAME_CORE}_$uuid.bin" + } else { + s"$prefix/${DELETION_VECTOR_FILE_NAME_CORE}_$uuid.bin" + } + createRelativePathDVDescriptor(path, sizeInBytes = 15, cardinality = 25, offset = offset) + } + + private def dvStructRow(dv: DeletionVectorDescriptor): Row = { + Row.fromSeq(DeletionVectorDescriptor.STRUCT_TYPE.fieldNames.map { + case "storageType" => dv.storageType + case "pathOrInlineDv" => dv.pathOrInlineDv + case "offset" => dv.offset.map(Int.box).orNull + case "sizeInBytes" => Int.box(dv.sizeInBytes) + case "cardinality" => Long.box(dv.cardinality) + case "maxRowIndex" => dv.maxRowIndex.map(Long.box).orNull + case field => fail(s"Unexpected deletion vector field in test schema: $field") + }) + } + + private def dvExpressionTestSchema = StructType(Seq( + StructField("dv", DeletionVectorDescriptor.STRUCT_TYPE, nullable = false), + StructField("expectedObjectId", StringType, nullable = false), + StructField("expectedLegacyId", StringType, nullable = false))) } object DeletionVectorsSuite { diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/deletionvectors/ManifestBitmapSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/deletionvectors/ManifestBitmapSuite.scala new file mode 100644 index 00000000000..09fa3aafbc3 --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/deletionvectors/ManifestBitmapSuite.scala @@ -0,0 +1,62 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.deletionvectors + +import org.apache.spark.SparkFunSuite + +class ManifestBitmapSuite extends SparkFunSuite { + + test("mutating a copy does not change the original bitmap") { + val original = ManifestBitmap.fromPositions(Seq(1, 3)) + val mutable = original.toMutable + + mutable.add(2) + + assert(original.toArrayForTesting.sameElements(Array(1L, 3L))) + assert(mutable.toManifestBitmap.toArrayForTesting.sameElements(Array(1L, 2L, 3L))) + } + + test("immutable snapshots do not change after further mutations") { + val mutable = ManifestBitmap.fromPositions(Seq(1)).toMutable + mutable.add(2) + val firstSnapshot = mutable.toManifestBitmap + + mutable.add(3) + val secondSnapshot = mutable.toManifestBitmap + + assert(firstSnapshot.toArrayForTesting.sameElements(Array(1L, 2L))) + assert(secondSnapshot.toArrayForTesting.sameElements(Array(1L, 2L, 3L))) + } + + test("deserialized bitmaps own their input bytes") { + val bytes = ManifestBitmap.fromPositions(Seq(1, 3)).serializeAsByteArray() + val bitmap = ManifestBitmap.fromSerializedByteArray(bytes) + + java.util.Arrays.fill(bytes, 0.toByte) + + assert(bitmap.toArrayForTesting.sameElements(Array(1L, 3L))) + } + + test("bitmaps created from positions own their input sequence") { + val positions = collection.mutable.ArrayBuffer(1, 3) + val bitmap = ManifestBitmap.fromPositions(positions) + + positions += 2 + + assert(bitmap.toArrayForTesting.sameElements(Array(1L, 3L))) + } +} diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/deletionvectors/MutableMumblingBitmapSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/deletionvectors/MutableMumblingBitmapSuite.scala new file mode 100644 index 00000000000..6f5991b904a --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/deletionvectors/MutableMumblingBitmapSuite.scala @@ -0,0 +1,380 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.deletionvectors + +import java.nio.ByteBuffer + +import scala.util.Random + +import org.apache.spark.sql.delta.deletionvectors.mumbling.{MumblingBitmap, MumblingBitmapWriter} + +import org.apache.spark.SparkFunSuite + +class MutableMumblingBitmapSuite extends SparkFunSuite { + + private def newMutableBitmap(positions: Long*): MutableMumblingBitmap = { + val empty = new MumblingBitmap( + ByteBuffer.wrap(MumblingBitmapWriter.serialize(Array.emptyIntArray))) + val bitmap = new MutableMumblingBitmap(empty) + positions.foreach(bitmap.add) + bitmap + } + + private def bufferBytes(bitmap: MumblingBitmap): Array[Byte] = { + val buffer = bitmap.buffer() + val bytes = new Array[Byte](buffer.remaining()) + buffer.get(bytes) + bytes + } + + private def merge(basePositions: Seq[Int], additions: Int*): MumblingBitmap = { + val base = new MumblingBitmap( + ByteBuffer.wrap(MumblingBitmapWriter.serialize(basePositions.toArray))) + val mutableBitmap = base.toMutable() + additions.foreach(position => mutableBitmap.add(position.toLong)) + mutableBitmap.toMumblingBitmap + } + + private def singleContainerDescriptor(bitmap: MumblingBitmap): Int = { + val bytes = bufferBytes(bitmap) + assert(bytes(4) == 1 && bytes(5) == 0, "expected exactly one container") + // PFOR stores a single descriptor as the chunk minimum at byte 8. + bytes(8) & 0xff + } + + /** Serialize as Mumbling, read back, and assert the positions survive a full round-trip. */ + private def roundTrip(positions: Seq[Long], name: String): Unit = { + val decoded = newMutableBitmap(positions: _*).toMumblingBitmap + val bytes = bufferBytes(decoded) + // A Mumbling bitmap always begins with the version byte 1. + assert(bytes.nonEmpty && bytes(0) == 1, s"$name: expected the Mumbling version byte") + val expected = positions.toSet + assert(decoded.cardinality() == expected.size, s"$name: cardinality") + expected.foreach(p => assert(decoded.isSet(p.toInt), s"$name: should contain $p")) + // Spot-check absent positions around the populated range. + val maxPos = if (expected.isEmpty) 0L else expected.max + (0L to math.min(maxPos + 300L, 3000L)).foreach { p => + assert(decoded.isSet(p.toInt) == expected.contains(p), s"$name: contains mismatch at $p") + } + } + + test("round-trip across sparse, dense, mixed, and container boundaries") { + roundTrip(Seq.empty, "empty") + roundTrip(Seq(0L), "single-0") + roundTrip(Seq(255L), "single-255") + roundTrip(Seq(0L, 5L, 100L, 255L), "sparse") + roundTrip((0 until 31).map(_.toLong * 8L), "full-sparse-31") + roundTrip((0 until 32).map(_.toLong), "dense-32") + roundTrip((0 until 256).map(_.toLong), "dense-256") + roundTrip((0 until 128).map(_.toLong * 2L), "dense-even-positions") + roundTrip(Seq(0L, 255L, 256L, 257L, 512L), "cross-container") + roundTrip(Seq(5L, 522L), "gap-with-empty-middle-container") + roundTrip((0 until 32).map(_.toLong) :+ 257L, "mixed-dense-and-sparse") + roundTrip( + Seq(MumblingBitmapWriter.MAX_POSITION_EXCLUSIVE.toLong - 1L), "maximum-position") + } + + test("empty bitmap serializes to a 6-byte header and reads back empty") { + val bitmap = newMutableBitmap().toMumblingBitmap + val bytes = bufferBytes(bitmap) + assert(bytes.length == 6) + assert(bitmap.cardinality() == 0) + } + + test("mutable bitmap produces independent immutable snapshots") { + val mutable = newMutableBitmap() + mutable.add(10L) + mutable.add(300L) + mutable.add(10L) // duplicate + val firstSnapshot = mutable.toMumblingBitmap + + mutable.add(11L) + val secondSnapshot = mutable.toMumblingBitmap + + assert(firstSnapshot.cardinality() == 2) + assert(firstSnapshot.isSet(10) && firstSnapshot.isSet(300)) + assert(!firstSnapshot.isSet(11)) + assert(secondSnapshot.cardinality() == 3) + assert(secondSnapshot.isSet(11)) + } + + test("reader rejects Roaring-serialized bitmaps") { + val roaringBytes = RoaringBitmapArray(0L, 7L, 300L) + .serializeAsByteArray(RoaringBitmapArrayFormat.Portable) + + val error = intercept[UnsupportedOperationException] { + new MumblingBitmap(ByteBuffer.wrap(roaringBytes)) + } + assert(error.getMessage.startsWith("Unsupported Mumbling bitmap version:")) + } + + test("immutable bitmap owns its serialized bytes") { + val bytes = MumblingBitmapWriter.serialize(Array(10, 300)) + val bitmap = new MumblingBitmap(ByteBuffer.wrap(bytes)) + + java.util.Arrays.fill(bytes, 0.toByte) + + assert(bitmap.cardinality() == 2) + assert(bitmap.isSet(10)) + assert(bitmap.isSet(300)) + } + + test("add rejects positions outside [0, container-index ceiling)") { + val bitmap = newMutableBitmap() + intercept[IllegalArgumentException](bitmap.add(-1L)) + intercept[IllegalArgumentException]( + bitmap.add(MumblingBitmapWriter.MAX_POSITION_EXCLUSIVE.toLong)) + // The largest representable position is allowed. + bitmap.add(MumblingBitmapWriter.MAX_POSITION_EXCLUSIVE.toLong - 1L) + assert(bitmap.toMumblingBitmap.cardinality() == 1) + } + + test("merge adds positions without changing existing positions") { + val existing = newMutableBitmap( + 0L, 2L, 30L, 31L, 255L, 256L, 522L).toMumblingBitmap + val mutableBitmap = existing.toMutable() + Seq(1024L, 300L, 2L, 256L, 32L).foreach(mutableBitmap.add) + val bitmap = mutableBitmap.toMumblingBitmap + val expected = Seq(0, 2, 30, 31, 32, 255, 256, 300, 522, 1024) + + assert(bitmap.cardinality() == expected.size) + expected.foreach(position => assert(bitmap.isSet(position))) + Seq(1, 33, 257, 521, 523, 1023).foreach(position => assert(!bitmap.isSet(position))) + } + + test("merge keeps a sparse base container sparse") { + val bitmap = merge(0 until 30, 30) + + assert(bitmap.cardinality() == 31) + assert(singleContainerDescriptor(bitmap) == 31) + } + + test("merge keeps a dense base container dense") { + val bitmap = merge(0 until 32, 32) + + assert(bitmap.cardinality() == 33) + assert(singleContainerDescriptor(bitmap) == 32) + } + + test("merge promotes a sparse base container to dense") { + val bitmap = merge(0 until 31, 31) + + assert(bitmap.cardinality() == 32) + assert(singleContainerDescriptor(bitmap) == 32) + } + + test("merge fast path 1 copies a source container without additions") { + val basePositions = 0 until 32 + val bitmap = merge(basePositions, 256) + val expected = MumblingBitmapWriter.serialize((basePositions :+ 256).toArray) + + assert(bufferBytes(bitmap).sameElements(expected)) + } + + test("merge fast path 2 emits an empty container gap") { + val bitmap = merge(Seq(1), 512) + val expected = MumblingBitmapWriter.serialize(Array(1, 512)) + + assert(bufferBytes(bitmap).sameElements(expected)) + } + + test("merge handles intermingled sparse, dense, promotion, and fast-path containers") { + def positions(container: Int, offsets: Seq[Int]): Seq[Int] = + offsets.map(offset => container * 256 + offset) + + // Container layout: + // 0, 5: sparse stays sparse + // 1, 4: fast path 1 copies untouched source containers + // 2, 7: sparse becomes dense + // 3, 6: dense stays dense + // 8, 10: fast path 2 emits empty gaps beyond the source containers + val sparseStaysSparse = Seq(0, 5) + val fastPath1 = Seq(1, 4) + val sparseBecomesDense = Seq(2, 7) + val denseStaysDense = Seq(3, 6) + val fastPath2 = Seq(8, 10) + + val basePositions = ( + sparseStaysSparse.flatMap(positions(_, 0 until 30)) ++ + positions(fastPath1.head, Seq(1, 7)) ++ + positions(fastPath1.last, 0 until 32) ++ + sparseBecomesDense.flatMap(positions(_, 0 until 31)) ++ + denseStaysDense.flatMap(positions(_, 0 until 32))).sorted + // New containers 9 and 11 leave fast-path-2 gaps after the last base container. + val additions = ( + sparseStaysSparse.map(_ * 256 + 30) ++ + sparseBecomesDense.map(_ * 256 + 31) ++ + denseStaysDense.map(_ * 256 + 32) ++ + Seq(9 * 256 + 9, 11 * 256 + 11)).sorted + val expectedPositions = (basePositions ++ additions).distinct.sorted + + val bitmap = merge(basePositions, additions: _*) + val expectedBytes = MumblingBitmapWriter.serialize(expectedPositions.toArray) + + assert(bitmap.cardinality() == expectedPositions.size) + expectedPositions.foreach(position => assert(bitmap.isSet(position))) + assert(bufferBytes(bitmap).sameElements(expectedBytes)) + fastPath2.foreach { container => + assert(!(0 until 256).exists(offset => bitmap.isSet(container * 256 + offset))) + } + } + + test("merge preserves and updates dense base containers") { + val basePositions = (0 until 64) ++ (256 until 288) ++ Seq(600) + val base = new MumblingBitmap( + ByteBuffer.wrap(MumblingBitmapWriter.serialize(basePositions.toArray))) + val mutableBitmap = base.toMutable() + val additions = Seq(10, 64, 601, 1024) + additions.foreach(position => mutableBitmap.add(position.toLong)) + + val bitmap = mutableBitmap.toMumblingBitmap + val expected = (basePositions ++ additions).toSet + assert(bitmap.cardinality() == expected.size) + expected.foreach(position => assert(bitmap.isSet(position))) + Seq(65, 300, 602, 1023).foreach(position => assert(!bitmap.isSet(position))) + } + + private val fuzzTestSeed = Random.nextLong() + + test(s"fuzz: random position sets round-trip through the Mumbling format " + + s"(seed: $fuzzTestSeed)") { + val rng = new Random(fuzzTestSeed) + for (iteration <- 0 until 50) { + val positions = (0 until rng.nextInt(2000)).map(_ => rng.nextInt(49999).toLong).toSet.toSeq + roundTrip(positions, s"iteration $iteration") + } + } + + test("reader honors a non-zero ByteBuffer position") { + val encoded = MumblingBitmapWriter.serialize(Array(42)) + val padded = ByteBuffer.allocate(encoded.length + 4) + padded.position(4) + padded.put(encoded) + padded.position(4) + + val bitmap = new MumblingBitmap(padded) + assert(bitmap.cardinality() == 1) + assert(!bitmap.isSet(41)) + assert(bitmap.isSet(42)) + assert(!bitmap.isSet(43)) + assert(bufferBytes(bitmap).sameElements(encoded)) + } + + test("reader decodes a mixed descriptor array with a PFOR exception") { + val sparsePositions = (0 until 15).filter(_ != 7).map(i => i * 256 + i) + val densePositions = (0 until 32).map(7 * 256 + _) + val positions = (sparsePositions ++ densePositions).sorted.toArray + val bitmap = new MumblingBitmap(ByteBuffer.wrap(MumblingBitmapWriter.serialize(positions))) + + assert(bitmap.cardinality() == positions.length) + positions.foreach(pos => assert(bitmap.isSet(pos))) + assert(!bitmap.isSet(7 * 256 - 1)) + assert(!bitmap.isSet(7 * 256 + 32)) + } + + test("reader rejects headers beyond the Mumbling limits") { + val excessiveContainers = hexBytes("01 00 00 00 01 20") + val excessiveCardinality = hexBytes("01 01 00 20 00 00") + + intercept[IllegalStateException](new MumblingBitmap(ByteBuffer.wrap(excessiveContainers))) + intercept[IllegalStateException](new MumblingBitmap(ByteBuffer.wrap(excessiveCardinality))) + } + + test("reader rejects descriptors that are neither sparse nor dense") { + val invalidDescriptor = hexBytes("01 00 00 00 01 00 00 00 40") + val bitmap = new MumblingBitmap(ByteBuffer.wrap(invalidDescriptor)) + + val error = intercept[IllegalStateException](bitmap.isSet(0)) + assert(error.getMessage.contains("Invalid descriptor")) + } + + test("merge accepts and canonicalizes dense descriptors with low bits set") { + val nonCanonicalDense = + hexBytes("01 20 00 00 01 00 00 00 21 FF FF FF FF") ++ Array.fill(28)(0.toByte) + val mutableBitmap = new MumblingBitmap(ByteBuffer.wrap(nonCanonicalDense)).toMutable() + mutableBitmap.add(32L) + + val bitmap = mutableBitmap.toMumblingBitmap + assert(bitmap.cardinality() == 33) + assert(bitmap.isSet(0)) + assert(bitmap.isSet(31)) + assert(bitmap.isSet(32)) + assert(bufferBytes(bitmap).slice(6, 9).sameElements(hexBytes("00 00 20"))) + } + + // ---- On-disk format vectors (Mumbling spec conformance) ---- + // Assert the exact serialized bytes, so a change that still round-trips but deviates from the + // spec layout (bit order, endianness, descriptor encoding) is caught. + + private def hexBytes(hex: String): Array[Byte] = + hex.split("\\s+").filter(_.nonEmpty).map(Integer.parseInt(_, 16).toByte) + + private def assertSerialized(positions: Array[Int], expected: Array[Byte], name: String): Unit = { + def show(bs: Array[Byte]) = bs.map(b => f"${b & 0xff}%02x").mkString(" ") + val actual = MumblingBitmapWriter.serialize(positions) + assert(actual.sameElements(expected), + s"$name:\n expected ${show(expected)}\n actual ${show(actual)}") + // Cross-check: the bytes decode back to the input positions. + val bitmap = new MumblingBitmap(ByteBuffer.wrap(actual)) + assert(bitmap.cardinality() == positions.length) + positions.foreach(position => assert(bitmap.isSet(position))) + } + + test("format: empty bitmap is a 6-byte header (version=1, cardinality=0, count=0)") { + assertSerialized(Array.empty[Int], hexBytes("01 00 00 00 00 00"), "empty") + } + + test("format: sparse container matches the spec layout") { + // Spec sparse example: descriptor 3 -> container 00 22 FF (positions 0, 34, 255). + // header ver=01 card=03 00 00 count=01 00 | PFOR of [3] = 00 00 03 | container 00 22 FF + assertSerialized(Array(0, 34, 255), + hexBytes("01 03 00 00 01 00 00 00 03 00 22 FF"), "sparse 0/34/255") + } + + test("format: dense container for 0..31 matches the spec layout") { + // Spec dense example: descriptor 0x20 -> FF FF FF FF 00...00 (positions 0-31). + val header = hexBytes("01 20 00 00 01 00") // cardinality = 32. + val descriptors = hexBytes("00 00 20") // PFOR of [32]. + val container = hexBytes("FF FF FF FF") ++ Array.fill(28)(0.toByte) + assertSerialized((0 until 32).toArray, header ++ descriptors ++ container, "dense 0..31") + } + + test("format: dense container for 0..32 sets the MSB of byte 4 (spec example)") { + // Spec: FF FF FF FF 80 ... 00 (positions 0-32; position 32 is the MSB of byte 4). + val header = hexBytes("01 21 00 00 01 00") // cardinality = 33. + val descriptors = hexBytes("00 00 20") + val container = hexBytes("FF FF FF FF 80") ++ Array.fill(27)(0.toByte) + assertSerialized((0 to 32).toArray, header ++ descriptors ++ container, "dense 0..32") + } + + test("format: dense container of even positions is 0xAA repeated (spec example)") { + // Spec: AA AA ... AA (even positions 0, 2, 4, ...). + val header = hexBytes("01 80 00 00 01 00") // cardinality = 128. + val descriptors = hexBytes("00 00 20") + val container = Array.fill(32)(0xAA.toByte) + assertSerialized((0 until 256 by 2).toArray, header ++ descriptors ++ container, "dense even") + } + + test("format: multi-container descriptor array is PFOR-encoded") { + // positions 5, 522 -> containers c0={5}, c1={} (empty), c2={522 & 0xff = 10} + // descriptors [1, 0, 1] PFOR-encode with b1=1 (01 00 00) + primary A0 (bits 1,0,1 MSB-first). + val header = hexBytes("01 02 00 00 03 00") // cardinality = 2, count = 3. + val descriptors = hexBytes("01 00 00 A0") + val containers = hexBytes("05 0A") + assertSerialized(Array(5, 522), header ++ descriptors ++ containers, "multi-container") + } +} diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/deletionvectors/RowIndexMarkingFiltersSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/deletionvectors/RowIndexMarkingFiltersSuite.scala index fad1b12d254..2d0da678213 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/deletionvectors/RowIndexMarkingFiltersSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/deletionvectors/RowIndexMarkingFiltersSuite.scala @@ -16,7 +16,7 @@ package org.apache.spark.sql.delta.deletionvectors -import org.apache.spark.sql.delta.RowIndexFilter +import org.apache.spark.sql.delta.{RowIndexFilter, RowIndexFilterProvider} import org.apache.spark.sql.delta.DeltaTestUtils.BOOLEAN_DOMAIN import org.apache.spark.sql.delta.actions.DeletionVectorDescriptor import org.apache.spark.sql.delta.actions.DeletionVectorDescriptor._ @@ -34,6 +34,23 @@ import org.apache.spark.util.Utils class RowIndexMarkingFiltersSuite extends QueryTest with SharedSparkSession { + for (positions <- Seq(Seq.empty[Int], Seq(0, 3, 9))) { + test(s"manifest provider survives serialization with positions $positions") { + val bitmap = ManifestBitmap.fromPositions(positions) + val provider = bitmap.toRowIndexFilterProvider(new Path("/unused")) + val restored = Utils.deserialize[RowIndexFilterProvider](Utils.serialize(provider)) + assert(getMarked(restored.retrieve(newHadoopConf), start = 0, end = 12) === positions) + } + } + + test("manifest provider uses an immutable bitmap snapshot") { + val mutableBitmap = ManifestBitmap.fromPositions(Seq(1, 3)).toMutable + val provider = mutableBitmap.toManifestBitmap + .toRowIndexFilterProvider(new Path("/unused")) + mutableBitmap.add(5) + assert(getMarked(provider.retrieve(newHadoopConf), start = 0, end = 8) === Seq(1, 3)) + } + test("empty deletion vector (drop filter)") { val rowIndexFilter = DropMarkedRowsFilter.createInstance( DeletionVectorDescriptor.EMPTY, diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/dataskipping/DataSkippingDeltaV1AMTTests.scala b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/dataskipping/DataSkippingDeltaV1AMTTests.scala new file mode 100644 index 00000000000..d8c66acee3f --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/dataskipping/DataSkippingDeltaV1AMTTests.scala @@ -0,0 +1,35 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// *********************************************************************************** +// * This file is automatically generated. Manual modification is not allowed. * +// * There is a unit test that should prevent merging a manual change. * +// * * +// * To make changes to the suites, modify the generator script config at * +// * SuiteGeneratorConfig.scala and run it. The generator can be run via the * +// * sbt command deltaSuiteGenerator / run. * +// * * +// * DO NOT TOUCH ANYTHING IN THIS FILE! * +// *********************************************************************************** + +// scalastyle:off line.size.limit +package org.apache.spark.sql.delta.generatedsuites.dataskipping + +import org.apache.spark.sql.delta._ +import org.apache.spark.sql.delta.coordinatedcommits._ +import org.apache.spark.sql.delta.stats._ + +class DataSkippingDeltaV1AMTSuite extends DataSkippingDeltaV1AMTTests diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/MergeIntoAnalysisExceptionTests.scala b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/MergeIntoAnalysisExceptionTests.scala new file mode 100644 index 00000000000..0e3ccdc3acd --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/MergeIntoAnalysisExceptionTests.scala @@ -0,0 +1,39 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// *********************************************************************************** +// * This file is automatically generated. Manual modification is not allowed. * +// * There is a unit test that should prevent merging a manual change. * +// * * +// * To make changes to the suites, modify the generator script config at * +// * SuiteGeneratorConfig.scala and run it. The generator can be run via the * +// * sbt command deltaSuiteGenerator / run. * +// * * +// * DO NOT TOUCH ANYTHING IN THIS FILE! * +// *********************************************************************************** + +// scalastyle:off line.size.limit +package org.apache.spark.sql.delta.generatedsuites.mergeamt + +import org.apache.spark.sql.delta._ +import org.apache.spark.sql.delta.amt._ +import org.apache.spark.sql.delta.rowid._ + +class MergeIntoAnalysisExceptionSQLNameBasedAMTSuite + extends MergeIntoAnalysisExceptionTests + with MergeIntoSQLMixin + with DeltaDMLTestUtilsNameBased + with MergeIntoAMTMixin diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/MergeIntoBasicTests.scala b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/MergeIntoBasicTests.scala new file mode 100644 index 00000000000..b9014858f8c --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/MergeIntoBasicTests.scala @@ -0,0 +1,39 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// *********************************************************************************** +// * This file is automatically generated. Manual modification is not allowed. * +// * There is a unit test that should prevent merging a manual change. * +// * * +// * To make changes to the suites, modify the generator script config at * +// * SuiteGeneratorConfig.scala and run it. The generator can be run via the * +// * sbt command deltaSuiteGenerator / run. * +// * * +// * DO NOT TOUCH ANYTHING IN THIS FILE! * +// *********************************************************************************** + +// scalastyle:off line.size.limit +package org.apache.spark.sql.delta.generatedsuites.mergeamt + +import org.apache.spark.sql.delta._ +import org.apache.spark.sql.delta.amt._ +import org.apache.spark.sql.delta.rowid._ + +class MergeIntoBasicSQLNameBasedAMTSuite + extends MergeIntoBasicTests + with MergeIntoSQLMixin + with DeltaDMLTestUtilsNameBased + with MergeIntoAMTMixin diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/MergeIntoExtendedSyntaxTests.scala b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/MergeIntoExtendedSyntaxTests.scala new file mode 100644 index 00000000000..9d90d8f277b --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/MergeIntoExtendedSyntaxTests.scala @@ -0,0 +1,39 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// *********************************************************************************** +// * This file is automatically generated. Manual modification is not allowed. * +// * There is a unit test that should prevent merging a manual change. * +// * * +// * To make changes to the suites, modify the generator script config at * +// * SuiteGeneratorConfig.scala and run it. The generator can be run via the * +// * sbt command deltaSuiteGenerator / run. * +// * * +// * DO NOT TOUCH ANYTHING IN THIS FILE! * +// *********************************************************************************** + +// scalastyle:off line.size.limit +package org.apache.spark.sql.delta.generatedsuites.mergeamt + +import org.apache.spark.sql.delta._ +import org.apache.spark.sql.delta.amt._ +import org.apache.spark.sql.delta.rowid._ + +class MergeIntoExtendedSyntaxSQLNameBasedAMTSuite + extends MergeIntoExtendedSyntaxTests + with MergeIntoSQLMixin + with DeltaDMLTestUtilsNameBased + with MergeIntoAMTMixin diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/MergeIntoNestedDataTests.scala b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/MergeIntoNestedDataTests.scala new file mode 100644 index 00000000000..29c52c31475 --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/MergeIntoNestedDataTests.scala @@ -0,0 +1,39 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// *********************************************************************************** +// * This file is automatically generated. Manual modification is not allowed. * +// * There is a unit test that should prevent merging a manual change. * +// * * +// * To make changes to the suites, modify the generator script config at * +// * SuiteGeneratorConfig.scala and run it. The generator can be run via the * +// * sbt command deltaSuiteGenerator / run. * +// * * +// * DO NOT TOUCH ANYTHING IN THIS FILE! * +// *********************************************************************************** + +// scalastyle:off line.size.limit +package org.apache.spark.sql.delta.generatedsuites.mergeamt + +import org.apache.spark.sql.delta._ +import org.apache.spark.sql.delta.amt._ +import org.apache.spark.sql.delta.rowid._ + +class MergeIntoNestedDataSQLNameBasedAMTSuite + extends MergeIntoNestedDataTests + with MergeIntoSQLMixin + with DeltaDMLTestUtilsNameBased + with MergeIntoAMTMixin diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/MergeIntoNestedStructEvolutionInsertTests.scala b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/MergeIntoNestedStructEvolutionInsertTests.scala new file mode 100644 index 00000000000..ccf6ded3453 --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/MergeIntoNestedStructEvolutionInsertTests.scala @@ -0,0 +1,39 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// *********************************************************************************** +// * This file is automatically generated. Manual modification is not allowed. * +// * There is a unit test that should prevent merging a manual change. * +// * * +// * To make changes to the suites, modify the generator script config at * +// * SuiteGeneratorConfig.scala and run it. The generator can be run via the * +// * sbt command deltaSuiteGenerator / run. * +// * * +// * DO NOT TOUCH ANYTHING IN THIS FILE! * +// *********************************************************************************** + +// scalastyle:off line.size.limit +package org.apache.spark.sql.delta.generatedsuites.mergeamt + +import org.apache.spark.sql.delta._ +import org.apache.spark.sql.delta.amt._ +import org.apache.spark.sql.delta.rowid._ + +class MergeIntoNestedStructEvolutionInsertSQLNameBasedAMTSuite + extends MergeIntoNestedStructEvolutionInsertTests + with MergeIntoSQLMixin + with DeltaDMLTestUtilsNameBased + with MergeIntoAMTMixin diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/MergeIntoNestedStructEvolutionUpdateOnlyTests.scala b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/MergeIntoNestedStructEvolutionUpdateOnlyTests.scala new file mode 100644 index 00000000000..984aaf6ed71 --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/MergeIntoNestedStructEvolutionUpdateOnlyTests.scala @@ -0,0 +1,39 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// *********************************************************************************** +// * This file is automatically generated. Manual modification is not allowed. * +// * There is a unit test that should prevent merging a manual change. * +// * * +// * To make changes to the suites, modify the generator script config at * +// * SuiteGeneratorConfig.scala and run it. The generator can be run via the * +// * sbt command deltaSuiteGenerator / run. * +// * * +// * DO NOT TOUCH ANYTHING IN THIS FILE! * +// *********************************************************************************** + +// scalastyle:off line.size.limit +package org.apache.spark.sql.delta.generatedsuites.mergeamt + +import org.apache.spark.sql.delta._ +import org.apache.spark.sql.delta.amt._ +import org.apache.spark.sql.delta.rowid._ + +class MergeIntoNestedStructEvolutionUpdateOnlySQLNameBasedAMTSuite + extends MergeIntoNestedStructEvolutionUpdateOnlyTests + with MergeIntoSQLMixin + with DeltaDMLTestUtilsNameBased + with MergeIntoAMTMixin diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/MergeIntoNestedStructInMapEvolutionTests.scala b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/MergeIntoNestedStructInMapEvolutionTests.scala new file mode 100644 index 00000000000..b7650fcde6a --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/MergeIntoNestedStructInMapEvolutionTests.scala @@ -0,0 +1,39 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// *********************************************************************************** +// * This file is automatically generated. Manual modification is not allowed. * +// * There is a unit test that should prevent merging a manual change. * +// * * +// * To make changes to the suites, modify the generator script config at * +// * SuiteGeneratorConfig.scala and run it. The generator can be run via the * +// * sbt command deltaSuiteGenerator / run. * +// * * +// * DO NOT TOUCH ANYTHING IN THIS FILE! * +// *********************************************************************************** + +// scalastyle:off line.size.limit +package org.apache.spark.sql.delta.generatedsuites.mergeamt + +import org.apache.spark.sql.delta._ +import org.apache.spark.sql.delta.amt._ +import org.apache.spark.sql.delta.rowid._ + +class MergeIntoNestedStructInMapEvolutionSQLNameBasedAMTSuite + extends MergeIntoNestedStructInMapEvolutionTests + with MergeIntoSQLMixin + with DeltaDMLTestUtilsNameBased + with MergeIntoAMTMixin diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/MergeIntoNotMatchedBySourceSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/MergeIntoNotMatchedBySourceSuite.scala new file mode 100644 index 00000000000..cd11a051a58 --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/MergeIntoNotMatchedBySourceSuite.scala @@ -0,0 +1,39 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// *********************************************************************************** +// * This file is automatically generated. Manual modification is not allowed. * +// * There is a unit test that should prevent merging a manual change. * +// * * +// * To make changes to the suites, modify the generator script config at * +// * SuiteGeneratorConfig.scala and run it. The generator can be run via the * +// * sbt command deltaSuiteGenerator / run. * +// * * +// * DO NOT TOUCH ANYTHING IN THIS FILE! * +// *********************************************************************************** + +// scalastyle:off line.size.limit +package org.apache.spark.sql.delta.generatedsuites.mergeamt + +import org.apache.spark.sql.delta._ +import org.apache.spark.sql.delta.amt._ +import org.apache.spark.sql.delta.rowid._ + +class MergeIntoNotMatchedBySourceSQLNameBasedAMTSuite + extends MergeIntoNotMatchedBySourceSuite + with MergeIntoSQLMixin + with DeltaDMLTestUtilsNameBased + with MergeIntoAMTMixin diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/MergeIntoNullTypeTests.scala b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/MergeIntoNullTypeTests.scala new file mode 100644 index 00000000000..8412245f9b9 --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/MergeIntoNullTypeTests.scala @@ -0,0 +1,39 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// *********************************************************************************** +// * This file is automatically generated. Manual modification is not allowed. * +// * There is a unit test that should prevent merging a manual change. * +// * * +// * To make changes to the suites, modify the generator script config at * +// * SuiteGeneratorConfig.scala and run it. The generator can be run via the * +// * sbt command deltaSuiteGenerator / run. * +// * * +// * DO NOT TOUCH ANYTHING IN THIS FILE! * +// *********************************************************************************** + +// scalastyle:off line.size.limit +package org.apache.spark.sql.delta.generatedsuites.mergeamt + +import org.apache.spark.sql.delta._ +import org.apache.spark.sql.delta.amt._ +import org.apache.spark.sql.delta.rowid._ + +class MergeIntoNullTypeSQLNameBasedAMTSuite + extends MergeIntoNullTypeTests + with MergeIntoSQLMixin + with DeltaDMLTestUtilsNameBased + with MergeIntoAMTMixin diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/MergeIntoSQLNondeterministicOrderTests.scala b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/MergeIntoSQLNondeterministicOrderTests.scala new file mode 100644 index 00000000000..3ca15adf7f6 --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/MergeIntoSQLNondeterministicOrderTests.scala @@ -0,0 +1,39 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// *********************************************************************************** +// * This file is automatically generated. Manual modification is not allowed. * +// * There is a unit test that should prevent merging a manual change. * +// * * +// * To make changes to the suites, modify the generator script config at * +// * SuiteGeneratorConfig.scala and run it. The generator can be run via the * +// * sbt command deltaSuiteGenerator / run. * +// * * +// * DO NOT TOUCH ANYTHING IN THIS FILE! * +// *********************************************************************************** + +// scalastyle:off line.size.limit +package org.apache.spark.sql.delta.generatedsuites.mergeamt + +import org.apache.spark.sql.delta._ +import org.apache.spark.sql.delta.amt._ +import org.apache.spark.sql.delta.rowid._ + +class MergeIntoSQLNondeterministicOrderSQLNameBasedAMTSuite + extends MergeIntoSQLNondeterministicOrderTests + with MergeIntoSQLMixin + with DeltaDMLTestUtilsNameBased + with MergeIntoAMTMixin diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/MergeIntoSQLTests.scala b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/MergeIntoSQLTests.scala new file mode 100644 index 00000000000..f2bfad67c22 --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/MergeIntoSQLTests.scala @@ -0,0 +1,39 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// *********************************************************************************** +// * This file is automatically generated. Manual modification is not allowed. * +// * There is a unit test that should prevent merging a manual change. * +// * * +// * To make changes to the suites, modify the generator script config at * +// * SuiteGeneratorConfig.scala and run it. The generator can be run via the * +// * sbt command deltaSuiteGenerator / run. * +// * * +// * DO NOT TOUCH ANYTHING IN THIS FILE! * +// *********************************************************************************** + +// scalastyle:off line.size.limit +package org.apache.spark.sql.delta.generatedsuites.mergeamt + +import org.apache.spark.sql.delta._ +import org.apache.spark.sql.delta.amt._ +import org.apache.spark.sql.delta.rowid._ + +class MergeIntoSQLSQLNameBasedAMTSuite + extends MergeIntoSQLTests + with MergeIntoSQLMixin + with DeltaDMLTestUtilsNameBased + with MergeIntoAMTMixin diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/MergeIntoSchemaEvoStoreAssignmentPolicyTests.scala b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/MergeIntoSchemaEvoStoreAssignmentPolicyTests.scala new file mode 100644 index 00000000000..6424831c67a --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/MergeIntoSchemaEvoStoreAssignmentPolicyTests.scala @@ -0,0 +1,39 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// *********************************************************************************** +// * This file is automatically generated. Manual modification is not allowed. * +// * There is a unit test that should prevent merging a manual change. * +// * * +// * To make changes to the suites, modify the generator script config at * +// * SuiteGeneratorConfig.scala and run it. The generator can be run via the * +// * sbt command deltaSuiteGenerator / run. * +// * * +// * DO NOT TOUCH ANYTHING IN THIS FILE! * +// *********************************************************************************** + +// scalastyle:off line.size.limit +package org.apache.spark.sql.delta.generatedsuites.mergeamt + +import org.apache.spark.sql.delta._ +import org.apache.spark.sql.delta.amt._ +import org.apache.spark.sql.delta.rowid._ + +class MergeIntoSchemaEvoStoreAssignmentPolicySQLNameBasedAMTSuite + extends MergeIntoSchemaEvoStoreAssignmentPolicyTests + with MergeIntoSQLMixin + with DeltaDMLTestUtilsNameBased + with MergeIntoAMTMixin diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/MergeIntoSchemaEvolutionBaseExistingColumnTests.scala b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/MergeIntoSchemaEvolutionBaseExistingColumnTests.scala new file mode 100644 index 00000000000..bb3142a7def --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/MergeIntoSchemaEvolutionBaseExistingColumnTests.scala @@ -0,0 +1,39 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// *********************************************************************************** +// * This file is automatically generated. Manual modification is not allowed. * +// * There is a unit test that should prevent merging a manual change. * +// * * +// * To make changes to the suites, modify the generator script config at * +// * SuiteGeneratorConfig.scala and run it. The generator can be run via the * +// * sbt command deltaSuiteGenerator / run. * +// * * +// * DO NOT TOUCH ANYTHING IN THIS FILE! * +// *********************************************************************************** + +// scalastyle:off line.size.limit +package org.apache.spark.sql.delta.generatedsuites.mergeamt + +import org.apache.spark.sql.delta._ +import org.apache.spark.sql.delta.amt._ +import org.apache.spark.sql.delta.rowid._ + +class MergeIntoSchemaEvolutionBaseExistingColumnSQLNameBasedAMTSuite + extends MergeIntoSchemaEvolutionBaseExistingColumnTests + with MergeIntoSQLMixin + with DeltaDMLTestUtilsNameBased + with MergeIntoAMTMixin diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/MergeIntoSchemaEvolutionBaseNewColumnTests.scala b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/MergeIntoSchemaEvolutionBaseNewColumnTests.scala new file mode 100644 index 00000000000..e3ad9b424b2 --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/MergeIntoSchemaEvolutionBaseNewColumnTests.scala @@ -0,0 +1,39 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// *********************************************************************************** +// * This file is automatically generated. Manual modification is not allowed. * +// * There is a unit test that should prevent merging a manual change. * +// * * +// * To make changes to the suites, modify the generator script config at * +// * SuiteGeneratorConfig.scala and run it. The generator can be run via the * +// * sbt command deltaSuiteGenerator / run. * +// * * +// * DO NOT TOUCH ANYTHING IN THIS FILE! * +// *********************************************************************************** + +// scalastyle:off line.size.limit +package org.apache.spark.sql.delta.generatedsuites.mergeamt + +import org.apache.spark.sql.delta._ +import org.apache.spark.sql.delta.amt._ +import org.apache.spark.sql.delta.rowid._ + +class MergeIntoSchemaEvolutionBaseNewColumnSQLNameBasedAMTSuite + extends MergeIntoSchemaEvolutionBaseNewColumnTests + with MergeIntoSQLMixin + with DeltaDMLTestUtilsNameBased + with MergeIntoAMTMixin diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/MergeIntoSchemaEvolutionCoreTests.scala b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/MergeIntoSchemaEvolutionCoreTests.scala new file mode 100644 index 00000000000..5a685937ad5 --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/MergeIntoSchemaEvolutionCoreTests.scala @@ -0,0 +1,39 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// *********************************************************************************** +// * This file is automatically generated. Manual modification is not allowed. * +// * There is a unit test that should prevent merging a manual change. * +// * * +// * To make changes to the suites, modify the generator script config at * +// * SuiteGeneratorConfig.scala and run it. The generator can be run via the * +// * sbt command deltaSuiteGenerator / run. * +// * * +// * DO NOT TOUCH ANYTHING IN THIS FILE! * +// *********************************************************************************** + +// scalastyle:off line.size.limit +package org.apache.spark.sql.delta.generatedsuites.mergeamt + +import org.apache.spark.sql.delta._ +import org.apache.spark.sql.delta.amt._ +import org.apache.spark.sql.delta.rowid._ + +class MergeIntoSchemaEvolutionCoreSQLNameBasedAMTSuite + extends MergeIntoSchemaEvolutionCoreTests + with MergeIntoSQLMixin + with DeltaDMLTestUtilsNameBased + with MergeIntoAMTMixin diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/MergeIntoSchemaEvolutionNotMatchedBySourceTests.scala b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/MergeIntoSchemaEvolutionNotMatchedBySourceTests.scala new file mode 100644 index 00000000000..e27f8beb61b --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/MergeIntoSchemaEvolutionNotMatchedBySourceTests.scala @@ -0,0 +1,39 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// *********************************************************************************** +// * This file is automatically generated. Manual modification is not allowed. * +// * There is a unit test that should prevent merging a manual change. * +// * * +// * To make changes to the suites, modify the generator script config at * +// * SuiteGeneratorConfig.scala and run it. The generator can be run via the * +// * sbt command deltaSuiteGenerator / run. * +// * * +// * DO NOT TOUCH ANYTHING IN THIS FILE! * +// *********************************************************************************** + +// scalastyle:off line.size.limit +package org.apache.spark.sql.delta.generatedsuites.mergeamt + +import org.apache.spark.sql.delta._ +import org.apache.spark.sql.delta.amt._ +import org.apache.spark.sql.delta.rowid._ + +class MergeIntoSchemaEvolutionNotMatchedBySourceSQLNameBasedAMTSuite + extends MergeIntoSchemaEvolutionNotMatchedBySourceTests + with MergeIntoSQLMixin + with DeltaDMLTestUtilsNameBased + with MergeIntoAMTMixin diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/MergeIntoTempViewsTests.scala b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/MergeIntoTempViewsTests.scala new file mode 100644 index 00000000000..ac2c58913d5 --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/MergeIntoTempViewsTests.scala @@ -0,0 +1,39 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// *********************************************************************************** +// * This file is automatically generated. Manual modification is not allowed. * +// * There is a unit test that should prevent merging a manual change. * +// * * +// * To make changes to the suites, modify the generator script config at * +// * SuiteGeneratorConfig.scala and run it. The generator can be run via the * +// * sbt command deltaSuiteGenerator / run. * +// * * +// * DO NOT TOUCH ANYTHING IN THIS FILE! * +// *********************************************************************************** + +// scalastyle:off line.size.limit +package org.apache.spark.sql.delta.generatedsuites.mergeamt + +import org.apache.spark.sql.delta._ +import org.apache.spark.sql.delta.amt._ +import org.apache.spark.sql.delta.rowid._ + +class MergeIntoTempViewsSQLNameBasedAMTSuite + extends MergeIntoTempViewsTests + with MergeIntoSQLMixin + with DeltaDMLTestUtilsNameBased + with MergeIntoAMTMixin diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/MergeIntoUnlimitedMergeClausesTests.scala b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/MergeIntoUnlimitedMergeClausesTests.scala new file mode 100644 index 00000000000..cf7b051c970 --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/MergeIntoUnlimitedMergeClausesTests.scala @@ -0,0 +1,39 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// *********************************************************************************** +// * This file is automatically generated. Manual modification is not allowed. * +// * There is a unit test that should prevent merging a manual change. * +// * * +// * To make changes to the suites, modify the generator script config at * +// * SuiteGeneratorConfig.scala and run it. The generator can be run via the * +// * sbt command deltaSuiteGenerator / run. * +// * * +// * DO NOT TOUCH ANYTHING IN THIS FILE! * +// *********************************************************************************** + +// scalastyle:off line.size.limit +package org.apache.spark.sql.delta.generatedsuites.mergeamt + +import org.apache.spark.sql.delta._ +import org.apache.spark.sql.delta.amt._ +import org.apache.spark.sql.delta.rowid._ + +class MergeIntoUnlimitedMergeClausesSQLNameBasedAMTSuite + extends MergeIntoUnlimitedMergeClausesTests + with MergeIntoSQLMixin + with DeltaDMLTestUtilsNameBased + with MergeIntoAMTMixin diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/RowTrackingMergeCommonTests.scala b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/RowTrackingMergeCommonTests.scala new file mode 100644 index 00000000000..9c1c04a2e63 --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/mergeamt/RowTrackingMergeCommonTests.scala @@ -0,0 +1,44 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// *********************************************************************************** +// * This file is automatically generated. Manual modification is not allowed. * +// * There is a unit test that should prevent merging a manual change. * +// * * +// * To make changes to the suites, modify the generator script config at * +// * SuiteGeneratorConfig.scala and run it. The generator can be run via the * +// * sbt command deltaSuiteGenerator / run. * +// * * +// * DO NOT TOUCH ANYTHING IN THIS FILE! * +// *********************************************************************************** + +// scalastyle:off line.size.limit +package org.apache.spark.sql.delta.generatedsuites.mergeamt + +import org.apache.spark.sql.delta._ +import org.apache.spark.sql.delta.amt._ +import org.apache.spark.sql.delta.rowid._ + +class RowTrackingMergeCommonNameBasedAMTRowTrackingMergeDVSuite + extends RowTrackingMergeCommonTests + with DeltaDMLTestUtilsNameBased + with MergeIntoAMTMixin + with RowTrackingMergeDVMixin + +class RowTrackingMergeCommonNameBasedAMTSuite + extends RowTrackingMergeCommonTests + with DeltaDMLTestUtilsNameBased + with MergeIntoAMTMixin diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/hooks/metrics/UpdateMetricsHookSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/hooks/metrics/UpdateMetricsHookSuite.scala index 976aee385fc..de1f1456bd6 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/hooks/metrics/UpdateMetricsHookSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/hooks/metrics/UpdateMetricsHookSuite.scala @@ -219,6 +219,7 @@ class UpdateMetricsHookSuite extends QueryTest operationParameters = Map.empty, job = None, notebook = None, clusterId = None, readVersion = None, isolationLevel = None, isBlindAppend = Some(true), + dataChange = None, operationMetrics = Some(Map("numOutputRows" -> "50")), userMetadata = None, tags = None, engineInfo = None, txnId = None, lastManifestCommit = None) diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/optimize/DeltaReorgSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/optimize/DeltaReorgSuite.scala index 13ed0c2b107..7cab8db6f4e 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/optimize/DeltaReorgSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/optimize/DeltaReorgSuite.scala @@ -26,7 +26,7 @@ import io.delta.tables.DeltaTable import org.apache.hadoop.fs.{FileStatus, Path} import org.apache.parquet.hadoop.Footer -import org.apache.spark.sql.QueryTest +import org.apache.spark.sql.{QueryTest, SparkSession} import org.apache.spark.sql.functions.col import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.util.SerializableConfiguration @@ -104,6 +104,30 @@ class DeltaReorgSuite extends QueryTest } } + test("Purge DVs when tasks cannot see a SparkSession") { + // On a cluster, executors have no active or default SparkSession. In local mode the tasks + // fall back to the JVM-wide default session, so clear it to reproduce executor behavior. + val targetDf = spark.range(0, 100, 1, numPartitions = 5).toDF() + withTempDeltaTable(targetDf) { (_, log) => + val path = log.dataPath.toString + sql(s"DELETE FROM delta.`$path` WHERE id IN (0, 99)") + + val defaultSession = SparkSession.getDefaultSession + SparkSession.clearDefaultSession() + try { + executePurge(path) + } finally { + defaultSession.foreach(SparkSession.setDefaultSession) + } + val (addFiles, _) = getFileActionsInLastVersion(log) + assert(addFiles.nonEmpty) + assert(addFiles.forall(_.deletionVector === null)) + checkAnswer( + sql(s"SELECT * FROM delta.`$path`"), + (1 to 98).toDF()) + } + } + test("Purge a non-DV table is a noop") { val targetDf = spark.range(0, 100, 1, numPartitions = 5).toDF() withTempDeltaTable(targetDf, enableDVs = false) { (_, log) => @@ -303,13 +327,20 @@ class DeltaReorgSuite extends QueryTest test("reorg on a catalog managed table should fail") { withCatalogManagedTable() { tableName => + spark.sql(s"INSERT INTO $tableName VALUES (1)") + val snapshotBefore = getSnapshot(tableName) + checkError( intercept[DeltaUnsupportedOperationException] { spark.sql(s"REORG TABLE $tableName APPLY (PURGE)") }, "DELTA_UNSUPPORTED_CATALOG_MANAGED_TABLE_OPERATION", - parameters = Map("operation" -> "OPTIMIZE") + parameters = Map("operation" -> "DATA_REORGANIZATION") ) + + val snapshotAfter = getSnapshot(tableName) + assert(snapshotAfter.version === snapshotBefore.version) + assert(snapshotAfter.allFiles.collect().toSet === snapshotBefore.allFiles.collect().toSet) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/optimize/OptimizeCompactionSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/optimize/OptimizeCompactionSuite.scala index 5dc5ecb5490..fb398dfe0c9 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/optimize/OptimizeCompactionSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/optimize/OptimizeCompactionSuite.scala @@ -642,13 +642,20 @@ trait OptimizeCompactionSuiteBase extends QueryTest test("optimize on a catalog managed table should fail") { withCatalogManagedTable() { tableName => + spark.sql(s"INSERT INTO $tableName VALUES (1)") + val snapshotBefore = getSnapshot(tableName) + checkError( intercept[DeltaUnsupportedOperationException] { spark.sql(s"OPTIMIZE $tableName") }, "DELTA_UNSUPPORTED_CATALOG_MANAGED_TABLE_OPERATION", - parameters = Map("operation" -> "OPTIMIZE") + parameters = Map("operation" -> "DATA_REORGANIZATION") ) + + val snapshotAfter = getSnapshot(tableName) + assert(snapshotAfter.version === snapshotBefore.version) + assert(snapshotAfter.allFiles.collect().toSet === snapshotBefore.allFiles.collect().toSet) } } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/perf/OptimizeGeneratedColumnSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/perf/OptimizeGeneratedColumnSuite.scala index 95e2774b0f9..0e7ffd56acc 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/perf/OptimizeGeneratedColumnSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/perf/OptimizeGeneratedColumnSuite.scala @@ -20,19 +20,24 @@ import java.sql.Timestamp import java.util.Locale import java.util.concurrent.{CountDownLatch, TimeUnit} +import scala.collection.JavaConverters._ import scala.util.matching.Regex // scalastyle:off import.ordering.noEmptyLine +import com.databricks.spark.util.Log4jUsageLogger import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.sources.DeltaSQLConf.{DELTA_COLLECT_STATS, GENERATED_COLUMN_PARTITION_FILTER_OPTIMIZATION_ENABLED} +import org.apache.spark.sql.delta.sources.DeltaSQLConf.GENERATED_COLUMN_PARTITION_FILTER_INFERENCE_MODE import org.apache.spark.sql.delta.stats.PrepareDeltaScanBase import org.apache.spark.sql.delta.test.DeltaTestImplicits._ +import org.apache.spark.sql.delta.util.JsonUtils -import org.apache.spark.sql.{DataFrame, Row} +import org.apache.spark.sql.{AnalysisException, DataFrame, Row} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.execution.{FileSourceScanLike, QueryExecution} +import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.TimestampType import org.apache.spark.util.ThreadUtils import org.apache.spark.util.Utils @@ -53,6 +58,107 @@ class OptimizeGeneratedColumnSuite extends GeneratedColumnTest { df.write.format("delta").mode("append").save(path) } + private case class LoggedInferenceCandidate( + descriptor: String, + sourceDataType: String, + partitionDataTypes: Seq[String], + literalDataType: Option[String], + operator: String, + hasKnownRisk: Boolean, + count: Int = 1) + + + private def inferenceConfs( + mode: String, + timeZone: Option[String] = None): Seq[(String, String)] = { + Seq( + GENERATED_COLUMN_PARTITION_FILTER_INFERENCE_MODE.key -> mode) ++ + timeZone.map(SQLConf.SESSION_LOCAL_TIMEZONE.key -> _) + } + + private def trackPartitionFilterInference( + body: => Unit): Seq[LoggedInferenceCandidate] = { + Log4jUsageLogger.track(body) + .filter(_.tags.getOrElse("opType", "") == + "delta.generatedColumns.partitionFilterInference") + .flatMap { record => + JsonUtils.mapper.readTree(record.blob).path("candidateSignatures").elements().asScala + .map { candidate => + LoggedInferenceCandidate( + descriptor = candidate.path("descriptor").asText(), + sourceDataType = candidate.path("sourceDataType").asText(), + partitionDataTypes = candidate.path("partitionDataTypes").elements().asScala + .map(_.asText()).toSeq, + literalDataType = Option(candidate.get("literalDataType")).map(_.asText()), + operator = candidate.path("operator").asText(), + hasKnownRisk = candidate.path("hasKnownRisk").asBoolean(), + count = candidate.path("count").asInt()) + } + } + .distinct + } + + private def getFiltersAndInferenceCandidates( + relation: DataFrame): (Seq[Expression], Seq[LoggedInferenceCandidate]) = { + var filters = Seq.empty[Expression] + val candidates = trackPartitionFilterInference { + filters = getPushedPartitionFilters(relation.queryExecution) + } + filters -> candidates + } + + private def assertInferenceCandidates( + actual: Seq[LoggedInferenceCandidate], + expected: LoggedInferenceCandidate*): Unit = { + assert(actual.size === expected.size) + assert(actual.toSet === expected.toSet) + } + + private def comparisonCandidate( + descriptor: String, + sourceDataType: String, + partitionDataType: String, + operator: String, + hasKnownRisk: Boolean, + count: Int = 1): LoggedInferenceCandidate = { + LoggedInferenceCandidate( + descriptor, + sourceDataType, + Seq(partitionDataType), + Some(sourceDataType), + operator, + hasKnownRisk, + count) + } + + private def hasFilterFor(filters: Seq[Expression], column: String): Boolean = { + filters.exists(_.references.exists(_.name == column)) + } + + private val unsafeTruncPartitionColumns = Seq("date", "dateCopy") + + private def withUnsafeStringTruncTable(testCode: String => Unit): Unit = { + withTableName("unsafe_string_trunc_partition_filter") { table => + createTable( + table, + None, + "eventDateStr STRING, date DATE, dateCopy DATE, prefix STRING", + Map( + "date" -> "trunc(eventDateStr, 'quarter')", + "dateCopy" -> "trunc(eventDateStr, 'quarter')", + "prefix" -> "substring(eventDateStr, 1, 4)"), + unsafeTruncPartitionColumns :+ "prefix") + withSQLConf( + DELTA_COLLECT_STATS.key -> "false", + SQLConf.ANSI_ENABLED.key -> "false") { + sql(s"""INSERT INTO $table (eventDateStr) VALUES + |('10000-01-01'), ('2022-04-01'), ('not-a-date')""".stripMargin) + } + testCode(table) + } + } + + /** * Verify we can recognize an `OptimizablePartitionExpression` and generate corresponding * partition filters correctly. @@ -1176,6 +1282,244 @@ class OptimizeGeneratedColumnSuite extends GeneratedColumnTest { ) ) + test("ASSERT retains string TRUNC inference for valid equality literals") { + withUnsafeStringTruncTable { table => + withSQLConf(inferenceConfs("ASSERT"): _*) { + Seq( + "eventDateStr = '2022-04-01'", + "'2022-04-01' = eventDateStr").foreach { predicate => + val filters = getPushedPartitionFilters( + sql(s"SELECT eventDateStr FROM $table WHERE $predicate").queryExecution) + (unsafeTruncPartitionColumns :+ "prefix").foreach { column => + assert(hasFilterFor(filters, column), + s"Expected inferred equality filter for $predicate: ${filters.map(_.sql)}") + } + } + + checkAnswer( + sql(s"SELECT eventDateStr FROM $table WHERE eventDateStr = '2022-04-01'"), + Row("2022-04-01")) + } + } + } + + test("ASSERT does not over-prune non-zero-padded string equality literals") { + withUnsafeStringTruncTable { table => + val value = "2026-4-1" + sql(s"INSERT INTO $table (eventDateStr) VALUES ('$value')") + + Seq(false, true).foreach { ansiEnabled => + val confs = inferenceConfs("ASSERT") :+ + (SQLConf.ANSI_ENABLED.key -> ansiEnabled.toString) + withSQLConf(confs: _*) { + val predicate = s"eventDateStr = '$value'" + val relation = sql(s"SELECT eventDateStr FROM $table WHERE $predicate") + val filters = getPushedPartitionFilters(relation.queryExecution) + unsafeTruncPartitionColumns.foreach { column => + assert(hasFilterFor(filters, column), + s"Expected inferred equality filter for $predicate: ${filters.map(_.sql)}") + } + checkAnswer(relation, Row(value)) + } + } + } + } + + test("ASSERT suppresses unsafe string TRUNC inference") { + withUnsafeStringTruncTable { table => + val rangePredicates = Seq( + "eventDateStr < '2022-04-01'", + "eventDateStr <= '2022-04-01'", + "eventDateStr > '2022-04-01'", + "eventDateStr >= '2022-04-01'", + "'2022-04-01' > eventDateStr", + "'2022-04-01' >= eventDateStr", + "'2022-04-01' < eventDateStr", + "'2022-04-01' <= eventDateStr") + + withSQLConf(inferenceConfs("ASSERT"): _*) { + rangePredicates.foreach { predicate => + val filters = getPushedPartitionFilters( + sql(s"SELECT eventDateStr FROM $table WHERE $predicate").queryExecution) + unsafeTruncPartitionColumns.foreach { column => + assert(!hasFilterFor(filters, column), + s"Unexpected inferred filter for $predicate: ${filters.map(_.sql)}") + } + assert(hasFilterFor(filters, "prefix"), + s"Expected safe substring filter for $predicate: ${filters.map(_.sql)}") + } + + val nullRelation = sql(s"SELECT * FROM $table WHERE eventDateStr IS NULL") + val (nullFilters, nullCandidates) = getFiltersAndInferenceCandidates(nullRelation) + (unsafeTruncPartitionColumns :+ "prefix").foreach { column => + assert(hasFilterFor(nullFilters, column)) + } + assertInferenceCandidates( + nullCandidates, + LoggedInferenceCandidate( + "SubstringPartitionExpr", "string", Seq("string"), None, "IsNull", false), + LoggedInferenceCandidate( + "TruncDatePartitionExpr", "string", Seq("date"), None, "IsNull", false, 2)) + + checkAnswer( + sql(s"SELECT eventDateStr FROM $table WHERE eventDateStr < '2022-04-01'"), + Row("10000-01-01")) + + withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") { + val relation = sql(s"SELECT * FROM $table WHERE eventDateStr = 'not-a-date'") + val (filters, candidates) = getFiltersAndInferenceCandidates(relation) + unsafeTruncPartitionColumns.foreach(column => assert(!hasFilterFor(filters, column))) + assert(hasFilterFor(filters, "prefix")) + assertInferenceCandidates( + candidates, + comparisonCandidate( + "SubstringPartitionExpr", "string", "string", "EqualTo", false), + comparisonCandidate( + "TruncDatePartitionExpr", "string", "date", "EqualTo", true, 2)) + checkAnswer(relation.select("eventDateStr"), Row("not-a-date")) + } + } + } + } + + test("typed string predicates only infer filters after string coercion") { + withUnsafeStringTruncTable { table => + withSQLConf(inferenceConfs("ASSERT"): _*) { + Seq( + "eventDateStr < 20", + "eventDateStr < DATE '2022-04-01'", + "eventDateStr < TIMESTAMP '2022-04-01 00:00:00'").foreach { predicate => + val relation = sql(s"SELECT eventDateStr FROM $table WHERE $predicate") + val (filters, candidates) = getFiltersAndInferenceCandidates(relation) + assert(filters.isEmpty, + s"Unexpected inferred filter for $predicate: ${filters.map(_.sql)}") + assert(candidates.isEmpty) + } + + withSQLConf( + SQLConf.ANSI_ENABLED.key -> "false", + SQLConf.LEGACY_CAST_DATETIME_TO_STRING.key -> "true") { + Seq( + "eventDateStr < DATE '2022-04-01'", + "eventDateStr < TIMESTAMP '2022-04-01 00:00:00'").foreach { predicate => + val relation = sql(s"SELECT eventDateStr FROM $table WHERE $predicate") + val (filters, candidates) = getFiltersAndInferenceCandidates(relation) + unsafeTruncPartitionColumns.foreach { column => + assert(!hasFilterFor(filters, column), + s"Unexpected inferred filter for $predicate: ${filters.map(_.sql)}") + } + assert(hasFilterFor(filters, "prefix"), + s"Expected safe substring filter for $predicate: ${filters.map(_.sql)}") + assertInferenceCandidates( + candidates, + comparisonCandidate( + "SubstringPartitionExpr", "string", "string", "LessThan", false), + comparisonCandidate( + "TruncDatePartitionExpr", "string", "date", "LessThan", true, 2)) + } + } + } + } + } + + test("partition filter inference flags behaves correctly") { + withUnsafeStringTruncTable { table => + Seq("OFF", "LOG_ONLY", "ASSERT").foreach { mode => + withSQLConf(inferenceConfs(mode): _*) { + val relation = sql(s"SELECT * FROM $table WHERE eventDateStr < '2022-04-01'") + val (filters, candidates) = getFiltersAndInferenceCandidates(relation) + + if (mode == "ASSERT") { + unsafeTruncPartitionColumns.foreach(column => assert(!hasFilterFor(filters, column))) + } else { + unsafeTruncPartitionColumns.foreach(column => assert(hasFilterFor(filters, column))) + } + assert(hasFilterFor(filters, "prefix")) + + if (mode == "OFF") { + assert(candidates.isEmpty) + } else { + assertInferenceCandidates( + candidates, + comparisonCandidate( + "SubstringPartitionExpr", "string", "string", "LessThan", false), + comparisonCandidate( + "TruncDatePartitionExpr", "string", "date", "LessThan", true, 2)) + } + } + } + } + } + + test("ASSERT retains safe temporal partition filter inference") { + Seq( + ("eventDate DATE", "date DATE", "trunc(eventDate, 'month')", + "eventDate", "date", "date", "TruncDatePartitionExpr"), + ("eventDate DATE", "year INT", "year(eventDate)", + "eventDate", "year", "int", "YearPartitionExpr"), + ("eventDate DATE", "formatted STRING", "date_format(eventDate, 'yyyy-MM')", + "eventDate", "formatted", "string", "DateFormatPartitionExpr") + ).foreach { + case (sourceSchema, partitionSchema, generationExpression, sourceColumn, + partitionColumn, partitionDataType, descriptor) => + withTableName("temporal_trunc_partition_filter") { table => + createTable( + table, + None, + s"$sourceSchema, $partitionSchema", + Map(partitionColumn -> generationExpression), + Seq(partitionColumn)) + withSQLConf(inferenceConfs("ASSERT"): _*) { + val relation = sql(s"SELECT * FROM $table WHERE $sourceColumn < '2022-04-01'") + val (filters, candidates) = getFiltersAndInferenceCandidates(relation) + assert(hasFilterFor(filters, partitionColumn)) + assertInferenceCandidates( + candidates, + comparisonCandidate( + descriptor, "date", partitionDataType, "LessThan", false)) + } + } + } + } + + test("unsupported conversions do not produce partition filter inference") { + Seq( + ("value STRING", "intPart INT", "CAST(value AS INT)", "value < '20'"), + ("value STRING", "datePart DATE", "CAST(value AS DATE)", + "value < '2022-04-01'"), + ("value STRING", "timestampPart TIMESTAMP", "CAST(value AS TIMESTAMP)", + "value < '2022-04-01'"), + ("value STRING", "timestampPart TIMESTAMP", "DATE_TRUNC('YEAR', value)", + "value < '2022-04-01'"), + ("value INT", "doublePart DOUBLE", "CAST(value AS DOUBLE)", "value < 20"), + ("value BIGINT", "doublePart DOUBLE", "CAST(value AS DOUBLE)", + "value < CAST(20 AS BIGINT)"), + ("value DECIMAL(20, 0)", "doublePart DOUBLE", "CAST(value AS DOUBLE)", + "value < CAST(20 AS DECIMAL(20, 0))")) + .zipWithIndex + .foreach { case ((sourceSchema, partitionSchema, generationExpression, predicate), index) => + withTableName(s"unsupported_conversion_partition_filter_$index") { table => + val partitionColumn = partitionSchema.takeWhile(_ != ' ') + createTable( + table, + None, + s"$sourceSchema, $partitionSchema", + Map(partitionColumn -> generationExpression), + Seq(partitionColumn)) + + val metadata = DeltaLog.forTable(spark, TableIdentifier(table)).update().metadata + assert(!metadata.optimizablePartitionExpressions.contains("value")) + withSQLConf(inferenceConfs("LOG_ONLY"): _*) { + val relation = sql(s"SELECT * FROM $table WHERE $predicate") + val (filters, candidates) = getFiltersAndInferenceCandidates(relation) + assert(filters.isEmpty) + assert(candidates.isEmpty) + } + } + } + } + + test("five digits year in a year month day partition column") { withTempDir { tempDir => withTableName("optimizable_partition_expression") { table => diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/rowid/RowTrackingMergeSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/rowid/RowTrackingMergeSuite.scala index 30cae47b553..490f64797b9 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/rowid/RowTrackingMergeSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/rowid/RowTrackingMergeSuite.scala @@ -43,7 +43,7 @@ trait RowTrackingMergeSuiteBase extends RowIdTestUtils .write.format("delta").saveAsTable(tableName) } - override def beforeAll(): Unit = { + override protected def beforeAll(): Unit = { super.beforeAll() spark.conf.set(DeltaConfigs.ROW_TRACKING_ENABLED.defaultTablePropertyKey, value = "true") diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/rowid/RowTrackingRemovalConcurrencySuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/rowid/RowTrackingRemovalConcurrencySuite.scala index 8f2b50a551b..1d1161f2872 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/rowid/RowTrackingRemovalConcurrencySuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/rowid/RowTrackingRemovalConcurrencySuite.scala @@ -502,10 +502,16 @@ class RowTrackingRemovalConcurrencySuite } assert(e.getErrorClass === "DELTA_ROW_TRACKING_ILLEGAL_PROPERTY_COMBINATION") } else { - // Backfill fails if run together backfill. - assertThrows[IllegalStateException] { + // Backfill fails while the table is suspended during unbackfill. + val e = intercept[DeltaIllegalStateException] { backfillTransaction(deltaLog) } + checkError( + e, + "DELTA_ROW_TRACKING_ILLEGAL_PROPERTY_COMBINATION", + parameters = Map( + "property1" -> DeltaConfigs.ROW_TRACKING_ENABLED.key, + "property2" -> DeltaConfigs.ROW_TRACKING_SUSPENDED.key)) } // Commit batch 2. diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/rowid/RowTrackingRemovalSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/rowid/RowTrackingRemovalSuite.scala index c7102d365a6..085f1846607 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/rowid/RowTrackingRemovalSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/rowid/RowTrackingRemovalSuite.scala @@ -255,13 +255,19 @@ class RowTrackingRemovalSuite extends RowTrackingRemovalSuiteBase { |${DeltaConfigs.ROW_TRACKING_ENABLED.key} = 'true' |)""".stripMargin) - assertThrows[IllegalStateException] { + val suspensionError = intercept[DeltaIllegalStateException] { sql( s"""ALTER TABLE delta.`${dir.getAbsolutePath}` |SET TBLPROPERTIES( |${DeltaConfigs.ROW_TRACKING_SUSPENDED.key} = 'true' |)""".stripMargin) } + checkError( + suspensionError, + "DELTA_ROW_TRACKING_ILLEGAL_PROPERTY_COMBINATION", + parameters = Map( + "property1" -> DeltaConfigs.ROW_TRACKING_ENABLED.key, + "property2" -> DeltaConfigs.ROW_TRACKING_SUSPENDED.key)) sql( s"""ALTER TABLE delta.`${dir.getAbsolutePath}` @@ -271,13 +277,19 @@ class RowTrackingRemovalSuite extends RowTrackingRemovalSuiteBase { |)""".stripMargin) - assertThrows[IllegalStateException] { + val reenableError = intercept[DeltaIllegalStateException] { sql( s"""ALTER TABLE delta.`${dir.getAbsolutePath}` |SET TBLPROPERTIES( |${DeltaConfigs.ROW_TRACKING_ENABLED.key} = 'true' |)""".stripMargin) } + checkError( + reenableError, + "DELTA_ROW_TRACKING_ILLEGAL_PROPERTY_COMBINATION", + parameters = Map( + "property1" -> DeltaConfigs.ROW_TRACKING_ENABLED.key, + "property2" -> DeltaConfigs.ROW_TRACKING_SUSPENDED.key)) } } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/rowtracking/RowTrackingReadWriteSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/rowtracking/RowTrackingReadWriteSuite.scala index 80fc7556251..28a1dd41ce1 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/rowtracking/RowTrackingReadWriteSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/rowtracking/RowTrackingReadWriteSuite.scala @@ -18,7 +18,7 @@ package org.apache.spark.sql.delta.rowtracking import scala.collection.JavaConverters._ -import org.apache.spark.sql.delta.{DeltaConfigs, DeltaLog, RowCommitVersion, RowId} +import org.apache.spark.sql.delta.{DeltaConfigs, DeltaLog, DeltaTableProvider, RowCommitVersion, RowId} import org.apache.spark.sql.delta.actions.AddFile import org.apache.spark.sql.delta.rowid.RowIdTestUtils @@ -31,7 +31,8 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{LongType, StructType} class RowTrackingReadWriteSuite extends RowIdTestUtils - with ParquetTest { + with ParquetTest + with DeltaTableProvider { private val testTableName = "target" @@ -59,7 +60,7 @@ class RowTrackingReadWriteSuite extends RowIdTestUtils withRowTrackingEnabled(enabled = true) { val numRows = 5L spark.range(numRows).toDF(testDataColumnName) - .write.format("delta").saveAsTable(testTableName) + .write.format(writeFormat).saveAsTable(testTableName) try { // Confirm that the materialized columns are not present in the Parquet file(s). @@ -279,7 +280,7 @@ class RowTrackingReadWriteSuite extends RowIdTestUtils .toDF(testDataColumnName) .withMaterializedRowIdColumn(columnName, lit(1L)) .write - .format("delta") + .format(writeFormat) .mode("append") .saveAsTable(testTableName) } @@ -297,7 +298,7 @@ class RowTrackingReadWriteSuite extends RowIdTestUtils .toDF(testDataColumnName) .withMaterializedRowCommitVersionColumn(columnName, lit(2L)) .write - .format("delta") + .format(writeFormat) .mode("append") .saveAsTable(testTableName) } @@ -371,7 +372,7 @@ class RowTrackingReadWriteSuite extends RowIdTestUtils withRowTrackingEnabled(enabled = true) { // Create the table if it does not exist already. df.limit(n = 0) - .write.format("delta").mode("append").saveAsTable(testTableName) + .write.format(writeFormat).mode("append").saveAsTable(testTableName) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(testTableName)) val materializedRowIdColumnName = @@ -385,7 +386,7 @@ class RowTrackingReadWriteSuite extends RowIdTestUtils materializedRowCommitVersionName, rowCommitVersionColumn) .write .mode("append") - .format("delta") + .format(writeFormat) .saveAsTable(testTableName) } } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/schema/CheckConstraintsSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/schema/CheckConstraintsSuite.scala index 415b2b4d276..006a4e00657 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/schema/CheckConstraintsSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/schema/CheckConstraintsSuite.scala @@ -18,7 +18,7 @@ package org.apache.spark.sql.delta.schema import scala.collection.JavaConverters._ -import org.apache.spark.sql.delta.{AllowedUserProvidedExpressions, DeltaConfigs, DeltaLog} +import org.apache.spark.sql.delta.{AllowedUserProvidedExpressions, DeltaConfigs, DeltaLog, DeltaTableProvider, DeltaTestUtils} import org.apache.spark.sql.delta.constraints.CharVarcharConstraint import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.sources.DeltaSQLConf.ValidateCheckConstraintsMode @@ -36,7 +36,8 @@ import org.apache.spark.sql.types.{ArrayType, BooleanType, IntegerType, MapType, class CheckConstraintsSuite extends QueryTest with SharedSparkSession with DeltaSQLCommandTest - with DeltaSQLTestUtils { + with DeltaSQLTestUtils + with DeltaTableProvider { import testImplicits._ @@ -49,7 +50,7 @@ class CheckConstraintsSuite extends QueryTest Seq( (1, "a"), (2, "b"), (3, "c"), (4, "d"), (5, "e"), (6, "f") - ).toDF("num", "text").write.format("delta").saveAsTable("checkConstraintsTest") + ).toDF("num", "text").write.format(writeFormat).saveAsTable("checkConstraintsTest") thunk("checkConstraintsTest") } } @@ -79,13 +80,10 @@ class CheckConstraintsSuite extends QueryTest test("Checking incorrect constraints added through table property in CREATE TABLE errors out") { val tableName = "test_tbl" withTable(tableName) { - sql( - s""" - |CREATE TABLE $tableName ( - |id INT, - |event_date DATE - |) USING DELTA - |TBLPROPERTIES('delta.constraints.ch' = 'event_date < 2025-06-12');""".stripMargin) + sql(createTableSQL( + tableName, + "id INT, event_date DATE", + props = Map("delta.constraints.ch" -> "event_date < 2025-06-12"))) val e = intercept[AnalysisException] { sql(s"INSERT INTO $tableName VALUES(1, '2025-06-11')") @@ -102,14 +100,10 @@ class CheckConstraintsSuite extends QueryTest withTable(tableName) { checkError( exception = intercept[AnalysisException] { - sql( - s""" - |CREATE TABLE $tableName ( - |id INT, - |value STRING - |) USING DELTA - |TBLPROPERTIES('delta.constraints.invalid' = 'non_existent_column > 0') - |""".stripMargin) + sql(createTableSQL( + tableName, + "id INT, value STRING", + props = Map("delta.constraints.invalid" -> "non_existent_column > 0"))) }, "DELTA_INVALID_CHECK_CONSTRAINT_REFERENCES", parameters = Map("colName" -> "`non_existent_column`") @@ -125,14 +119,10 @@ class CheckConstraintsSuite extends QueryTest withTable(tableName) { checkError( exception = intercept[AnalysisException] { - sql( - s""" - |CREATE TABLE $tableName ( - |id INT, - |value STRING - |) USING DELTA - |TBLPROPERTIES('delta.constraints.nonbool' = 'id + 1') - |""".stripMargin) + sql(createTableSQL( + tableName, + "id INT, value STRING", + props = Map("delta.constraints.nonbool" -> "id + 1"))) }, "DELTA_NON_BOOLEAN_CHECK_CONSTRAINT", parameters = Map( @@ -149,14 +139,10 @@ class CheckConstraintsSuite extends QueryTest ValidateCheckConstraintsMode.ASSERT.toString) { val tableName = "test_create_valid_constraint" withTable(tableName) { - sql( - s""" - |CREATE TABLE $tableName ( - |id INT, - |value STRING - |) USING DELTA - |TBLPROPERTIES('delta.constraints.positive_id' = 'id > 0') - |""".stripMargin) + sql(createTableSQL( + tableName, + "id INT, value STRING", + props = Map("delta.constraints.positive_id" -> "id > 0"))) } } } @@ -425,7 +411,7 @@ class CheckConstraintsSuite extends QueryTest StructField("m", MapType(IntegerType, IntegerType, valueContainsNull = true)), StructField("arr", ArrayType(IntegerType, containsNull = true))))))) spark.createDataFrame(rows.toList.asJava, schema) - .write.format("delta").saveAsTable("checkConstraintsTest") + .write.format(writeFormat).saveAsTable("checkConstraintsTest") // Constraints checking for a null value should work. sql("ALTER TABLE checkConstraintsTest ADD CONSTRAINT textNull CHECK (text IS NULL)") @@ -453,7 +439,7 @@ class CheckConstraintsSuite extends QueryTest newRows.foreach { r => e = intercept[InvariantViolationException] { spark.createDataFrame(List(r).asJava, schema) - .write.format("delta").mode("append").saveAsTable("checkConstraintsTest") + .write.format(writeFormat).mode("append").saveAsTable("checkConstraintsTest") } errorContains(e.getMessage, "CHECK constraint arr0 (nested.arr[0] < 100) violated by row") @@ -464,10 +450,10 @@ class CheckConstraintsSuite extends QueryTest sql("ALTER TABLE checkConstraintsTest DROP CONSTRAINT arr0") newRows.foreach { r => spark.createDataFrame(List(r).asJava, schema) - .write.format("delta").mode("append").saveAsTable("checkConstraintsTest") + .write.format(writeFormat).mode("append").saveAsTable("checkConstraintsTest") } checkAnswer( - spark.read.format("delta").table("checkConstraintsTest").select("id"), + spark.read.format(writeFormat).table("checkConstraintsTest").select("id"), (0 to 12).toDF("id")) } } @@ -492,7 +478,7 @@ class CheckConstraintsSuite extends QueryTest StructField("m", MapType(IntegerType, IntegerType, valueContainsNull = false)), StructField("arr", ArrayType(IntegerType, containsNull = false))))))) spark.createDataFrame(rows.toList.asJava, schema) - .write.format("delta").saveAsTable("checkConstraintsTest") + .write.format(writeFormat).saveAsTable("checkConstraintsTest") sql("ALTER TABLE checkConstraintsTest ADD CONSTRAINT arrLen CHECK (SIZE(nested.arr) = 3)") sql("ALTER TABLE checkConstraintsTest ADD CONSTRAINT mapIntegrity " + "CHECK (nested.m[id] = id)") @@ -510,7 +496,7 @@ class CheckConstraintsSuite extends QueryTest // TODO: https://github.com/delta-io/delta/issues/831 test("SET NOT NULL constraint fails") { withTable("my_table") { - sql("CREATE TABLE my_table (id INT) USING DELTA;") + sql(createTableSQL("my_table", "id INT")) sql("INSERT INTO my_table VALUES (1);") val e = intercept[AnalysisException] { sql("ALTER TABLE my_table CHANGE COLUMN id SET NOT NULL;") @@ -521,7 +507,7 @@ class CheckConstraintsSuite extends QueryTest testQuietly("ending semi-colons no longer makes ADD, DROP constraint commands fail") { withTable("my_table") { - sql("CREATE TABLE my_table (birthday DATE) USING DELTA;") + sql(createTableSQL("my_table", "birthday DATE")) sql("INSERT INTO my_table VALUES ('2021-11-11');") sql("ALTER TABLE my_table ADD CONSTRAINT aaa CHECK (birthday > '1900-01-01')") @@ -538,14 +524,10 @@ class CheckConstraintsSuite extends QueryTest ValidateCheckConstraintsMode.ASSERT.toString, SQLConf.READ_SIDE_CHAR_PADDING.key -> "true") { withTable("charVarcharConstraintTest") { - sql( - """CREATE TABLE charVarcharConstraintTest ( - | id INT, - | name VARCHAR(50), - | code CHAR(10) - |) USING DELTA - |TBLPROPERTIES('delta.constraints.positive_id' = 'id > 0') - |""".stripMargin) + sql(createTableSQL( + "charVarcharConstraintTest", + "id INT, name VARCHAR(50), code CHAR(10)", + props = Map("delta.constraints.positive_id" -> "id > 0"))) sql("INSERT INTO charVarcharConstraintTest VALUES (1, 'test', 'ABC')") checkAnswer( sql("SELECT id, name, code FROM charVarcharConstraintTest"), @@ -556,7 +538,7 @@ class CheckConstraintsSuite extends QueryTest test("constraint induced by varchar") { withTable("table") { - sql("CREATE TABLE table (id INT, value VARCHAR(12)) USING DELTA") + sql(createTableSQL("table", "id INT, value VARCHAR(12)")) sql("INSERT INTO table VALUES (1, 'short string')") val exception = intercept[DeltaInvariantViolationException] { sql("INSERT INTO table VALUES (2, 'a very long string')") @@ -576,8 +558,8 @@ class CheckConstraintsSuite extends QueryTest withSQLConf( DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.defaultTablePropertyKey -> false.toString) { withTable("table") { - sql("CREATE TABLE table (a INT, b INT) USING DELTA " + - "TBLPROPERTIES ('delta.feature.checkConstraints' = 'supported')") + sql(createTableSQL("table", "a INT, b INT", + props = Map("delta.feature.checkConstraints" -> "supported"))) sql("ALTER TABLE table ADD CONSTRAINT c1 CHECK (a > 0)") sql("ALTER TABLE table ADD CONSTRAINT c2 CHECK (b > 0)") @@ -622,8 +604,8 @@ class CheckConstraintsSuite extends QueryTest ValidateCheckConstraintsMode.ASSERT.toString) { val testTable = "tbl" withTable(testTable) { - sql(s"CREATE TABLE $testTable (id STRING, value BOOLEAN) USING DELTA " + - "TBLPROPERTIES ('delta.feature.checkConstraints' = 'supported')") + sql(createTableSQL(testTable, "id STRING, value BOOLEAN", + props = Map("delta.feature.checkConstraints" -> "supported"))) sql(s"ALTER TABLE $testTable ADD CONSTRAINT c1 CHECK (value == $expression(id, 'A'))") sql(s"INSERT INTO $testTable VALUES ('ABA', true), ('DEF', false)") } @@ -636,8 +618,8 @@ class CheckConstraintsSuite extends QueryTest ValidateCheckConstraintsMode.ASSERT.toString) { val testTable = "like_any_all_test" withTable(testTable) { - sql(s"CREATE TABLE $testTable (id INT, name STRING, code STRING) USING DELTA " + - "TBLPROPERTIES ('delta.feature.checkConstraints' = 'supported')") + sql(createTableSQL(testTable, "id INT, name STRING, code STRING", + props = Map("delta.feature.checkConstraints" -> "supported"))) sql(s"ALTER TABLE $testTable ADD CONSTRAINT c_like_any " + "CHECK (name LIKE ANY ('%test%', '%prod%'))") sql(s"ALTER TABLE $testTable ADD CONSTRAINT c_not_like_any " + @@ -654,13 +636,47 @@ class CheckConstraintsSuite extends QueryTest } } + test("check constraints with timestamp + interval (TimestampAddInterval) expression") { + assume( + DeltaTestUtils.sparkVersionBucket(spark) == "4.2+", + "TimestampAddInterval is only allowlisted in Spark 4.2") + withSQLConf(DeltaSQLConf.VALIDATE_CHECK_CONSTRAINTS.key -> + ValidateCheckConstraintsMode.ASSERT.toString) { + val testTable = "time_add_test" + withTable(testTable) { + sql(createTableSQL(testTable, "id INT, event_ts TIMESTAMP", + props = Map("delta.feature.checkConstraints" -> "supported"))) + sql(s"ALTER TABLE $testTable ADD CONSTRAINT c_time_add " + + "CHECK (event_ts > CAST('2020-01-01' AS TIMESTAMP) + INTERVAL 1 DAY)") + sql(s"INSERT INTO $testTable VALUES (1, '2025-06-15 10:00:00')") + checkAnswer( + sql(s"SELECT id FROM $testTable"), + Seq(Row(1))) + } + } + } + + test("CREATE TABLE with a NULLIF check constraint succeeds") { + withSQLConf(DeltaSQLConf.VALIDATE_CHECK_CONSTRAINTS.key -> + ValidateCheckConstraintsMode.ASSERT.toString) { + val tableName = "test_create_nullif_constraint" + withTable(tableName) { + sql(createTableSQL( + tableName, + "id INT, value STRING", + props = Map("delta.constraints.nullif_value" -> "NULLIF(value, value) IS NULL"))) + sql(s"INSERT INTO $tableName VALUES (1, '')") + } + } + } + test("check constraints with array_size and array_compact expressions") { withSQLConf(DeltaSQLConf.VALIDATE_CHECK_CONSTRAINTS.key -> ValidateCheckConstraintsMode.ASSERT.toString) { val testTable = "array_funcs_test" withTable(testTable) { - sql(s"CREATE TABLE $testTable (id INT, tags ARRAY) USING DELTA " + - "TBLPROPERTIES ('delta.feature.checkConstraints' = 'supported')") + sql(createTableSQL(testTable, "id INT, tags ARRAY", + props = Map("delta.feature.checkConstraints" -> "supported"))) sql(s"ALTER TABLE $testTable ADD CONSTRAINT c_array_size " + "CHECK (array_size(tags) > 0)") sql(s"ALTER TABLE $testTable ADD CONSTRAINT c_array_compact " + @@ -678,8 +694,8 @@ class CheckConstraintsSuite extends QueryTest ValidateCheckConstraintsMode.ASSERT.toString) { val testTable = "array_modify_funcs_test" withTable(testTable) { - sql(s"CREATE TABLE $testTable (id INT, tags ARRAY) USING DELTA " + - "TBLPROPERTIES ('delta.feature.checkConstraints' = 'supported')") + sql(createTableSQL(testTable, "id INT, tags ARRAY", + props = Map("delta.feature.checkConstraints" -> "supported"))) sql(s"ALTER TABLE $testTable ADD CONSTRAINT c_array_append " + "CHECK (array_size(array_append(tags, 'x')) > 1)") sql(s"ALTER TABLE $testTable ADD CONSTRAINT c_array_prepend " + @@ -699,8 +715,8 @@ class CheckConstraintsSuite extends QueryTest ValidateCheckConstraintsMode.ASSERT.toString) { val testTable = "tbl" withTable(testTable) { - sql(s"CREATE TABLE $testTable (id INT) USING DELTA " + - "TBLPROPERTIES ('delta.feature.checkConstraints' = 'supported')") + sql(createTableSQL(testTable, "id INT", + props = Map("delta.feature.checkConstraints" -> "supported"))) checkError( exception = intercept[AnalysisException] { sql(s"ALTER TABLE $testTable ADD CONSTRAINT c1 " + @@ -719,8 +735,8 @@ class CheckConstraintsSuite extends QueryTest val testTable = "check_external_udf_test" withTable(testTable) { withUserDefinedFunction("external_udf" -> true) { - sql(s"CREATE TABLE $testTable (id INT, value INT) USING DELTA " + - "TBLPROPERTIES ('delta.feature.checkConstraints' = 'supported')") + sql(createTableSQL(testTable, "id INT, value INT", + props = Map("delta.feature.checkConstraints" -> "supported"))) spark.udf.register("external_udf", (x: Int) => x > 0) diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/skipping/clustering/ClusteredTableDDLSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/skipping/clustering/ClusteredTableDDLSuite.scala index 0bfd7b98909..f636375af5c 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/skipping/clustering/ClusteredTableDDLSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/skipping/clustering/ClusteredTableDDLSuite.scala @@ -42,12 +42,12 @@ trait ClusteredTableCreateOrReplaceDDLSuiteBase extends QueryTest with SharedSparkSession with ClusteredTableTestUtils { - override def beforeAll(): Unit = { + override protected def beforeAll(): Unit = { super.beforeAll() spark.conf.set(DeltaSQLConf.DELTA_UPDATE_CATALOG_ENABLED.key, "true") } - override def afterAll(): Unit = { + override protected def afterAll(): Unit = { // Reset UpdateCatalog's thread pool to ensure it is re-initialized in the next test suite. // This is necessary because the [[SparkThreadLocalForwardingThreadPoolExecutor]] // retains a reference to the SparkContext. Without resetting, the new test suite would diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/skipping/clustering/ReplacePartitionedByWithClusterBySuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/skipping/clustering/ReplacePartitionedByWithClusterBySuite.scala new file mode 100644 index 00000000000..6113ff27460 --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/skipping/clustering/ReplacePartitionedByWithClusterBySuite.scala @@ -0,0 +1,312 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.skipping.clustering + +import org.apache.spark.sql.delta.skipping.ClusteredTableTestUtils +import org.apache.spark.sql.delta.skipping.clustering.temp.{AlterTableReplacePartitionedByWithClusterBy, ClusterBySpec} +import org.apache.spark.sql.delta.{DeltaAnalysisException, DeltaLog} +import org.apache.spark.sql.delta.actions.AddFile +import org.apache.spark.sql.delta.commands.ReplacePartitionedByUtils +import org.apache.spark.sql.delta.sources.DeltaSQLConf +import org.apache.spark.sql.delta.stats.DataSkippingDeltaTestsUtils +import org.apache.spark.sql.delta.test.DeltaSQLCommandTest +import org.apache.spark.sql.delta.util.JsonUtils + +import org.apache.spark.sql.{QueryTest, Row} +import org.apache.spark.sql.catalyst.analysis.UnresolvedTable +import org.apache.spark.sql.types.{DateType, IntegerType, StringType, StructField, TimestampType} + +class ReplacePartitionedByWithClusterBySuite + extends QueryTest + with ClusteredTableTestUtils + with DeltaSQLCommandTest + with DataSkippingDeltaTestsUtils { + + private val opName = "REPLACE PARTITIONED BY WITH CLUSTER BY" + + private def createPartitionedTable(path: String, partitionColumns: String*): Unit = { + spark.range(0, 100, 1, 4) + .selectExpr("id", "cast(id % 5 as string) as p", "cast(id % 3 as int) as q", "id * 2 as v") + .write.format("delta").partitionBy(partitionColumns: _*).save(path) + } + + private def readTable(path: String) = spark.read.format("delta").load(path) + + private def statsOf(file: AddFile): Map[String, Any] = + JsonUtils.fromJson[Map[String, Any]](file.stats) + + test("parser") { + val parser = spark.sessionState.sqlParser + val none = parser.parsePlan("ALTER TABLE t REPLACE PARTITIONED BY WITH CLUSTER BY NONE") + assert(none === AlterTableReplacePartitionedByWithClusterBy( + UnresolvedTable(Seq("t"), "ALTER TABLE ... REPLACE PARTITIONED BY WITH CLUSTER BY"), None)) + val cols = parser.parsePlan( + "ALTER TABLE a.t REPLACE PARTITIONED BY WITH CLUSTER BY (c1, s.c2)") + assert(cols === AlterTableReplacePartitionedByWithClusterBy( + UnresolvedTable(Seq("a", "t"), "ALTER TABLE ... REPLACE PARTITIONED BY WITH CLUSTER BY"), + Some(ClusterBySpec(Seq(Seq("c1"), Seq("s", "c2")))))) + // `with` is not reserved. + parser.parsePlan("VACUUM with") + } + + test("CLUSTER BY NONE drops the partitioning without rewriting data files") { + withTempDir { dir => + val path = dir.getCanonicalPath + createPartitionedTable(path, "p", "q") + val deltaLog = DeltaLog.forTable(spark, path) + val before = deltaLog.update() + val expected = readTable(path).collect() + + sql(s"ALTER TABLE delta.`$path` REPLACE PARTITIONED BY WITH CLUSTER BY NONE") + + val after = deltaLog.update() + assert(after.version === before.version + 1) + assert(after.metadata.partitionColumns.isEmpty) + assert(after.schema === before.schema) + // No new table features, the table is a plain unpartitioned table. + assert(after.protocol === before.protocol) + assert(!ClusteredTableUtils.isSupported(after.protocol)) + assert(after.allFiles.collect().map(_.path).toSet === + before.allFiles.collect().map(_.path).toSet) + assert(after.allFiles.collect().forall(_.partitionValues.isEmpty)) + checkAnswer(readTable(path), expected) + + val lastCommit = deltaLog.history.getHistory(Some(1)).head + assert(lastCommit.operation === opName) + assert(lastCommit.operationParameters("oldPartitioningColumns") === "p,q") + assert(lastCommit.operationParameters("newClusteringColumns") === "") + + val changes = deltaLog.getChanges(after.version).next()._2 + assert(changes.collect { case a: AddFile => a }.forall(!_.dataChange)) + + // Time travel to the partitioned version still works. + checkAnswer( + spark.read.format("delta").option("versionAsOf", before.version).load(path), expected) + } + } + + test("statistics are synthesized for the former partition columns") { + withTempDir { dir => + val path = dir.getCanonicalPath + createPartitionedTable(path, "p", "q") + sql(s"ALTER TABLE delta.`$path` REPLACE PARTITIONED BY WITH CLUSTER BY NONE") + val deltaLog = DeltaLog.forTable(spark, path) + deltaLog.update().allFiles.collect().foreach { f => + val stats = statsOf(f) + val p = stats("minValues").asInstanceOf[Map[String, Any]]("p") + assert(stats("maxValues").asInstanceOf[Map[String, Any]]("p") === p) + assert(stats("nullCount").asInstanceOf[Map[String, Any]]("p") === 0) + assert(stats("minValues").asInstanceOf[Map[String, Any]].contains("q")) + assert(stats("minValues").asInstanceOf[Map[String, Any]].contains("id")) + } + // Each former partition value is in its own set of files, which can be skipped. + val numFiles = deltaLog.update().numOfFiles + val numFilesForP3 = filesRead(spark, deltaLog, "p = '3'", checkEmptyUnusedFilters = false) + assert(numFilesForP3 > 0 && numFilesForP3 < numFiles) + checkAnswer( + readTable(path).where("p = '3'").selectExpr("count(*)"), Row(20L)) + val numFilesForQ1 = filesRead(spark, deltaLog, "q = 1", checkEmptyUnusedFilters = false) + assert(numFilesForQ1 > 0 && numFilesForQ1 < numFiles) + } + } + + test("writes, DML and OPTIMIZE after dropping the partitioning") { + withTempDir { dir => + val path = dir.getCanonicalPath + createPartitionedTable(path, "p") + sql(s"ALTER TABLE delta.`$path` REPLACE PARTITIONED BY WITH CLUSTER BY NONE") + val deltaLog = DeltaLog.forTable(spark, path) + val filesBefore = deltaLog.update().allFiles.collect().map(_.path).toSet + + spark.range(100, 110).selectExpr("id", "'9' as p", "cast(id % 3 as int) as q", "id * 2 as v") + .write.format("delta").mode("append").save(path) + val newFiles = deltaLog.update().allFiles.collect().filterNot(f => filesBefore(f.path)) + assert(newFiles.nonEmpty) + // New files are written at the root of the table. + assert(newFiles.forall(f => !f.path.contains("/") && f.partitionValues.isEmpty)) + + sql(s"UPDATE delta.`$path` SET v = -1 WHERE p = '1'") + sql(s"DELETE FROM delta.`$path` WHERE p = '2'") + sql(s"""MERGE INTO delta.`$path` t USING (SELECT 0L AS id, 'x' AS p) s ON t.id = s.id + |WHEN MATCHED THEN UPDATE SET t.p = s.p""".stripMargin) + checkAnswer(readTable(path).where("p = '1'").selectExpr("count(*)", "max(v)"), + Row(20L, -1L)) + checkAnswer(readTable(path).where("p = '2'").selectExpr("count(*)"), Row(0L)) + checkAnswer(readTable(path).where("id = 0").select("p"), Row("x")) + checkAnswer(readTable(path).selectExpr("count(*)"), Row(90L)) + + sql(s"OPTIMIZE delta.`$path`") + checkAnswer(readTable(path).selectExpr("count(*)"), Row(90L)) + assert(deltaLog.update().metadata.partitionColumns.isEmpty) + } + } + + test("CLUSTER BY columns converts a partitioned table into a clustered table") { + withTempDir { dir => + val path = dir.getCanonicalPath + createPartitionedTable(path, "p") + val expected = readTable(path).collect() + sql(s"ALTER TABLE delta.`$path` REPLACE PARTITIONED BY WITH CLUSTER BY (p, id)") + val deltaLog = DeltaLog.forTable(spark, path) + val snapshot = deltaLog.update() + assert(snapshot.metadata.partitionColumns.isEmpty) + assert(ClusteredTableUtils.isSupported(snapshot.protocol)) + verifyClusteringColumnsInDomainMetadata(snapshot, Seq("p", "id")) + checkAnswer(readTable(path), expected) + val lastCommit = deltaLog.history.getHistory(Some(1)).head + assert(lastCommit.operation === opName) + assert(lastCommit.operationParameters("newClusteringColumns") === "p,id") + + // The table can be clustered and further altered like any clustered table. + sql(s"OPTIMIZE delta.`$path`") + checkAnswer(readTable(path), expected) + sql(s"ALTER TABLE delta.`$path` CLUSTER BY (id)") + verifyClusteringColumnsInDomainMetadata(deltaLog.update(), Seq("id")) + } + } + + test("works with catalog tables") { + withTable("tbl") { + sql("CREATE TABLE tbl (id BIGINT, p STRING) USING delta PARTITIONED BY (p)") + sql("INSERT INTO tbl VALUES (1, 'a'), (2, 'b'), (3, null)") + sql("ALTER TABLE tbl REPLACE PARTITIONED BY WITH CLUSTER BY NONE") + checkAnswer(sql("SELECT * FROM tbl"), Seq(Row(1L, "a"), Row(2L, "b"), Row(3L, null))) + checkAnswer(sql("SELECT id FROM tbl WHERE p IS NULL"), Row(3L)) + val detail = sql("DESCRIBE DETAIL tbl").select("partitionColumns").head() + assert(detail.getSeq[String](0).isEmpty) + } + } + + test("works with column mapping") { + withTempDir { dir => + val path = dir.getCanonicalPath + sql(s"""CREATE TABLE delta.`$path` (id BIGINT, p STRING) USING delta PARTITIONED BY (p) + |TBLPROPERTIES ('delta.columnMapping.mode' = 'name')""".stripMargin) + sql(s"INSERT INTO delta.`$path` VALUES (1, 'a'), (2, 'b')") + sql(s"ALTER TABLE delta.`$path` RENAME COLUMN p TO p2") + sql(s"ALTER TABLE delta.`$path` REPLACE PARTITIONED BY WITH CLUSTER BY NONE") + checkAnswer(readTable(path), Seq(Row(1L, "a"), Row(2L, "b"))) + checkAnswer(readTable(path).where("p2 = 'b'").select("id"), Row(2L)) + val deltaLog = DeltaLog.forTable(spark, path) + assert(filesRead(spark, deltaLog, "p2 = 'b'", checkEmptyUnusedFilters = false) === 1) + } + } + + test("works with deletion vectors and row tracking") { + withTempDir { dir => + val path = dir.getCanonicalPath + withSQLConf( + "spark.databricks.delta.properties.defaults.enableDeletionVectors" -> "true", + "spark.databricks.delta.properties.defaults.enableRowTracking" -> "true") { + createPartitionedTable(path, "p") + } + sql(s"DELETE FROM delta.`$path` WHERE id % 10 = 0") + val deltaLog = DeltaLog.forTable(spark, path) + val before = deltaLog.update() + assert(before.allFiles.collect().exists(_.deletionVector != null)) + val expected = readTable(path).selectExpr("*", "_metadata.row_id").collect() + + sql(s"ALTER TABLE delta.`$path` REPLACE PARTITIONED BY WITH CLUSTER BY NONE") + + val after = deltaLog.update() + val beforeByPath = before.allFiles.collect().map(f => f.path -> f).toMap + after.allFiles.collect().foreach { f => + val old = beforeByPath(f.path) + assert(f.deletionVector === old.deletionVector) + assert(f.baseRowId === old.baseRowId) + assert(f.defaultRowCommitVersion === old.defaultRowCommitVersion) + } + checkAnswer(readTable(path).selectExpr("*", "_metadata.row_id"), expected) + } + } + + test("fails on an unpartitioned table") { + withTempDir { dir => + val path = dir.getCanonicalPath + createPartitionedTable(path) + val e = intercept[DeltaAnalysisException] { + sql(s"ALTER TABLE delta.`$path` REPLACE PARTITIONED BY WITH CLUSTER BY NONE") + } + assert(e.getErrorClass === "DELTA_REPLACE_PARTITIONED_BY_ON_UNPARTITIONED_TABLE") + } + } + + test("fails when data files do not store the partition columns") { + withTempDir { dir => + val path = dir.getCanonicalPath + sql(s"""CREATE TABLE delta.`$path` (id BIGINT, p STRING) USING delta PARTITIONED BY (p) + |TBLPROPERTIES ('delta.writePartitionColumnsToParquet' = 'false')""".stripMargin) + sql(s"INSERT INTO delta.`$path` VALUES (1, 'a'), (2, 'b')") + val deltaLog = DeltaLog.forTable(spark, path) + val versionBefore = deltaLog.update().version + val e = intercept[DeltaAnalysisException] { + sql(s"ALTER TABLE delta.`$path` REPLACE PARTITIONED BY WITH CLUSTER BY NONE") + } + assert(e.getErrorClass === + "DELTA_REPLACE_PARTITIONED_BY_PARTITION_COLUMNS_NOT_MATERIALIZED") + assert(e.getMessage.contains("2 data file(s)")) + assert(deltaLog.update().version === versionBefore) + assert(deltaLog.update().metadata.partitionColumns === Seq("p")) + } + } + + test("verification can be disabled") { + withTempDir { dir => + val path = dir.getCanonicalPath + createPartitionedTable(path, "p") + withSQLConf(DeltaSQLConf + .DELTA_REPLACE_PARTITIONED_BY_VERIFY_MATERIALIZED_PARTITION_COLUMNS.key -> "false") { + sql(s"ALTER TABLE delta.`$path` REPLACE PARTITIONED BY WITH CLUSTER BY NONE") + } + checkAnswer(readTable(path).selectExpr("count(*)"), Row(100L)) + } + } + + test("fails on a non-existing clustering column") { + withTempDir { dir => + val path = dir.getCanonicalPath + createPartitionedTable(path, "p") + intercept[Exception] { + sql(s"ALTER TABLE delta.`$path` REPLACE PARTITIONED BY WITH CLUSTER BY (missing)") + } + assert(DeltaLog.forTable(spark, path).update().metadata.partitionColumns === Seq("p")) + } + } + + test("addPartitionValuesToStats") { + val fields = Seq( + StructField("s", StringType), StructField("i", IntegerType), + StructField("d", DateType), StructField("t", TimestampType), StructField("n", StringType)) + val stats = + """{"numRecords":3,"minValues":{"id":1},"maxValues":{"id":3},"nullCount":{"id":0}}""" + val values = Map("s" -> "abc", "i" -> "7", "d" -> "2024-01-02", + "t" -> "2024-01-02 03:04:05", "n" -> null) + val result = JsonUtils.fromJson[Map[String, Map[String, Any]]]( + ReplacePartitionedByUtils.addPartitionValuesToStats(stats, values, fields, 32) + .replace("\"numRecords\":3,", "")) + assert(result("minValues") === Map("id" -> 1, "s" -> "abc", "i" -> 7, "d" -> "2024-01-02")) + assert(result("maxValues") === Map("id" -> 3, "s" -> "abc", "i" -> 7, "d" -> "2024-01-02")) + assert(result("nullCount") === + Map("id" -> 0, "s" -> 0, "i" -> 0, "d" -> 0, "t" -> 0, "n" -> 3)) + // Long strings are not added since collected string stats would be truncated. + assert(!ReplacePartitionedByUtils.addPartitionValuesToStats( + stats, Map("s" -> ("x" * 33)), fields.take(1), 32).contains("xxx")) + // Files without stats are left untouched. + assert(ReplacePartitionedByUtils.addPartitionValuesToStats(null, values, fields, 32) === null) + assert(ReplacePartitionedByUtils.addPartitionValuesToStats("", values, fields, 32) === "") + } +} diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/stats/DataSkippingAMTTests.scala b/spark/src/test/scala/org/apache/spark/sql/delta/stats/DataSkippingAMTTests.scala new file mode 100644 index 00000000000..a7a2631f143 --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/stats/DataSkippingAMTTests.scala @@ -0,0 +1,167 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.stats + +// scalastyle:off import.ordering.noEmptyLine +import org.apache.spark.sql.delta.{AdaptiveMetadataTableFeature, DeltaLog, DeltaOperations} +import org.apache.spark.sql.delta.actions.TableFeatureProtocolUtils.{defaultPropertyKey, FEATURE_PROP_SUPPORTED} +import org.apache.spark.sql.delta.amt.{AMTCheckpointProvider, AMTTriggerMode} +import org.apache.spark.sql.delta.sources.DeltaSQLConf + +import org.apache.spark.SparkConf +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.types.StructType + +/** + * Runs the DataSkipping V1 corpus against AMT tables. + */ +trait DataSkippingAMTBase extends DataSkippingDeltaTestsBase +{ + + import testImplicits._ + + // AMT requires the catalogManaged feature. `catalogOwnedCoordinatorBackfillBatchSize = Some(1)` + // registers the in-memory commit coordinator (and enables catalog-owned-by-default) so + // catalog-managed AMT tables can be created locally, backfilling every commit immediately. + override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(1) + + override protected def sparkConf: SparkConf = + super.sparkConf + // Every table created in this suite is an AMT table by default. + .set(defaultPropertyKey(AdaptiveMetadataTableFeature), FEATURE_PROP_SUPPORTED) + // Pack few entries per manifest leaf so these small tables produce a real multi-leaf AMT + // tree (root + leaf pointers) instead of a single promoted manifest, exercising the tree + // reconstruction path. + .set(DeltaSQLConf.AMT_ENTRIES_PER_LEAF.key, "2") + + protected def amtExcludedTests: Seq[String] = Seq( + // Column mapping mode conflict: AMT forces `id`, but these create the table in `name`/`none` + // mode explicitly and throw DELTA_ADAPTIVE_METADATA_REQUIRES_COLUMN_MAPPING_ID_MODE at create. + "data skipping with missing workload-based column stats and column mapping (name)", + "data skipping with missing workload-based column stats, column mapping, and RENAME COLUMN", + "data skipping with missing stats for any column and column mapping (name)", + "data skipping with column mapping and upper case column names - +columnMapping = none", + "data skipping with column mapping and upper case column names - +columnMapping = name", + "Data skipping with delta statistic column drop column", + "Data skipping with delta statistic column rename column", + "data skipping flags", + // Row tracking is always enabled on AMT, but these tests disable it explicitly and hit + // DELTA_ADAPTIVE_METADATA_REQUIRES_DEPENDENT_FEATURE_ENABLED at create. + "base_row_id filter throws FIELD_NOT_FOUND when row tracking is disabled", + "default_row_commit_version filter throws FIELD_NOT_FOUND when row tracking is disabled" + ) + + // Substring match; ignoring the base test name here also prevents the corpus from registering + // its DataFrame-schema twin. + override protected def test(testName: String, testTags: org.scalatest.Tag*) + (testFun: => Any)(implicit pos: org.scalactic.source.Position): Unit = { + if (amtExcludedTests.exists(testName.contains)) { + ignore(testName, testTags: _*)(testFun)(pos) + } else { + super.test(testName, testTags: _*)(testFun)(pos) + } + } + + /** + * Emits an AMT checkpoint on `log` (a full rewrite -- self-contained, needs no prior checkpoint), + * then re-resolves the log so the next read picks up the new checkpoint. + */ + override protected def checkpointAndCreateNewLogIfNecessary(log: DeltaLog): DeltaLog = { + log.startTransaction().commit( + Seq.empty, + DeltaOperations.OptimizeCheckpoint( + incremental = false, triggerName = AMTTriggerMode.CheckpointIntervalFull.name)) + DeltaLog.clearCache() + DeltaLog.forTable(spark, log.dataPath) + } + + /** Asserts the AMT V1 skipping path serves reads: V2 disabled, checkpoint is AMT-backed. */ + protected def assertAMTV1(log: DeltaLog): Unit = { + val snapshot = log.update() + assert(snapshot.checkpointProvider.isInstanceOf[AMTCheckpointProvider], + "Expected an AMTCheckpointProvider after checkpoint, got " + + snapshot.checkpointProvider.getClass.getSimpleName) + } + + // AMT manifest stats only exist after a checkpoint, so run each predicate case against a real AMT + // checkpoint, in two source combinations: all files in the manifest; and the manifest with its + // files removed and re-added by a delta on top (mixed log replay of file actions). + override protected def testSkipping( + name: String, + data: String, + schema: StructType = null, + hits: Seq[String], + misses: Seq[String], + sqlConfs: Seq[(String, String)] = Nil, + indexedCols: Int = defaultNumIndexedCols, + deltaStatsColNamesOpt: Option[String] = None, + checkEmptyUnusedFiltersForHits: Boolean = false, + exceptionOpt: Option[Throwable] = None): Unit = { + // No corpus case passes an `exceptionOpt` today. The error path asserts a throw during setup + // via the base's `intercept`, which the AMT checkpoint flow below does not replicate -- so if a + // case ever starts using it, fail loudly here rather than silently mis-running it as a success. + require(exceptionOpt.isEmpty, + s"exceptionOpt is not yet supported by the AMT testSkipping override (test: $name)") + val jsonRecords = data.split("\n").toSeq + val allConfs = sqlConfs ++ getDataSkippingConfs(indexedCols, deltaStatsColNamesOpt) + + def writeTable(t: String, overwrite: Boolean = false): DeltaLog = { + val reader = spark.read + if (schema != null) { reader.schema(schema) } + val writer = reader.json(jsonRecords.toDS()).coalesce(1).write.format("delta") + if (overwrite) { writer.mode("overwrite") } + writer.saveAsTable(t) + DeltaLog.forTable(spark, TableIdentifier(t)) + } + + test(s"data skipping by stats using AMT checkpoint - $name") { + withSQLConf(allConfs: _*) { + val t = "ds_amt_checkpoint" + withTable(t) { + val r = writeTable(t) + val log = checkpointAndCreateNewLogIfNecessary(r) + assertAMTV1(log) + checkSkipping(log, hits, misses, data, checkEmptyUnusedFiltersForHits) + } + } + } + + test(s"data skipping by stats using AMT checkpoint and delta removes - $name") { + withSQLConf(allConfs: _*) { + val t = "ds_amt_checkpoint_and_delta_removes" + withTable(t) { + val r = writeTable(t) + checkpointAndCreateNewLogIfNecessary(r) + // Overwrite with the same data: the delta on top removes the manifest's files and re-adds + // identical ones, so log replay must merge manifest AddFiles with the delta's + // RemoveFile/AddFile actions. The data (hence hits/misses) is unchanged. + writeTable(t, overwrite = true) + assertAMTV1(r) + checkSkipping(r, hits, misses, data, checkEmptyUnusedFiltersForHits) + } + } + } + } +} + +/** + * The base DataSkipping corpus on AMT tables (V1 reader). + */ +trait DataSkippingDeltaV1AMTTests + extends DataSkippingDeltaV1Tests + with DataSkippingAMTBase + with DataSkippingDeltaTestV1ColumnMappingMode diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/stats/StatsCollectionSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/stats/StatsCollectionSuite.scala index 65d49537f60..6795db81f74 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/stats/StatsCollectionSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/stats/StatsCollectionSuite.scala @@ -29,7 +29,7 @@ import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.stats.StatisticsCollection.{ASCII_MAX_CHARACTER, UTF8_MAX_CHARACTER} import org.apache.spark.sql.delta.test.{DeltaExceptionTestUtils, DeltaSQLCommandTest, DeltaSQLTestUtils, TestsStatistics} import org.apache.spark.sql.delta.test.DeltaTestImplicits._ -import org.apache.spark.sql.delta.util.{FileNames, JsonUtils} +import org.apache.spark.sql.delta.util.{DeltaSqlParserUtils, FileNames, JsonUtils} import org.apache.hadoop.fs.Path import org.scalatest.exceptions.TestFailedException @@ -484,6 +484,41 @@ class StatsCollectionSuite } } + test("drop Delta statistics columns round-trips escaped names") { + val tableName = "delta_table" + withTable(tableName) { + // Delta requires column mapping for DROP COLUMN, so use name mode for the ALTER TABLE path. + sql( + s"""CREATE TABLE $tableName ( + | `a,b` LONG, + | `x.y` LONG, + | x STRUCT, + | `a``b` LONG, + | `a````b` LONG, + | `a/b` LONG, + | d LONG) + |USING delta + |TBLPROPERTIES ( + | 'delta.dataSkippingStatsColumns' = + | '`a,b`,`x.y`,x.y,`a``b`,`a````b`,`a/b`,d', + | 'delta.columnMapping.mode' = 'name')""".stripMargin) + sql(s"ALTER TABLE $tableName DROP COLUMN d") + + val metadata = DeltaLog.forTable(spark, TableIdentifier(tableName)).update().metadata + val statsColumns = DeltaConfigs.DATA_SKIPPING_STATS_COLUMNS.fromMetaData(metadata).get + assert(statsColumns == "`a,b`,`x.y`,x.y,`a``b`,`a````b`,`a/b`") + assert( + DeltaSqlParserUtils.parseMultipartColumnList(statsColumns).get.map(_.nameParts) == Seq( + Seq("a,b"), + Seq("x.y"), + Seq("x", "y"), + Seq("a`b"), + Seq("a``b"), + Seq("a/b"))) + StatisticsCollection.validateDeltaStatsColumns(metadata) + } + } + Seq("c1.", "c2*", "c3,", "c-4").foreach { col => test(s"Delta statistic column: invalid special characters $col") { val tableName = "delta_table_1" diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/storage/dv/DeletionVectorFileSizeSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/storage/dv/DeletionVectorFileSizeSuite.scala index 2a6cb0baff8..2f922fcb0d6 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/storage/dv/DeletionVectorFileSizeSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/storage/dv/DeletionVectorFileSizeSuite.scala @@ -26,14 +26,15 @@ import org.apache.spark.sql.delta.test.{DeltaSQLCommandTest, DeltaSQLTestUtils} import org.apache.commons.io.FileUtils import org.apache.hadoop.fs.Path +import org.apache.spark.SparkConf import org.apache.spark.sql.QueryTest -import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SharedSparkSession class DeletionVectorFileSizeSuite extends QueryTest with SharedSparkSession with DeltaSQLTestUtils with DeltaSQLCommandTest { + private def getAddAndRemoveFilesFromCommitVersion( deltaLog: DeltaLog, commitVersion: Long): (Seq[AddFile], Seq[RemoveFile]) = { diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/test/DeltaSQLCommandTest.scala b/spark/src/test/scala/org/apache/spark/sql/delta/test/DeltaSQLCommandTest.scala index 98f3b9b96b9..ae71de745dc 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/test/DeltaSQLCommandTest.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/test/DeltaSQLCommandTest.scala @@ -29,6 +29,9 @@ import org.apache.spark.sql.test.SharedSparkSession */ trait DeltaSQLCommandTest extends SharedSparkSession { + /** Executes DDL and DML; connector-specific suites can override the execution mode. */ + protected def executeDml(sqlText: String): Unit = sql(sqlText) + override protected def sparkConf: SparkConf = { super.sparkConf .set(StaticSQLConf.SPARK_SESSION_EXTENSIONS.key, @@ -37,6 +40,32 @@ trait DeltaSQLCommandTest extends SharedSparkSession { classOf[DeltaCatalog].getName) } + /** + * Sets all configurations specified in `pairs`, calls `f`, and then restores all configurations. + * + * Use this instead of `withSQLConf` as [[internal.SQLConf SQLConf]] is not part of Spark's public + * API. + */ + protected def withConf[T](pairs: (String, String)*)(f: => T): T = { + val (keys, values) = pairs.unzip + val currentValues = keys.map { key => + if (spark.conf.contains(key)) { + Some(spark.conf.get(key)) + } else { + None + } + } + keys.lazyZip(values).foreach { (k, v) => + spark.conf.set(k, v) + } + try f finally { + keys.zip(currentValues).foreach { + case (key, Some(value)) => spark.conf.set(key, value) + case (key, None) => spark.conf.unset(key) + } + } + } + /** * Spark master now routes some invalid Delta table-property updates through the newer * UNSUPPORTED_TABLE_CHANGE error class while released Spark versions still report the legacy diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/typewidening/TypeWideningAlterTableSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/typewidening/TypeWideningAlterTableSuite.scala index 710bdcaa6c2..a746b920227 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/typewidening/TypeWideningAlterTableSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/typewidening/TypeWideningAlterTableSuite.scala @@ -39,7 +39,8 @@ class TypeWideningAlterTableSuite trait TypeWideningAlterTableTests extends QueryTest with QueryErrorsBase - with TypeWideningTestCases { + with TypeWideningTestCases + with DeltaTableProvider { self: QueryTest with ParquetTest with TypeWideningTestMixin => import testImplicits._ @@ -157,7 +158,7 @@ trait TypeWideningAlterTableTests extends QueryTest test( "widening Date -> TimestampNTZ rejected when TimestampNTZ feature isn't supported") { withTimestampNTZDisabled { - sql(s"CREATE TABLE $tableSQLIdentifier (a date) USING DELTA") + sql(createTableSQL(tableSQLIdentifier, "a date")) val currentProtocol = deltaLog.unsafeVolatileSnapshot.protocol val currentFeatures = currentProtocol.implicitlyAndExplicitlySupportedFeatures .map(_.name) @@ -179,7 +180,7 @@ trait TypeWideningAlterTableTests extends QueryTest } test("type widening type change metrics") { - sql(s"CREATE TABLE $tableSQLIdentifier (a byte) USING DELTA") + sql(createTableSQL(tableSQLIdentifier, "a byte")) val usageLogs = Log4jUsageLogger.track { sql(s"ALTER TABLE $tableSQLIdentifier CHANGE COLUMN a TYPE int") } @@ -204,7 +205,7 @@ trait TypeWideningAlterTableTests extends QueryTest } test("type widening with null type in table") { - sql(s"CREATE TABLE $tableSQLIdentifier (a byte, n VOID) USING DELTA") + sql(createTableSQL(tableSQLIdentifier, "a byte, n VOID")) sql(s"ALTER TABLE $tableSQLIdentifier CHANGE COLUMN a TYPE int") } } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/typewidening/TypeWideningStreamingSourceSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/typewidening/TypeWideningStreamingSourceSuite.scala index ed6b192b410..4e342cfeba9 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/typewidening/TypeWideningStreamingSourceSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/typewidening/TypeWideningStreamingSourceSuite.scala @@ -27,7 +27,7 @@ import org.scalatest.BeforeAndAfterAll import org.apache.spark.{SparkException, SparkThrowable} import org.apache.spark.SparkArithmeticException -import org.apache.spark.sql.{DataFrame, Row, SaveMode} +import org.apache.spark.sql.{DataFrame, QueryTest, Row, SaveMode} import org.apache.spark.sql.functions.{col, count, lit} import org.apache.spark.sql.streaming._ import org.apache.spark.sql.test.SQLTestUtils @@ -70,14 +70,14 @@ trait TypeWideningStreamingSourceTestMixin /** Unblocks the stream after a widening type change. */ protected def withUnblockedTypeChanges(fn: => Unit): Unit - override def beforeAll(): Unit = { + override protected def beforeAll(): Unit = { super.beforeAll() spark.sessionState.conf.setConf( DeltaSQLConf.DELTA_TYPE_WIDENING_ENABLE_STREAMING_SCHEMA_TRACKING, schemaTrackingEnabled) spark.udf.register("scala_udf", (x: Int) => x + 1) } - override def afterAll(): Unit = { + override protected def afterAll(): Unit = { // The scala UDF is a temporary function, no need to drop it. super.afterAll() } @@ -114,12 +114,6 @@ trait TypeWideningStreamingSourceTestMixin } } - /** - * Executes a DDL/DML SQL statement. Overridable so that V2 suites can route it through the V1 - * connector, since DeltaV2Table (V2) is read-only and does not support writes/DDL. - */ - protected def executeDml(sqlText: String): Unit = sql(sqlText) - /** Test action checking that the stream fails due to a metadata change - typ. a schema change. */ object ExpectMetadataEvolutionException { def apply(): StreamAction = if (schemaTrackingEnabled) { @@ -160,9 +154,10 @@ trait TypeWideningStreamingSourceTestMixin * Can run both with and without schema tracking. */ trait TypeWideningStreamingSourceTests - extends StreamTest + extends QueryTest with SQLTestUtils - with TypeWideningStreamingSourceTestMixin { + with TypeWideningStreamingSourceTestMixin + with StreamTest { import testImplicits._ @@ -525,9 +520,10 @@ trait TypeWideningStreamingSourceTests /** Tests that specifically cover type widening without schema tracking. */ trait TypeWideningStreamingSourceWithoutSchemaTrackingTests - extends StreamTest + extends QueryTest with SQLTestUtils - with TypeWideningStreamingSourceTestMixin { + with TypeWideningStreamingSourceTestMixin + with StreamTest { import testImplicits._ @@ -601,9 +597,10 @@ trait TypeWideningStreamingSourceWithoutSchemaTrackingTests /** Tests that specifically cover schema tracking for type widening. */ trait TypeWideningStreamingSourceSchemaTrackingTests - extends StreamTest + extends QueryTest with SQLTestUtils - with TypeWideningStreamingSourceTestMixin { + with TypeWideningStreamingSourceTestMixin + with StreamTest { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/typewidening/TypeWideningTestCases.scala b/spark/src/test/scala/org/apache/spark/sql/delta/typewidening/TypeWideningTestCases.scala index d53b0fa3fa5..1e1883204ec 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/typewidening/TypeWideningTestCases.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/typewidening/TypeWideningTestCases.scala @@ -23,7 +23,7 @@ import org.apache.spark.sql.types._ /** * Trait collecting supported and unsupported type change test cases. */ -trait TypeWideningTestCases extends SQLTestUtils { self: SharedSparkSession => +trait TypeWideningTestCases extends SQLTestUtils { import testImplicits._ /** diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/util/JsonUtilsSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/util/JsonUtilsSuite.scala index 806cfc354fd..75f25612fe1 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/util/JsonUtilsSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/util/JsonUtilsSuite.scala @@ -19,7 +19,7 @@ package org.apache.spark.sql.delta.util import scala.util.Random import org.apache.spark.sql.delta.DeltaLog -import org.apache.spark.sql.delta.actions.CommitInfo +import org.apache.spark.sql.delta.actions.{CommitInfo, LastManifestCommit} import org.apache.spark.sql.delta.stats.DataSize import com.fasterxml.jackson.core.StreamReadConstraints @@ -61,10 +61,12 @@ class JsonUtilsSuite readVersion = Some(1), isolationLevel = None, isBlindAppend = Some(false), + dataChange = Some(true), operationMetrics = None, userMetadata = Some("I am a test and not a user"), tags = None, - txnId = Some("Transaction with a veryyyyyyy large commit info") + txnId = Some("Transaction with a veryyyyyyy large commit info"), + lastManifestCommit = Some(LastManifestCommit(version = 43, contentRootVersion = 41)) ) val serialized = JsonUtils.toJson(commitInfo) diff --git a/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDeltaAPITest.java b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDeltaAPITest.java index 6904f883030..4f7a12c215e 100644 --- a/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDeltaAPITest.java +++ b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDeltaAPITest.java @@ -18,7 +18,6 @@ import static org.apache.spark.sql.functions.col; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; import io.delta.tables.DeltaTable; import java.util.List; @@ -128,15 +127,10 @@ public void testOptimize(TableType tableType) throws Exception { tableType, tableName -> { sql("INSERT INTO %s VALUES (1, 'A'), (2, 'B'), (3, 'A')", tableName); - if (tableType == TableType.MANAGED) { - assertThatThrownBy(() -> forName(tableName).optimize().executeCompaction()) - .hasMessageContaining("DELTA_UNSUPPORTED_CATALOG_MANAGED_TABLE_OPERATION"); - } else { - assertThat(forName(tableName).optimize().executeCompaction().collectAsList()) - .isNotEmpty(); - assertThat(forName(tableName).optimize().executeZOrderBy("category").collectAsList()) - .isNotEmpty(); - } + assertThat(forName(tableName).optimize().executeCompaction().collectAsList()) + .isNotEmpty(); + assertThat(forName(tableName).optimize().executeZOrderBy("category").collectAsList()) + .isNotEmpty(); }); } diff --git a/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaUtilityTest.java b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaUtilityTest.java index ac9d2b6dbe9..a4d76d4effa 100644 --- a/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaUtilityTest.java +++ b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaUtilityTest.java @@ -16,11 +16,22 @@ package io.sparkuctest; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - +import io.delta.tables.DeltaTable; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.FileTime; import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.spark.sql.Row; import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.Test; public class UCDeltaUtilityTest extends UCDeltaTableIntegrationBaseTest { @@ -140,25 +151,205 @@ public void testFsPropertiesHiddenFromTableProperties(TableType tableType) throw } @Test - public void testMaintenanceOpsBlockedOnManagedTable() throws Exception { + public void testOptimizeRewritesManagedTableFiles() throws Exception { + withNewTable( + "optimize_rewrites_files", + "id INT", + TableType.MANAGED, + tableName -> { + sql("INSERT INTO %s VALUES (1)", tableName); + sql("INSERT INTO %s VALUES (2)", tableName); + sql("INSERT INTO %s VALUES (3)", tableName); + + Map> filesBefore = activeFileRows(tableName); + Assertions.assertThat(filesBefore.values()) + .containsExactlyInAnyOrder(List.of(1), List.of(2), List.of(3)); + + long versionBefore = currentVersion(tableName); + sql("OPTIMIZE %s", tableName); + + Map> filesAfter = activeFileRows(tableName); + Assertions.assertThat(filesAfter).hasSize(1); + Assertions.assertThat(filesAfter.values()).containsExactly(List.of(1, 2, 3)); + Assertions.assertThat(filesAfter.keySet()) + .doesNotContainAnyElementsOf(filesBefore.keySet()); + Assertions.assertThat(currentVersion(tableName)).isEqualTo(versionBefore + 1); + assertLatestOperation(tableName, "OPTIMIZE"); + }); + } + + @Test + public void testVacuumDeletesObsoleteManagedTableFiles() throws Exception { withNewTable( - "maintenance_blocked", + "vacuum_deletes_files", "id INT", TableType.MANAGED, tableName -> { sql("INSERT INTO %s VALUES (1)", tableName); + sql("INSERT INTO %s VALUES (2)", tableName); + sql("INSERT INTO %s VALUES (3)", tableName); + + Set obsoleteFiles = activeFileRows(tableName).keySet(); + sql("OPTIMIZE %s", tableName); + Map> activeFiles = activeFileRows(tableName); + Assertions.assertThat(activeFiles).hasSize(1); - assertThatThrownBy(() -> sql("OPTIMIZE %s", tableName)) - .hasMessageContaining("DELTA_UNSUPPORTED_CATALOG_MANAGED_TABLE_OPERATION") - .hasMessageContaining("OPTIMIZE"); + String retentionCheck = "spark.databricks.delta.retentionDurationCheck.enabled"; + String previousRetentionCheck = spark().conf().get(retentionCheck, "true"); + spark().conf().set(retentionCheck, "false"); + try { + Set candidatesBefore = vacuumCandidateNames(tableName); + Set obsoleteFileNames = + obsoleteFiles.stream().map(this::fileName).collect(Collectors.toSet()); + Assertions.assertThat(candidatesBefore).containsAll(obsoleteFileNames); - assertThatThrownBy(() -> sql("VACUUM %s", tableName)) - .hasMessageContaining("DELTA_UNSUPPORTED_CATALOG_MANAGED_TABLE_OPERATION") - .hasMessageContaining("VACUUM"); + sql("VACUUM %s RETAIN 0 HOURS", tableName); - assertThatThrownBy(() -> sql("REORG TABLE %s APPLY (PURGE)", tableName)) - .hasMessageContaining("DELTA_UNSUPPORTED_CATALOG_MANAGED_TABLE_OPERATION") - .hasMessageContaining("OPTIMIZE"); + Assertions.assertThat(vacuumCandidateNames(tableName)) + .doesNotContainAnyElementsOf(obsoleteFileNames); + Assertions.assertThat(activeFileRows(tableName)).isEqualTo(activeFiles); + check(tableName, List.of(row("1"), row("2"), row("3"))); + } finally { + spark().conf().set(retentionCheck, previousRetentionCheck); + } }); } + + @Test + public void testReorgPurgesManagedTableDeletionVector() throws Exception { + withNewTable( + "reorg_purges_deletion_vector", + "id INT", + TableType.MANAGED, + tableName -> { + sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); + sql("DELETE FROM %s WHERE id = 2", tableName); + + Map deleteMetrics = latestOperationMetrics(tableName, "DELETE"); + Assertions.assertThat(Long.parseLong(deleteMetrics.get("numDeletionVectorsAdded"))) + .isGreaterThan(0); + Map> filesBefore = activeFileRows(tableName); + Assertions.assertThat(allRows(filesBefore)).containsExactly(1, 3); + + sql("REORG TABLE %s APPLY (PURGE)", tableName); + + Map reorgMetrics = latestOperationMetrics(tableName, "REORG"); + long removedDeletionVectors = + Long.parseLong(reorgMetrics.get("numDeletionVectorsRemoved")); + long addedFiles = Long.parseLong(reorgMetrics.get("numAddedFiles")); + long removedFiles = Long.parseLong(reorgMetrics.get("numRemovedFiles")); + Assertions.assertThat(removedDeletionVectors).isGreaterThan(0); + Assertions.assertThat(addedFiles).isGreaterThan(0); + Assertions.assertThat(removedFiles).isGreaterThan(0); + Map> filesAfter = activeFileRows(tableName); + Assertions.assertThat(allRows(filesAfter)).containsExactly(1, 3); + Assertions.assertThat( + filesAfter.keySet().stream() + .filter(path -> !filesBefore.containsKey(path)) + .count()) + .isEqualTo(addedFiles); + Assertions.assertThat( + filesBefore.keySet().stream() + .filter(path -> !filesAfter.containsKey(path)) + .count()) + .isEqualTo(removedFiles); + }); + } + + @Test + public void testCheckpointDeletesExpiredManagedTableLogs() throws Exception { + Assumptions.assumeFalse( + isUCRemoteConfigured(), "This test changes timestamps in the local fake S3 filesystem."); + withNewTable( + "checkpoint_deletes_logs", + "id INT", + null, + TableType.MANAGED, + "'delta.checkpointInterval'='1', " + + "'delta.logRetentionDuration'='interval 0 seconds', " + + "'delta.deletedFileRetentionDuration'='interval 0 seconds'", + tableName -> { + sql("INSERT INTO %s VALUES (1)", tableName); + + Path logPath = localDeltaLogPath(tableName); + Path versionZero = logPath.resolve("00000000000000000000.json"); + Path versionOne = logPath.resolve("00000000000000000001.json"); + Assertions.assertThat(versionZero).exists(); + Assertions.assertThat(versionOne).exists(); + try (Stream files = Files.list(logPath)) { + for (Path path : files.filter(this::isVersionedLogFile).collect(Collectors.toList())) { + Files.setLastModifiedTime(path, FileTime.fromMillis(0)); + } + } + + sql("INSERT INTO %s VALUES (2)", tableName); + + Assertions.assertThat(versionZero).doesNotExist(); + Assertions.assertThat(versionOne).doesNotExist(); + try (Stream files = Files.list(logPath)) { + Assertions.assertThat(files.filter(this::isCheckpointFile).count()).isGreaterThan(0); + } + check(tableName, List.of(row("1"), row("2"))); + }); + } + + private Map> activeFileRows(String tableName) { + Map> rowsByFile = new HashMap<>(); + for (Row row : spark().sql("SELECT input_file_name(), id FROM " + tableName).collectAsList()) { + rowsByFile.computeIfAbsent(row.getString(0), ignored -> new ArrayList<>()).add(row.getInt(1)); + } + rowsByFile.values().forEach(Collections::sort); + return rowsByFile; + } + + private Set vacuumCandidateNames(String tableName) { + return sql("VACUUM %s RETAIN 0 HOURS DRY RUN", tableName).stream() + .map(row -> fileName(row.get(0))) + .collect(Collectors.toSet()); + } + + private List allRows(Map> rowsByFile) { + return rowsByFile.values().stream().flatMap(List::stream).sorted().collect(Collectors.toList()); + } + + private String fileName(String path) { + return Path.of(URI.create(path).getPath()).getFileName().toString(); + } + + private void assertLatestOperation(String tableName, String expectedOperation) { + latestOperationMetrics(tableName, expectedOperation); + } + + private Map latestOperationMetrics(String tableName, String expectedOperation) { + Row history = + DeltaTable.forName(spark(), tableName) + .history(1) + .select("operation", "operationMetrics") + .head(); + Assertions.assertThat(history.getString(0)).isEqualTo(expectedOperation); + return history.getJavaMap(1); + } + + private Path localDeltaLogPath(String tableName) { + String location = + sql("DESCRIBE FORMATTED %s", tableName).stream() + .filter(row -> row.size() >= 2 && "Location".equalsIgnoreCase(row.get(0).trim())) + .map(row -> row.get(1).trim()) + .findFirst() + .orElseThrow(() -> new AssertionError("Could not retrieve table location")); + URI locationUri = URI.create(location); + if ("file".equals(locationUri.getScheme())) { + return Path.of(locationUri).resolve("_delta_log"); + } + Assertions.assertThat(locationUri.getHost()).as(location).isEqualTo(FAKE_S3_BUCKET); + return Path.of(locationUri.getPath()).resolve("_delta_log"); + } + + private boolean isVersionedLogFile(Path path) { + return path.getFileName().toString().matches("[0-9]{20}\\..*"); + } + + private boolean isCheckpointFile(Path path) { + return path.getFileName().toString().contains(".checkpoint."); + } } diff --git a/spark/v2/src/main/java-shims/spark-4.2/io/delta/spark/internal/v2/catalog/DeltaV2TableShims.java b/spark/v2/src/main/java-shims/spark-4.2/io/delta/spark/internal/v2/catalog/DeltaV2TableShims.java index e23c7990512..5676660ab07 100644 --- a/spark/v2/src/main/java-shims/spark-4.2/io/delta/spark/internal/v2/catalog/DeltaV2TableShims.java +++ b/spark/v2/src/main/java-shims/spark-4.2/io/delta/spark/internal/v2/catalog/DeltaV2TableShims.java @@ -16,26 +16,39 @@ package io.delta.spark.internal.v2.catalog; +import io.delta.kernel.engine.Engine; +import org.apache.spark.sql.delta.Snapshot; import org.apache.spark.sql.delta.ColumnTypeChangeSupport; import org.apache.spark.sql.delta.v2.interop.AbstractMetadata; import org.apache.spark.sql.delta.v2.interop.AbstractProtocol; import io.delta.spark.internal.v2.utils.ScalaUtils; +import io.delta.spark.internal.v2.write.DeltaMetadataOnlyDeleteExecutor; +import java.util.Arrays; import java.util.Optional; import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.catalyst.catalog.CatalogTable; +import org.apache.spark.sql.connector.catalog.SupportsDeleteV2; import org.apache.spark.sql.connector.catalog.SupportsSchemaEvolution; import org.apache.spark.sql.connector.catalog.TableCapability; import org.apache.spark.sql.connector.catalog.TableChange; +import org.apache.spark.sql.connector.expressions.filter.AlwaysTrue; +import org.apache.spark.sql.connector.expressions.filter.PartitionPredicate; +import org.apache.spark.sql.connector.expressions.filter.Predicate; /** * Shim to build the DSv2 Delta table connector against different Spark versions. * This is the shim for Spark 4.2: * - Schema evolution support in DSV2 (add columns, change type) was added in Spark 4.2. + * - Partition-aware metadata-only DELETE support depends on Spark 4.2's PartitionPredicate. */ -public abstract class DeltaV2TableShims implements SupportsSchemaEvolution { +public abstract class DeltaV2TableShims implements SupportsSchemaEvolution, SupportsDeleteV2 { /** Implemented in DeltaV2Table to provide access to the table protocol/metadata. */ protected abstract AbstractProtocol protocol(); protected abstract AbstractMetadata metadata(); + protected abstract Engine kernelEngine(); + protected abstract Snapshot initialSnapshot(); + protected abstract Optional catalogTable(); /** * Whether an analyzer-proposed schema-evolution column change (adding a column or widening a @@ -63,4 +76,21 @@ public boolean supportsColumnChange(TableChange.ColumnChange change) { public static Optional schemaEvolutionCapability() { return Optional.of(TableCapability.AUTOMATIC_SCHEMA_EVOLUTION); } + + /** + * A DELETE qualifies for metadata-only execution when every predicate can be evaluated against + * partition values alone: either a whole-table {@link AlwaysTrue}, or a + * {@link PartitionPredicate} from a partition-column condition. + */ + @Override + public boolean canDeleteWhere(Predicate[] predicates) { + return Arrays.stream(predicates) + .allMatch(p -> p instanceof AlwaysTrue || p instanceof PartitionPredicate); + } + + @Override + public void deleteWhere(Predicate[] predicates) { + DeltaMetadataOnlyDeleteExecutor.deleteWhere( + kernelEngine(), initialSnapshot(), catalogTable(), predicates); + } } diff --git a/spark/v2/src/main/java-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaChangelog.java b/spark/v2/src/main/java-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaV2Changelog.java similarity index 82% rename from spark/v2/src/main/java-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaChangelog.java rename to spark/v2/src/main/java-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaV2Changelog.java index b65f6c0e337..24784776dc9 100644 --- a/spark/v2/src/main/java-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaChangelog.java +++ b/spark/v2/src/main/java-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaV2Changelog.java @@ -1,6 +1,6 @@ package io.delta.spark.internal.v2.read.changelog; -import io.delta.kernel.Snapshot; +import org.apache.spark.sql.delta.Snapshot; import io.delta.spark.internal.v2.catalog.DeltaV2Table; import io.delta.spark.internal.v2.shims.CatalogV2UtilShims; import io.delta.spark.internal.v2.utils.SchemaUtils; @@ -18,18 +18,22 @@ * *

Wraps the {@link DeltaV2Table} resolved by {@code TableCatalog.loadTable(ident)}. The * connector-level work (snapshot loads, row tracking validation, metadata-action inspection - * across the range) is deferred to the read path inside {@link DeltaChangelogBatch}. The schema + * across the range) is deferred to the read path inside {@link DeltaV2ChangelogBatch}. The schema * exposed by {@link #columns()} is the end-version schema. It matches the {@code dataSchema} the * scan builds against, so analysis-time column resolution agrees with the per-commit Metadata * validation performed at scan planning. * *

Row tracking is required at the table protocol. Without it the SPIP analyzer rule cannot * partition by {@code rowId / rowVersion}. Validation is performed by the read path, not here. + * + *

This class remains public because catalog integration in a sibling package constructs it. + * The scan builder, scan, and batch implementations stay package-private so callers cannot couple + * to Delta's internal V2 read path. */ -public class DeltaChangelog implements Changelog { +public class DeltaV2Changelog implements Changelog { private final String tableName; - private final DeltaV2Table sparkTable; + private final DeltaV2Table deltaV2Table; private final long startVersion; private final long endVersion; @@ -41,10 +45,10 @@ public class DeltaChangelog implements Changelog { .add(ROW_ID_FIELD, DataTypes.LongType, false) .add(ROW_COMMIT_VERSION_FIELD, DataTypes.LongType, false); - public DeltaChangelog( - String tableName, DeltaV2Table sparkTable, long startVersion, long endVersion) { + public DeltaV2Changelog( + String tableName, DeltaV2Table deltaV2Table, long startVersion, long endVersion) { this.tableName = tableName; - this.sparkTable = sparkTable; + this.deltaV2Table = deltaV2Table; this.startVersion = startVersion; this.endVersion = endVersion; } @@ -58,8 +62,8 @@ public String name() { public Column[] columns() { // Resolve lazily so catalog construction stays side-effect free. The scan path validates // each per-commit Metadata against this same end-version schema. - Snapshot endSnapshot = sparkTable.getSnapshotManager().loadSnapshotAt(endVersion); - StructType endSchema = SchemaUtils.convertKernelSchemaToSparkSchema(endSnapshot.getSchema()); + Snapshot endSnapshot = deltaV2Table.getSnapshotManager().loadSnapshotAt(endVersion); + StructType endSchema = endSnapshot.schema(); StructType cdcSchema = endSchema .add(METADATA_COLUMN, METADATA_STRUCT, false) @@ -94,7 +98,7 @@ public boolean representsUpdateAsDeleteAndInsert() { @Override public ScanBuilder newScanBuilder(CaseInsensitiveStringMap options) { - return new DeltaChangelogScanBuilder(sparkTable, startVersion, endVersion, options); + return new DeltaV2ChangelogScanBuilder(deltaV2Table, startVersion, endVersion, options); } @Override diff --git a/spark/v2/src/main/java-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaChangelogBatch.java b/spark/v2/src/main/java-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaV2ChangelogBatch.java similarity index 97% rename from spark/v2/src/main/java-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaChangelogBatch.java rename to spark/v2/src/main/java-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaV2ChangelogBatch.java index b486c93ef9e..ce991b70ed4 100644 --- a/spark/v2/src/main/java-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaChangelogBatch.java +++ b/spark/v2/src/main/java-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaV2ChangelogBatch.java @@ -56,8 +56,10 @@ * stable. {@link #createReaderFactory()} wraps the Parquet reader with * {@link CDCPartitionReaderFactory} to append the CDC tail columns ({@code _change_type}, * {@code _commit_version}, {@code _commit_timestamp}). + * + *

Package privacy prevents callers from coupling to Delta's internal V2 implementation. */ -public class DeltaChangelogBatch implements Batch { +class DeltaV2ChangelogBatch implements Batch { private static final Set CHANGELOG_ACTION_SET = Set.of( DeltaLogActionUtils.DeltaAction.ADD, @@ -72,7 +74,7 @@ public class DeltaChangelogBatch implements Batch { private final Snapshot snapshot; private final Configuration hadoopConf; - public DeltaChangelogBatch( + DeltaV2ChangelogBatch( CommitRange commitRange, Engine engine, StructType endDataSchema, @@ -240,7 +242,7 @@ public InputPartition[] planInputPartitions() { totalBytes += remove.getSize().orElse(0L); } // Validate Metadata actions: schema and row-tracking config must match the - // end-version baseline established by DeltaChangelogScanBuilder. Mid-range + // end-version baseline established by DeltaV2ChangelogScanBuilder. Mid-range // schema evolution or row-tracking-toggle would silently corrupt downstream // CDC post-processing (row identity / column mapping drift). Optional metadataOpt = StreamingHelper.getMetadata(batch, rowId); @@ -299,7 +301,8 @@ private static SparkPath sparkPathFromRawPath(String tablePath, String filePath) public PartitionReaderFactory createReaderFactory() { StructType partitionSchema = new StructType(); StructType readDataSchema = - endDataSchema.add(DeltaChangelog.METADATA_COLUMN, DeltaChangelog.METADATA_STRUCT, false); + endDataSchema.add( + DeltaV2Changelog.METADATA_COLUMN, DeltaV2Changelog.METADATA_STRUCT, false); Filter[] dataFilters = new Filter[0]; scala.collection.immutable.Map scalaOptions = Map$.MODULE$.empty(); diff --git a/spark/v2/src/main/java-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaChangelogScan.java b/spark/v2/src/main/java-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaV2ChangelogScan.java similarity index 80% rename from spark/v2/src/main/java-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaChangelogScan.java rename to spark/v2/src/main/java-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaV2ChangelogScan.java index 38feab4e0a1..3813dc4cdc7 100644 --- a/spark/v2/src/main/java-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaChangelogScan.java +++ b/spark/v2/src/main/java-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaV2ChangelogScan.java @@ -8,7 +8,12 @@ import org.apache.spark.sql.connector.read.Scan; import org.apache.spark.sql.types.StructType; -public class DeltaChangelogScan implements Scan { +/** + * Package-private scan for Delta's V2 changelog read path. + * + *

Package privacy prevents callers from coupling to Delta's internal V2 implementation. + */ +class DeltaV2ChangelogScan implements Scan { private final Engine engine; private final StructType readSchema; private final CommitRange commitRange; @@ -18,7 +23,7 @@ public class DeltaChangelogScan implements Scan { private final long endVersion; private final Configuration hadoopConf; - public DeltaChangelogScan( + DeltaV2ChangelogScan( StructType readSchema, CommitRange commitRange, Engine engine, @@ -45,12 +50,12 @@ public StructType readSchema() { @Override public String description() { return String.format( - "DeltaChangelogScan [startVersion=%d, endVersion=%d]", startVersion, endVersion); + "DeltaV2ChangelogScan [startVersion=%d, endVersion=%d]", startVersion, endVersion); } @Override public Batch toBatch() { - return new DeltaChangelogBatch(commitRange, engine, dataSchema, snapshot, hadoopConf); + return new DeltaV2ChangelogBatch(commitRange, engine, dataSchema, snapshot, hadoopConf); } // TODO: implement toMicroBatchStream() so spark.readStream...loadChangelog(...) can drive a diff --git a/spark/v2/src/main/java-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaChangelogScanBuilder.java b/spark/v2/src/main/java-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaV2ChangelogScanBuilder.java similarity index 61% rename from spark/v2/src/main/java-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaChangelogScanBuilder.java rename to spark/v2/src/main/java-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaV2ChangelogScanBuilder.java index 1f7f27dbfb4..078880273de 100644 --- a/spark/v2/src/main/java-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaChangelogScanBuilder.java +++ b/spark/v2/src/main/java-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaV2ChangelogScanBuilder.java @@ -1,13 +1,14 @@ package io.delta.spark.internal.v2.read.changelog; import io.delta.kernel.CommitRange; -import io.delta.kernel.Snapshot; -import io.delta.kernel.defaults.engine.DefaultEngine; +import org.apache.spark.sql.delta.Snapshot; import io.delta.kernel.engine.Engine; import io.delta.kernel.internal.SnapshotImpl; -import io.delta.kernel.internal.rowtracking.RowTracking; +import org.apache.spark.sql.delta.RowTracking$; import io.delta.spark.internal.v2.catalog.DeltaV2Table; -import io.delta.spark.internal.v2.snapshot.DeltaSnapshotManager; +import io.delta.spark.internal.v2.kernel.KernelEngineFactory; +import org.apache.spark.sql.delta.v2.interop.DeltaV2Snapshot$; +import org.apache.spark.sql.delta.v2.interop.DeltaV2SnapshotManager; import io.delta.spark.internal.v2.utils.SchemaUtils; import java.util.Objects; import java.util.Optional; @@ -20,16 +21,24 @@ import org.apache.spark.sql.types.StructType; import org.apache.spark.sql.util.CaseInsensitiveStringMap; -public class DeltaChangelogScanBuilder implements ScanBuilder { +/** + * Package-private scan builder for Delta's V2 changelog read path. + * + *

Package privacy prevents callers from coupling to Delta's internal V2 implementation. + */ +class DeltaV2ChangelogScanBuilder implements ScanBuilder { - private final DeltaV2Table sparkTable; + private final DeltaV2Table deltaV2Table; private final long startVersion; private final long endVersion; private final CaseInsensitiveStringMap options; - public DeltaChangelogScanBuilder( - DeltaV2Table sparkTable, long startVersion, long endVersion, CaseInsensitiveStringMap options) { - this.sparkTable = sparkTable; + DeltaV2ChangelogScanBuilder( + DeltaV2Table deltaV2Table, + long startVersion, + long endVersion, + CaseInsensitiveStringMap options) { + this.deltaV2Table = deltaV2Table; this.startVersion = startVersion; this.endVersion = endVersion; this.options = options; @@ -40,12 +49,12 @@ public Scan build() { Configuration hadoopConf = Objects.requireNonNull( SparkSession.active().sparkContext().hadoopConfiguration(), "hadoopConf is null"); - Engine engine = DefaultEngine.create(hadoopConf); - DeltaSnapshotManager snapshotManager = sparkTable.getSnapshotManager(); + Engine engine = KernelEngineFactory.createDefaultEngine(hadoopConf); + DeltaV2SnapshotManager snapshotManager = deltaV2Table.getSnapshotManager(); CommitRange commitRange = snapshotManager.getTableChanges(engine, startVersion, Optional.of(endVersion)); // Boundary checks: both endpoints must already carry the schema + RT state that - // DeltaChangelogBatch will validate each in-range Metadata action against. Without these, + // DeltaV2ChangelogBatch will validate each in-range Metadata action against. Without these, // an RT-disabled boundary with no in-range toggle commit would surface as a raw // IllegalStateException "missing baseRowId" downstream. // @@ -55,29 +64,28 @@ public Scan build() { // but the start does not, the toggle happened within the range -- emit // DELTA_CHANGELOG_ROW_TRACKING_DISABLED_IN_RANGE with the offending start version. Snapshot startSnapshot = snapshotManager.loadSnapshotAt(startVersion); - SnapshotImpl startSnapshotImpl = (SnapshotImpl) startSnapshot; + SnapshotImpl startSnapshotImpl = DeltaV2Snapshot$.MODULE$.getKernelSnapshot(startSnapshot); Snapshot endSnapshot = snapshotManager.loadSnapshotAt(endVersion); - SnapshotImpl endSnapshotImpl = (SnapshotImpl) endSnapshot; - StructType endSchema = SchemaUtils.convertKernelSchemaToSparkSchema(endSnapshot.getSchema()); - if (!RowTracking.isEnabled(endSnapshotImpl.getProtocol(), endSnapshotImpl.getMetadata())) { - DeltaErrors.throwChangelogRequiresRowTracking(sparkTable.name()); + StructType endSchema = endSnapshot.schema(); + if (!RowTracking$.MODULE$.isEnabled(endSnapshot.protocol(), endSnapshot.metadata())) { + DeltaErrors.throwChangelogRequiresRowTracking(deltaV2Table.name()); } - if (!RowTracking.isEnabled(startSnapshotImpl.getProtocol(), startSnapshotImpl.getMetadata())) { + if (!RowTracking$.MODULE$.isEnabled(startSnapshot.protocol(), startSnapshot.metadata())) { DeltaErrors.throwChangelogRowTrackingDisabledInRange(startVersion); } StructType cdcSchema = endSchema - .add(DeltaChangelog.METADATA_COLUMN, DeltaChangelog.METADATA_STRUCT, false) + .add(DeltaV2Changelog.METADATA_COLUMN, DeltaV2Changelog.METADATA_STRUCT, false) .add("_change_type", DataTypes.StringType, false) .add("_commit_version", DataTypes.LongType, false) .add("_commit_timestamp", DataTypes.TimestampType, false); - return new DeltaChangelogScan( + return new DeltaV2ChangelogScan( cdcSchema, commitRange, engine, endSchema, - startSnapshot, + startSnapshotImpl, startVersion, endVersion, hadoopConf); diff --git a/spark/v2/src/main/java-shims/spark-4.2/io/delta/spark/internal/v2/write/DeltaMetadataOnlyDeleteExecutor.java b/spark/v2/src/main/java-shims/spark-4.2/io/delta/spark/internal/v2/write/DeltaMetadataOnlyDeleteExecutor.java new file mode 100644 index 00000000000..07a447001ba --- /dev/null +++ b/spark/v2/src/main/java-shims/spark-4.2/io/delta/spark/internal/v2/write/DeltaMetadataOnlyDeleteExecutor.java @@ -0,0 +1,193 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.spark.internal.v2.write; + +import static io.delta.spark.internal.v2.utils.ScalaUtils.toJavaMap; +import static io.delta.spark.internal.v2.utils.ScalaUtils.toScalaList; +import static java.util.Objects.requireNonNull; + +import io.delta.kernel.Operation; +import io.delta.kernel.Transaction; +import io.delta.kernel.TransactionCommitResult; +import io.delta.kernel.data.MapValue; +import io.delta.kernel.data.Row; +import io.delta.kernel.engine.Engine; +import io.delta.kernel.internal.actions.DeletionVectorDescriptor; +import io.delta.kernel.internal.actions.GenerateIcebergCompatActionUtils; +import io.delta.kernel.internal.actions.SingleAction; +import io.delta.kernel.internal.util.Utils; +import io.delta.kernel.internal.util.VectorUtils; +import io.delta.kernel.statistics.DataFileStatistics; +import io.delta.kernel.types.StructType; +import io.delta.kernel.utils.CloseableIterable; +import org.apache.spark.sql.delta.DeltaColumnMapping; +import org.apache.spark.sql.delta.Snapshot; +import org.apache.spark.sql.delta.actions.AddFile; +import org.apache.spark.sql.delta.v2.interop.DeltaV2Snapshot; +import org.apache.spark.sql.delta.v2.interop.DeltaV2Snapshot$; +import io.delta.spark.internal.v2.utils.SchemaUtils; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import org.apache.spark.sql.catalyst.catalog.CatalogTable; +import org.apache.spark.sql.catalyst.expressions.Expression; +import org.apache.spark.sql.catalyst.expressions.V2ExpressionUtils; +import org.apache.spark.sql.connector.expressions.filter.AlwaysTrue; +import org.apache.spark.sql.connector.expressions.filter.Predicate; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import scala.Option; +import scala.jdk.javaapi.CollectionConverters; + +/** + * Executes a metadata-only DELETE on the DSv2 Delta table: the set of files whose rows all match + * the delete condition are removed in their entirety by committing {@code RemoveFile} actions, + * without scanning or rewriting any data. + */ +public final class DeltaMetadataOnlyDeleteExecutor { + + private static final Logger LOG = + LoggerFactory.getLogger(DeltaMetadataOnlyDeleteExecutor.class); + + private DeltaMetadataOnlyDeleteExecutor() {} + + /** Selects and removes every file whose rows match {@code predicates}. */ + public static void deleteWhere( + Engine engine, + Snapshot initialSnapshot, + Optional catalogTable, + Predicate[] predicates) { + requireNonNull(engine, "engine is null"); + requireNonNull(initialSnapshot, "initialSnapshot is null"); + requireNonNull(catalogTable, "catalogTable is null"); + requireNonNull(predicates, "predicates is null"); + + List filesToRemove = + selectFilesToRemove(engine, initialSnapshot, catalogTable, predicates); + + if (filesToRemove.isEmpty()) { + LOG.info("Metadata-only delete matched no files; nothing to commit"); + return; + } + + deleteFiles(engine, initialSnapshot, filesToRemove); + } + + private static List selectFilesToRemove( + Engine engine, + Snapshot initialSnapshot, + Optional catalogTable, + Predicate[] predicates) { + DeltaV2Snapshot snapshot = + (DeltaV2Snapshot) initialSnapshot; + List catalystFilters = new ArrayList<>(predicates.length); + for (Predicate predicate : predicates) { + if (!(predicate instanceof AlwaysTrue)) { + catalystFilters.add(toCatalyst(predicate)); + } + } + + return CollectionConverters.asJava( + snapshot + .filesForScan( + toScalaList(catalystFilters.toArray(new Expression[0])), + true) + .files()); + } + + private static Expression toCatalyst(Predicate predicate) { + // Conversion is all-or-nothing: None means this predicate or one of its children is + // unsupported. + Option expression = V2ExpressionUtils.toCatalyst(predicate); + if (expression.isEmpty()) { + throw new IllegalStateException("Could not convert " + predicate + " to Catalyst"); + } + return expression.get(); + } + + private static void deleteFiles( + Engine engine, Snapshot initialSnapshot, List filesToRemove) { + // The operation is committed as WRITE, not DELETE as Kernel Transactions don't support it. + Transaction transaction = + DeltaV2Snapshot$.MODULE$.getKernelSnapshot(initialSnapshot) + .buildUpdateTableTransaction(DeltaV2BatchWrite.getEngineInfo(), Operation.WRITE) + .build(engine); + + StructType physicalSchema = + SchemaUtils.convertSparkSchemaToKernelSchema( + DeltaColumnMapping.renameColumns(initialSnapshot.schema())); + long removeTimestamp = System.currentTimeMillis(); + + List removeActionRows = new ArrayList<>(filesToRemove.size()); + for (AddFile file : filesToRemove) { + removeActionRows.add(createRemoveFileActionRow(file, physicalSchema, removeTimestamp)); + } + + CloseableIterable dataActions = + CloseableIterable.inMemoryIterable(Utils.toCloseableIterator(removeActionRows.iterator())); + TransactionCommitResult result = transaction.commit(engine, dataActions); + LOG.info( + "Metadata-only delete committed at version {}, removed {} files", + result.getVersion(), + filesToRemove.size()); + } + + // Kernel's extended-metadata RemoveFile builder does not accept tags, so this conversion drops + // them. This conversion code is temporary. + private static Row createRemoveFileActionRow( + AddFile file, StructType physicalSchema, long removeTimestamp) { + MapValue partitionValues = + VectorUtils.stringStringMapValue(toJavaMap(file.partitionValues())); + Optional stats = + Optional.ofNullable(file.stats()) + .flatMap(json -> DataFileStatistics.deserializeFromJson(json, physicalSchema)); + Row removeFileRow = + GenerateIcebergCompatActionUtils.createRemoveFileRowWithExtendedFileMetadata( + file.path(), + removeTimestamp, + /* dataChange = */ true, + partitionValues, + file.size(), + stats, + physicalSchema, + toJavaLong(file.baseRowId()), + toJavaLong(file.defaultRowCommitVersion()), + toKernelDeletionVector(file)); + return SingleAction.createRemoveFileSingleAction(removeFileRow); + } + + private static Optional toKernelDeletionVector(AddFile file) { + return Optional.ofNullable(file.deletionVector()) + .map( + deletionVector -> + new DeletionVectorDescriptor( + deletionVector.storageType(), + deletionVector.pathOrInlineDv(), + toJavaInt(deletionVector.offset()), + deletionVector.sizeInBytes(), + deletionVector.cardinality())); + } + + private static Optional toJavaLong(scala.Option value) { + return value.isDefined() + ? Optional.of(((Number) value.get()).longValue()) + : Optional.empty(); + } + + private static Optional toJavaInt(scala.Option value) { + return value.isDefined() ? Optional.of((Integer) value.get()) : Optional.empty(); + } +} diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/catalog/DeltaV2Table.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/catalog/DeltaV2Table.java index 24fad020fae..e6bdcdb535f 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/catalog/DeltaV2Table.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/catalog/DeltaV2Table.java @@ -17,17 +17,12 @@ import static io.delta.spark.internal.v2.utils.ScalaUtils.toJavaOptional; import static io.delta.spark.internal.v2.utils.ScalaUtils.toScalaMap; +import static io.delta.spark.internal.v2.utils.ScalaUtils.toScalaOption; import static io.delta.spark.internal.v2.utils.StatsUtils.toV2Statistics; import static java.util.Objects.requireNonNull; -import io.delta.kernel.Snapshot; -import io.delta.kernel.defaults.engine.DefaultEngine; import io.delta.kernel.engine.Engine; import io.delta.kernel.internal.DeltaHistoryManager; -import io.delta.kernel.internal.SnapshotImpl; -import io.delta.kernel.internal.rowtracking.RowTracking; -import io.delta.spark.internal.v2.adapters.KernelMetadataAdapter; -import io.delta.spark.internal.v2.adapters.KernelProtocolAdapter; import io.delta.spark.internal.v2.exception.NoRecreatableHistoryException; import io.delta.spark.internal.v2.exception.TableNotFoundException; import io.delta.spark.internal.v2.exception.TimestampOutOfRangeException; @@ -35,9 +30,8 @@ import io.delta.spark.internal.v2.read.MetadataEvolutionHandler; import io.delta.spark.internal.v2.read.cdc.CDCSchemaContext; import io.delta.spark.internal.v2.shims.CatalogV2UtilShims; -import io.delta.spark.internal.v2.snapshot.DeltaSnapshotManager; -import io.delta.spark.internal.v2.snapshot.SnapshotManagerFactory; -import io.delta.spark.internal.v2.utils.SchemaUtils; +import io.delta.spark.internal.v2.tablemanager.DeltaV2TableManager; +import io.delta.spark.internal.v2.tablemanager.DeltaV2TableManagerCache$; import io.delta.spark.internal.v2.write.DeltaRowLevelOperationBuilder; import io.delta.spark.internal.v2.write.DeltaV2WriteBuilder; import java.util.ArrayList; @@ -61,6 +55,7 @@ import org.apache.spark.sql.connector.catalog.MetadataColumn; import org.apache.spark.sql.connector.catalog.SupportsMetadataColumns; import org.apache.spark.sql.connector.catalog.SupportsRead; +import org.apache.spark.sql.connector.catalog.SupportsRowLevelOperations; import org.apache.spark.sql.connector.catalog.SupportsWrite; import org.apache.spark.sql.connector.catalog.Table; import org.apache.spark.sql.connector.catalog.TableCapability; @@ -75,22 +70,34 @@ import org.apache.spark.sql.delta.DeltaTableUtils; import org.apache.spark.sql.delta.RowCommitVersion$; import org.apache.spark.sql.delta.RowId$; +import org.apache.spark.sql.delta.RowTracking$; +import org.apache.spark.sql.delta.Snapshot; import org.apache.spark.sql.delta.catalog.DeltaV2TableMarker; import org.apache.spark.sql.delta.commands.cdc.CDCReader; import org.apache.spark.sql.delta.sources.PersistedMetadata; +import org.apache.spark.sql.delta.util.DeltaFileSystemOptions; import org.apache.spark.sql.delta.v2.interop.AbstractMetadata; import org.apache.spark.sql.delta.v2.interop.AbstractProtocol; +import org.apache.spark.sql.delta.v2.interop.DeltaV2QueryContext; +import org.apache.spark.sql.delta.v2.interop.DeltaV2QueryContext$; +import org.apache.spark.sql.delta.v2.interop.DeltaV2SnapshotManager; import org.apache.spark.sql.execution.datasources.FileFormat$; import org.apache.spark.sql.types.DataType; import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.types.StructType; import org.apache.spark.sql.util.CaseInsensitiveStringMap; +import scala.Option; import scala.jdk.javaapi.CollectionConverters; /** DataSource V2 Table implementation for Delta Lake using the Delta Kernel API. */ -public class DeltaV2Table extends DeltaV2TableShims - implements Table, SupportsRead, SupportsWrite, SupportsMetadataColumns, DeltaV2TableMarker { +public class DeltaV2Table extends DeltaV2TableShimsWithLogging + implements Table, + SupportsRead, + SupportsWrite, + SupportsMetadataColumns, + SupportsRowLevelOperations, + DeltaV2TableMarker { private static final String METADATA_COLUMN_NAME = FileFormat$.MODULE$.METADATA_NAME(); private static final String ROW_ID_METADATA_FIELD_NAME = RowId$.MODULE$.ROW_ID(); private static final String ROW_COMMIT_VERSION_METADATA_FIELD_NAME = @@ -118,7 +125,8 @@ private static Set buildReadOnlyCapabilities() { private final Identifier identifier; private final String tablePath; private final Map options; - private final DeltaSnapshotManager snapshotManager; + private final DeltaV2SnapshotManager snapshotManager; + private final DeltaV2QueryContext queryContext; /** Snapshot created during connector setup */ private final Snapshot initialSnapshot; @@ -226,33 +234,34 @@ private DeltaV2Table( this.identifier = requireNonNull(identifier, "identifier is null"); this.tablePath = requireNonNull(tablePath, "tablePath is null"); this.catalogTable = catalogTable; + this.queryContext = DeltaV2QueryContext$.MODULE$.apply(catalogTable); + Option catalogTableOpt = toScalaOption(catalogTable); // Merge options: file system options from catalog + user options (user takes precedence) // This follows the same pattern as DeltaTableV2 in delta-spark - Map merged = new HashMap<>(); - // Only extract file system options from table storage properties - catalogTable.ifPresent( - table -> - scala.collection.JavaConverters.mapAsJavaMap(table.storage().properties()) - .forEach( - (key, value) -> { - if (DeltaTableUtils.validDeltaTableHadoopPrefixes() - .exists(prefix -> key.startsWith(prefix))) { - merged.put(key, value); - } - })); + Map merged = + new HashMap<>( + scala.collection.JavaConverters.mapAsJavaMap( + DeltaFileSystemOptions.extractCatalogTableFsOptions(catalogTableOpt))); // User options override catalog properties merged.putAll(userOptions); this.options = Collections.unmodifiableMap(merged); - this.hadoopConf = - SparkSession.active().sessionState().newHadoopConfWithOptions(toScalaMap(options)); - this.kernelEngine = DefaultEngine.create(this.hadoopConf); - this.snapshotManager = SnapshotManagerFactory.create(tablePath, kernelEngine, catalogTable); + SparkSession activeSession = SparkSession.active(); + this.hadoopConf = activeSession.sessionState().newHadoopConfWithOptions(toScalaMap(options)); + DeltaV2TableManager tableManager = + DeltaV2TableManagerCache$.MODULE$.forTable( + activeSession, tablePath, options, catalogTableOpt); + this.kernelEngine = tableManager.kernelContext().getDefaultEngine(); + this.snapshotManager = tableManager.snapshotManager(catalogTableOpt); try { - this.initialSnapshot = - timeTravelVersion.isPresent() - ? loadSnapshotAtCheckedVersion(snapshotManager, timeTravelVersion.getAsLong()) - : snapshotManager.loadLatestSnapshot(); + if (timeTravelVersion.isPresent()) { + this.initialSnapshot = + loadSnapshotAtCheckedVersion(snapshotManager, timeTravelVersion.getAsLong()); + } else { + Supplier loadLatestSnapshot = + () -> snapshotManager.loadLatestSnapshot(queryContext); + this.initialSnapshot = recordFrameProfileValue("snapshot.loadLatest", loadLatestSnapshot); + } } catch (io.delta.kernel.exceptions.TableNotFoundException e) { // Rethrow as the Delta-module wrapper so catalog/interop layer never names a Kernel type. throw new TableNotFoundException(tablePath); @@ -270,11 +279,7 @@ private DeltaV2Table( Optional persistedMetadata = MetadataEvolutionHandler.getPersistedMetadataForMicroBatchStream( - SparkSession.active(), - (SnapshotImpl) initialSnapshot, - options, - snapshotManager, - kernelEngine); + SparkSession.active(), initialSnapshot, options, snapshotManager, kernelEngine); StructType rawSchema; List partitionColumnNames; @@ -283,8 +288,8 @@ private DeltaV2Table( rawSchema = persisted.dataSchema(); partitionColumnNames = Arrays.asList(persisted.partitionSchema().fieldNames()); } else { - rawSchema = SchemaUtils.convertKernelSchemaToSparkSchema(initialSnapshot.getSchema()); - partitionColumnNames = new ArrayList<>(initialSnapshot.getPartitionColumnNames()); + rawSchema = initialSnapshot.schema(); + partitionColumnNames = new ArrayList<>(initialSnapshot.metadata().getPartitionColumns()); } // Schema-related metadata is lazily computed on first access within SchemaProvider this.schemaProvider = @@ -350,18 +355,31 @@ public Map getOptions() { * (TableCatalog.loadChangelog) use this to resolve versions, timestamps, and snapshots without * having to build their own snapshot manager. */ - public DeltaSnapshotManager getSnapshotManager() { + public DeltaV2SnapshotManager getSnapshotManager() { return snapshotManager; } /** The table protocol from the initial snapshot. */ protected AbstractProtocol protocol() { - return new KernelProtocolAdapter(((SnapshotImpl) initialSnapshot).getProtocol()); + return initialSnapshot.protocol(); } /** The table metadata from the initial snapshot. */ protected AbstractMetadata metadata() { - return new KernelMetadataAdapter(((SnapshotImpl) initialSnapshot).getMetadata()); + return initialSnapshot.metadata(); + } + + /** Inputs exposed to the Spark-version shim for metadata-only DELETE. */ + protected Engine kernelEngine() { + return kernelEngine; + } + + protected Snapshot initialSnapshot() { + return initialSnapshot; + } + + protected Optional catalogTable() { + return catalogTable; } /** Returns a copy of this table pinned to {@code version}. */ @@ -373,7 +391,7 @@ public DeltaV2Table withVersion(long version) { /** Returns a copy of this table pinned to the snapshot active at {@code timestampMicros}. */ public DeltaV2Table withTimestamp(long timestampMicros) { - return withVersion(resolveTimestampToVersion(snapshotManager, timestampMicros)); + return withVersion(resolveTimestampToVersion(snapshotManager, timestampMicros, queryContext)); } /** @@ -384,15 +402,16 @@ public DeltaV2Table withTimestamp(long timestampMicros) { * share a singular load once the snapshot manager exposes it TODO(#5999). */ private static long resolveTimestampToVersion( - DeltaSnapshotManager manager, long timestampMicros) { + DeltaV2SnapshotManager manager, long timestampMicros, DeltaV2QueryContext queryContext) { long timeMillis = timestampMicros / 1000; DeltaHistoryManager.Commit commit = manager.getActiveCommitAtTime( timeMillis, /* canReturnLastCommit = */ true, /* mustBeRecreatable = */ true, - /* canReturnEarliestCommit = */ true); - long latestVersion = manager.loadLatestSnapshot().getVersion(); + /* canReturnEarliestCommit = */ true, + queryContext); + long latestVersion = manager.loadLatestSnapshot(queryContext).version(); if (commit.getTimestamp() > timeMillis) { // The earliest available commit is younger than the requested time. throw new TimestampOutOfRangeException(timeMillis, commit.getTimestamp(), false); @@ -437,7 +456,7 @@ public Transform[] partitioning() { @Override public Map properties() { - Map props = new HashMap<>(initialSnapshot.getTableProperties()); + Map props = new HashMap<>(initialSnapshot.metadata().getConfiguration()); return Collections.unmodifiableMap(props); } @@ -454,7 +473,7 @@ public Set capabilities() { *

Add {@code @Override} annotation eventually: TODO(#7128) */ public String version() { - return Long.toString(initialSnapshot.getVersion()); + return Long.toString(initialSnapshot.version()); } /** @@ -474,9 +493,8 @@ public String version() { */ @Override public MetadataColumn[] metadataColumns() { - SnapshotImpl snapshotImpl = (SnapshotImpl) initialSnapshot; boolean rowTrackingEnabled = - RowTracking.isEnabled(snapshotImpl.getProtocol(), snapshotImpl.getMetadata()); + RowTracking$.MODULE$.isEnabled(initialSnapshot.protocol(), initialSnapshot.metadata()); StructType metadataType = new StructType(); for (StructField field : @@ -530,6 +548,8 @@ public ScanBuilder newScanBuilder(CaseInsensitiveStringMap scanOptions) { return DeltaV2ScanUtils.newScanBuilder( name(), initialSnapshot, + kernelEngine, + catalogTable, snapshotManager, schemaProvider.getDataSchema(), schemaProvider.getPartitionSchema(), @@ -541,10 +561,12 @@ public ScanBuilder newScanBuilder(CaseInsensitiveStringMap scanOptions) { /** * Validates that {@code version} exists in the Delta log, then loads the snapshot pinned to it. */ - private static Snapshot loadSnapshotAtCheckedVersion(DeltaSnapshotManager manager, long version) { - manager.checkVersionExists( - version, /* mustBeRecreatable = */ true, /* allowOutOfRange = */ false); - return manager.loadSnapshotAt(version); + private Snapshot loadSnapshotAtCheckedVersion(DeltaV2SnapshotManager manager, long version) { + boolean mustBeRecreatable = true; + boolean allowOutOfRange = false; + manager.checkVersionExists(version, mustBeRecreatable, allowOutOfRange, queryContext); + final Supplier loadSnapshot = () -> manager.loadSnapshotAt(version, queryContext); + return recordFrameProfileValue("snapshot.loadAtVersion", loadSnapshot); } @Override @@ -566,6 +588,7 @@ public WriteBuilder newWriteBuilder(LogicalWriteInfo info) { * Delta's copy-on-write operation shell; the concrete ReplaceData write path is introduced in a * follow-up PR. */ + @Override public RowLevelOperationBuilder newRowLevelOperationBuilder(RowLevelOperationInfo info) { requireNonNull(info, "row-level operation info is null"); return new DeltaRowLevelOperationBuilder(this, kernelEngine, hadoopConf, initialSnapshot, info); @@ -589,8 +612,11 @@ public boolean equals(Object o) { && Objects.equals(tablePath, that.tablePath) && Objects.equals(options, that.options) && Objects.equals(catalogTable, that.catalogTable) - && Objects.equals(initialSnapshot.getPath(), that.initialSnapshot.getPath()) - && initialSnapshot.getVersion() == that.initialSnapshot.getVersion(); + && Objects.equals( + initialSnapshot.dataPath().toString(), that.initialSnapshot.dataPath().toString()) + && initialSnapshot.version() == that.initialSnapshot.version() + && Objects.equals( + initialSnapshot.metadata().getId(), that.initialSnapshot.metadata().getId()); } @Override @@ -600,8 +626,9 @@ public int hashCode() { tablePath, options, catalogTable, - initialSnapshot.getPath(), - initialSnapshot.getVersion()); + initialSnapshot.dataPath().toString(), + initialSnapshot.version(), + initialSnapshot.metadata().getId()); } /** diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/ddl/CreateTableBuilder.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/ddl/CreateTableBuilder.java index ef720c0b466..7d6be940837 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/ddl/CreateTableBuilder.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/ddl/CreateTableBuilder.java @@ -19,11 +19,11 @@ import io.delta.kernel.TableManager; import io.delta.kernel.Transaction; -import io.delta.kernel.defaults.engine.DefaultEngine; import io.delta.kernel.transaction.CreateTableTransactionBuilder; import io.delta.kernel.transaction.DataLayoutSpec; import io.delta.kernel.unitycatalog.UCCatalogManagedClient; import io.delta.kernel.unitycatalog.UCTableIdentifier; +import io.delta.spark.internal.v2.kernel.KernelEngineFactory; import io.delta.spark.internal.v2.snapshot.unitycatalog.UCTableInfo; import io.delta.spark.internal.v2.utils.SchemaUtils; import io.delta.storage.commit.uccommitcoordinator.UCClient; @@ -89,7 +89,7 @@ public static DDLRequestContext prepare( DDLUtils.filterProperties(properties), comment, dataLayoutSpec, - DefaultEngine.create(hadoopConf), + KernelEngineFactory.createDefaultEngine(hadoopConf), ucTableInfo, txnBuilder); } diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/DeltaScanFile.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/DeltaScanFile.java deleted file mode 100644 index a7cc422fa95..00000000000 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/DeltaScanFile.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright (2025) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.delta.spark.internal.v2.read; - -import io.delta.kernel.data.MapValue; -import io.delta.kernel.internal.actions.AddFile; -import io.delta.kernel.internal.actions.DeletionVectorDescriptor; -import java.util.Objects; -import java.util.Optional; - -/** Narrow descriptor for a Delta file selected by a DSv2 scan. */ -public final class DeltaScanFile { - - private final String path; - private final MapValue partitionValues; - private final long size; - private final long modificationTime; - private final Optional baseRowId; - private final Optional defaultRowCommitVersion; - private final Optional deletionVector; - - DeltaScanFile(AddFile addFile) { - this.path = Objects.requireNonNull(addFile, "addFile is null").getPath(); - this.partitionValues = addFile.getPartitionValues(); - this.size = addFile.getSize(); - this.modificationTime = addFile.getModificationTime(); - this.baseRowId = addFile.getBaseRowId(); - this.defaultRowCommitVersion = addFile.getDefaultRowCommitVersion(); - this.deletionVector = addFile.getDeletionVector(); - } - - public String getPath() { - return path; - } - - public MapValue getPartitionValues() { - return partitionValues; - } - - public long getSize() { - return size; - } - - public long getModificationTime() { - return modificationTime; - } - - public Optional getBaseRowId() { - return baseRowId; - } - - public Optional getDefaultRowCommitVersion() { - return defaultRowCommitVersion; - } - - public Optional getDeletionVector() { - return deletionVector; - } -} diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/DeltaV2Batch.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/DeltaV2Batch.java deleted file mode 100644 index f9ef4443680..00000000000 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/DeltaV2Batch.java +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Copyright (2025) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.delta.spark.internal.v2.read; - -import io.delta.kernel.Snapshot; -import io.delta.kernel.expressions.Predicate; -import io.delta.spark.internal.v2.utils.PartitionUtils; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Objects; -import java.util.Set; -import org.apache.hadoop.conf.Configuration; -import org.apache.spark.sql.SparkSession; -import org.apache.spark.sql.connector.read.Batch; -import org.apache.spark.sql.connector.read.InputPartition; -import org.apache.spark.sql.connector.read.PartitionReaderFactory; -import org.apache.spark.sql.execution.datasources.PartitionedFile; -import org.apache.spark.sql.internal.SQLConf; -import org.apache.spark.sql.sources.Filter; -import org.apache.spark.sql.types.StructType; - -/** - * Package-private batch implementation for Delta's Spark DataSource V2 read path. - * - *

This class must remain package-private so callers outside {@code v2.read} depend only on - * Spark's public connector interfaces instead of coupling to Delta's internal V2 implementation. - */ -class DeltaV2Batch implements Batch { - private final Snapshot snapshot; - private final StructType readDataSchema; - private final StructType dataSchema; - private final StructType partitionSchema; - private final StructType ddlOrderedReadOutputSchema; - private final Predicate[] pushedToKernelFilters; - private final Filter[] dataFilters; - // Derived Sets used only for equals/hashCode: filters are AND-ed at eval time, - // so list order has no semantic meaning. - private final Set pushedToKernelFiltersSet; - private final Set dataFiltersSet; - private final Configuration hadoopConf; - private final SQLConf sqlConf; - private final long totalBytes; - private scala.collection.immutable.Map scalaOptions; - private final List partitionedFiles; - - public DeltaV2Batch( - Snapshot snapshot, - StructType dataSchema, - StructType partitionSchema, - StructType readDataSchema, - StructType ddlOrderedReadOutputSchema, - List partitionedFiles, - Predicate[] pushedToKernelFilters, - Filter[] dataFilters, - long totalBytes, - scala.collection.immutable.Map scalaOptions, - Configuration hadoopConf) { - - this.snapshot = Objects.requireNonNull(snapshot, "snapshot is null"); - this.dataSchema = Objects.requireNonNull(dataSchema, "dataSchema is null"); - this.partitionSchema = Objects.requireNonNull(partitionSchema, "partitionSchema is null"); - this.readDataSchema = Objects.requireNonNull(readDataSchema, "readDataSchema is null"); - this.ddlOrderedReadOutputSchema = - Objects.requireNonNull(ddlOrderedReadOutputSchema, "ddlOrderedReadOutputSchema is null"); - this.partitionedFiles = - java.util.Collections.unmodifiableList( - new ArrayList<>(Objects.requireNonNull(partitionedFiles, "partitionedFiles is null"))); - this.pushedToKernelFilters = - pushedToKernelFilters == null ? new Predicate[0] : pushedToKernelFilters.clone(); - this.dataFilters = dataFilters == null ? new Filter[0] : dataFilters.clone(); - this.pushedToKernelFiltersSet = Set.copyOf(Arrays.asList(this.pushedToKernelFilters)); - this.dataFiltersSet = Set.copyOf(Arrays.asList(this.dataFilters)); - this.totalBytes = totalBytes; - this.scalaOptions = Objects.requireNonNull(scalaOptions, "scalaOptions is null"); - this.hadoopConf = Objects.requireNonNull(hadoopConf, "hadoopConf is null"); - this.sqlConf = SQLConf.get(); - } - - @Override - public InputPartition[] planInputPartitions() { - return PartitionUtils.planInputPartitions( - SparkSession.active(), partitionedFiles, totalBytes, hadoopConf, sqlConf); - } - - @Override - public PartitionReaderFactory createReaderFactory() { - // Non-CDC plain table scan. Write-time CDF streaming reads route through - // DeltaV2MicroBatchStream; read-time CDF batch reads route through DeltaChangelogBatch. - return PartitionUtils.createDeltaParquetReaderFactory( - snapshot, - dataSchema, - partitionSchema, - readDataSchema, - ddlOrderedReadOutputSchema, - dataFilters, - scalaOptions, - hadoopConf, - sqlConf); - } - - @Override - public boolean equals(Object obj) { - if (this == obj) return true; - if (!(obj instanceof DeltaV2Batch)) return false; - - DeltaV2Batch that = (DeltaV2Batch) obj; - return Objects.equals(this.snapshot, that.snapshot) - && Objects.equals(this.readDataSchema, that.readDataSchema) - && Objects.equals(this.dataSchema, that.dataSchema) - && Objects.equals(this.partitionSchema, that.partitionSchema) - && Objects.equals(this.pushedToKernelFiltersSet, that.pushedToKernelFiltersSet) - && Objects.equals(this.dataFiltersSet, that.dataFiltersSet) - && partitionedFiles.size() == that.partitionedFiles.size(); - } - - @Override - public int hashCode() { - return Objects.hash( - snapshot, - readDataSchema, - dataSchema, - partitionSchema, - pushedToKernelFiltersSet, - dataFiltersSet, - partitionedFiles.size()); - } -} diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/DeltaV2MicroBatchStream.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/DeltaV2MicroBatchStream.java index 3aaff700cb8..a1e339e6ebb 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/DeltaV2MicroBatchStream.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/DeltaV2MicroBatchStream.java @@ -22,11 +22,7 @@ import com.google.common.collect.ImmutableSet; import io.delta.kernel.CommitActions; import io.delta.kernel.CommitRange; -import io.delta.kernel.Scan; -import io.delta.kernel.Snapshot; import io.delta.kernel.data.ColumnarBatch; -import io.delta.kernel.data.FilteredColumnarBatch; -import io.delta.kernel.defaults.engine.DefaultEngine; import io.delta.kernel.engine.Engine; import io.delta.kernel.exceptions.UnsupportedProtocolVersionException; import io.delta.kernel.exceptions.UnsupportedTableFeatureException; @@ -45,12 +41,10 @@ import io.delta.kernel.internal.util.VectorUtils; import io.delta.kernel.utils.CloseableIterator; import io.delta.spark.internal.v2.adapters.KernelMetadataAdapter; -import io.delta.spark.internal.v2.snapshot.DeltaSnapshotManager; +import io.delta.spark.internal.v2.kernel.KernelEngineFactory; import io.delta.spark.internal.v2.utils.PartitionUtils; import io.delta.spark.internal.v2.utils.ScalaUtils; import io.delta.spark.internal.v2.utils.SchemaUtils; -import io.delta.spark.internal.v2.utils.SerializableReadOnlySnapshot; -import io.delta.spark.internal.v2.utils.SparkRowToKernelRow; import io.delta.spark.internal.v2.utils.StreamingHelper; import java.io.IOException; import java.io.UncheckedIOException; @@ -58,14 +52,9 @@ import java.sql.Timestamp; import java.time.ZoneId; import java.util.*; -import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import javax.annotation.Nullable; import org.apache.hadoop.conf.Configuration; -import org.apache.spark.api.java.JavaRDD; -import org.apache.spark.sql.Dataset; -import org.apache.spark.sql.Row; -import org.apache.spark.sql.RowFactory; import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.catalyst.expressions.Literal$; import org.apache.spark.sql.connector.read.InputPartition; @@ -76,9 +65,11 @@ import org.apache.spark.sql.delta.DeltaOptions; import org.apache.spark.sql.delta.DeltaStartingVersion; import org.apache.spark.sql.delta.DeltaTimeTravelSpec; +import org.apache.spark.sql.delta.Snapshot; import org.apache.spark.sql.delta.StartingVersion; import org.apache.spark.sql.delta.StartingVersionLatest$; import org.apache.spark.sql.delta.TypeWidening; +import org.apache.spark.sql.delta.files.DeltaSourceSnapshot; import org.apache.spark.sql.delta.sources.AdmittableFile; import org.apache.spark.sql.delta.sources.DeltaSQLConf; import org.apache.spark.sql.delta.sources.DeltaSource; @@ -88,21 +79,20 @@ import org.apache.spark.sql.delta.sources.DeltaStreamUtils; import org.apache.spark.sql.delta.sources.PersistedMetadata; import org.apache.spark.sql.delta.v2.CDCDeletionVectorHelper; +import org.apache.spark.sql.delta.v2.interop.DeltaV2Snapshot$; +import org.apache.spark.sql.delta.v2.interop.DeltaV2SnapshotManager; +import org.apache.spark.sql.delta.v2.kernel.KernelActionUtils$; import org.apache.spark.sql.execution.datasources.PartitionedFile; -import org.apache.spark.sql.functions; import org.apache.spark.sql.internal.SQLConf; import org.apache.spark.sql.sources.Filter; import org.apache.spark.sql.types.DataType; -import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.types.StructType; -import org.apache.spark.storage.StorageLevel; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import scala.Function2; import scala.Option; import scala.Some; -import scala.Tuple2; import scala.collection.immutable.Seq; import scala.collection.immutable.Seq$; import scala.jdk.javaapi.CollectionConverters; @@ -121,8 +111,6 @@ class DeltaV2MicroBatchStream private static final Logger logger = LoggerFactory.getLogger(DeltaV2MicroBatchStream.class); - private static final String FILE_IDX_COL = "_file_idx"; - public static final Set ACTION_SET = ImmutableSet.of( DeltaAction.ADD, @@ -145,7 +133,7 @@ class DeltaV2MicroBatchStream private static final int FAIL_ON_DATA_LOSS_FALSE_MAX_ATTEMPTS = 3; private final Engine engine; - private final DeltaSnapshotManager snapshotManager; + private final DeltaV2SnapshotManager snapshotManager; private final DeltaOptions options; private final boolean ignoreFileDeletion; private final boolean skipChangeCommits; @@ -169,13 +157,16 @@ class DeltaV2MicroBatchStream /** Handler for non-additive schema evolution (rename/drop/type widening). */ private final MetadataEvolutionHandler metadataEvolutionHandler; + // Spark can call initialOffset multiple times while constructing and executing the first batch. + private DeltaSourceOffset cachedInitialOffset; + /** * Tracks whether this is the first batch for this stream (no checkpointed offset). * *

- First batch: initialOffset() -> latestOffset(Offset, ReadLimit) - Set `isFirstBatch` to * true in initialOffset() - in latestOffset(Offset, ReadLimit), use `isFirstBatch` to determine - * whether to return null vs previousOffset (when no data is available) - set `isFirstBatch` to - * false - Subsequent batches: latestOffset(Offset, ReadLimit) + * whether to return null vs previousOffset (when no data is available) - keep `isFirstBatch` true + * until a non-null offset is returned - Subsequent batches: latestOffset(Offset, ReadLimit) */ private boolean isFirstBatch = false; @@ -209,31 +200,12 @@ class DeltaV2MicroBatchStream // getStartingVersion() must return the same value across multiple calls. private volatile Optional cachedStartingVersion = null; - // Cache for the initial snapshot files to avoid re-sorting on repeated access. - private static class InitialSnapshotCache { - final Long version; - final List files; - final long commitTimestamp; - - InitialSnapshotCache(Long version, List files, long commitTimestamp) { - this.version = version; - this.files = files; - this.commitTimestamp = commitTimestamp; - } - } - - private final AtomicReference cachedInitialSnapshot = - new AtomicReference<>(null); - - private final AtomicReference cachedDataFrameSnapshot = - new AtomicReference<>(null); - - private final int maxInitialSnapshotFiles; - private final boolean useDistributedInitialSnapshot; - private final StorageLevel snapshotCacheStorageLevel; + // A metadata snapshot when starting the query. + private DeltaSourceSnapshot initialSnapshot = null; + private long initialSnapshotVersion = -1L; public DeltaV2MicroBatchStream( - DeltaSnapshotManager snapshotManager, + DeltaV2SnapshotManager snapshotManager, Snapshot snapshotAtSourceInit, Configuration hadoopConf, SparkSession spark, @@ -250,7 +222,7 @@ public DeltaV2MicroBatchStream( this.snapshotManager = Objects.requireNonNull(snapshotManager, "snapshotManager is null"); this.hadoopConf = Objects.requireNonNull(hadoopConf, "hadoopConf is null"); this.spark = Objects.requireNonNull(spark, "spark is null"); - this.engine = DefaultEngine.create(hadoopConf); + this.engine = KernelEngineFactory.createDefaultEngine(hadoopConf); this.options = Objects.requireNonNull(options, "options is null"); this.ignoreFileDeletion = this.options.ignoreFileDeletion(); // Deprecated. Please use `skipChangeCommits` from now on. @@ -273,7 +245,7 @@ public DeltaV2MicroBatchStream( this.sqlConf = SQLConf.get(); this.scalaOptions = Objects.requireNonNull(scalaOptions, "scalaOptions is null"); - this.snapshotAtSourceInit = (SnapshotImpl) snapshotAtSourceInit; + this.snapshotAtSourceInit = DeltaV2Snapshot$.MODULE$.getKernelSnapshot(snapshotAtSourceInit); this.tableId = this.snapshotAtSourceInit.getMetadata().getId(); // The effective snapshot for reading, mirroring v1's readSnapshotDescriptor. When schema @@ -304,25 +276,6 @@ public DeltaV2MicroBatchStream( spark.sessionState().conf().getConf(DeltaSQLConf.STREAMING_OFFSET_VALIDATION()), "shouldValidateOffsets is null"); this.excludeRegex = ScalaUtils.toJavaOptional(options.excludeRegex()); - this.maxInitialSnapshotFiles = - (Integer) - spark - .sessionState() - .conf() - .getConf(DeltaSQLConf.DELTA_STREAMING_INITIAL_SNAPSHOT_MAX_FILES()); - this.useDistributedInitialSnapshot = - (Boolean) - spark - .sessionState() - .conf() - .getConf(DeltaSQLConf.DELTA_STREAMING_USE_DISTRIBUTED_INITIAL_SNAPSHOT()); - this.snapshotCacheStorageLevel = - StorageLevel.fromString( - (String) - spark - .sessionState() - .conf() - .getConf(DeltaSQLConf.DELTA_SNAPSHOT_CACHE_STORAGE_LEVEL())); boolean isStreamingFromColumnMappingTable = ColumnMapping.getColumnMappingMode( @@ -383,64 +336,14 @@ private void initForTriggerAvailableNowIfNeeded(DeltaSourceOffset startOffsetOpt private void initLastOffsetForTriggerAvailableNow(DeltaSourceOffset startOffsetOpt) { // TODO(#6817): Explore unifying TAN offset computation into latestOffsetInternal. // Blocked: latestOffset per-batch shares the same conditions but must NOT bypass rate limiting. - if (useDistributedInitialSnapshot && startOffsetOpt.isInitialSnapshot()) { - lastOffsetForTriggerAvailableNow = getLastOffsetForAvailableNowViaDataFrame(startOffsetOpt); - } else { - lastOffsetForTriggerAvailableNow = - latestOffsetInternal(startOffsetOpt, ReadLimit.allAvailable()); - } + lastOffsetForTriggerAvailableNow = + latestOffsetInternal(startOffsetOpt, ReadLimit.allAvailable()); lastOffsetForTriggerAvailableNow.ifPresent( lastOffset -> logger.info("lastOffset for Trigger.AvailableNow has set to " + lastOffset.json())); } - /** - * Computes the end offset for Trigger.AvailableNow without iterating all snapshot files. Avoids - * O(n) driver memory pressure that causes OOM for large tables. - */ - private Optional getLastOffsetForAvailableNowViaDataFrame( - DeltaSourceOffset startOffset) { - long snapshotVersion = startOffset.reservoirVersion(); - - checkReadIncompatibleSchemaChangeOnStreamStartOnce( - snapshotVersion, /* batchEndVersion= */ null); - - // iteratorLast exhausts the iterator and closes it — do not wrap in try-with-resources - // (that would double-close). Use a plain try block for exception context. - try { - CloseableIterator deltaChanges = - filterDeltaLogs(snapshotVersion + 1, /* endOffset= */ Optional.empty()); - Optional lastDelta = Utils.iteratorLast(deltaChanges); - - if (lastDelta.isPresent()) { - IndexedFile lastFile = lastDelta.get(); - return Optional.of( - DeltaSource.buildOffsetFromIndexedFile( - tableId, - lastFile.getVersion(), - lastFile.getIndex(), - startOffset.reservoirVersion(), - startOffset.isInitialSnapshot())); - } - } catch (UncheckedIOException e) { - throw new RuntimeException( - String.format( - "Failed to iterate delta changes for table %s after version %d", - tablePath, snapshotVersion), - e); - } - - // No delta changes — END_INDEX bumps offset to (snapshotVersion + 1, BASE_INDEX, false). - return Optional.of( - DeltaSource.buildOffsetFromIndexedFile( - tableId, - snapshotVersion, - DeltaSourceOffset.END_INDEX(), - startOffset.reservoirVersion(), - startOffset.isInitialSnapshot())); - } - //////////// // offset // //////////// @@ -451,6 +354,10 @@ private Optional getLastOffsetForAvailableNowViaDataFrame( */ @Override public Offset initialOffset() { + if (cachedInitialOffset != null) { + return cachedInitialOffset; + } + Optional startingVersionOpt = getStartingVersion(); long version; boolean isInitialSnapshot; @@ -466,8 +373,10 @@ public Offset initialOffset() { isInitialSnapshot = true; } - return DeltaSourceOffset.apply( - tableId, version, DeltaSourceOffset.BASE_INDEX(), isInitialSnapshot); + cachedInitialOffset = + DeltaSourceOffset.apply( + tableId, version, DeltaSourceOffset.BASE_INDEX(), isInitialSnapshot); + return cachedInitialOffset; } @Override @@ -493,7 +402,9 @@ public Offset latestOffset(Offset startOffset, ReadLimit limit) { initForTriggerAvailableNowIfNeeded(deltaStartOffset); // Return null when no data is available for this batch. DeltaSourceOffset endOffset = latestOffsetInternal(deltaStartOffset, limit).orElse(null); - isFirstBatch = false; + if (endOffset != null) { + isFirstBatch = false; + } return endOffset; } catch (Exception e) { // Kernel's DefaultJsonHandler wraps ClosedByInterruptException (thrown by NIO @@ -732,19 +643,23 @@ public void commit(Offset end) { // IMPORTANT: for future developers, please place any work you would like to do in commit() // before updateMetadataTrackingLogAndFailTheStreamIfNeeded as it may throw an exception. DeltaSourceOffset offset = DeltaSourceOffset.apply(tableId, end); + if (!offset.isInitialSnapshot()) { + // Release snapshot resources after initial snapshot processing has completed. + cleanUpSnapshotResources(); + } metadataEvolutionHandler.updateMetadataTrackingLogAndFailTheStreamIfNeeded(offset); } @Override public void stop() { - cachedInitialSnapshot.set(null); - invalidateDataFrameCache(); + cleanUpSnapshotResources(); } - private void invalidateDataFrameCache() { - DataFrameSnapshotCache prev = cachedDataFrameSnapshot.getAndSet(null); - if (prev != null) { - prev.close(); + private void cleanUpSnapshotResources() { + if (initialSnapshot != null) { + initialSnapshot.close( + /* unpersistSnapshot= */ initialSnapshotVersion < snapshotAtSourceInit.getVersion()); + initialSnapshot = null; } } @@ -809,7 +724,7 @@ synchronized Optional getStartingVersion() { if (startingVersion instanceof StartingVersionLatest$) { Snapshot latestSnapshot = snapshotManager.loadLatestSnapshot(); // "latest": start reading from the next commit - cachedStartingVersion = Optional.of(latestSnapshot.getVersion() + 1); + cachedStartingVersion = Optional.of(latestSnapshot.version() + 1); return cachedStartingVersion; } else if (startingVersion instanceof StartingVersion) { long version = ((StartingVersion) startingVersion).version(); @@ -848,13 +763,13 @@ synchronized Optional getStartingVersion() { * Returns the earliest commit version whose timestamp is >= the provided timestamp. * *

This method fetches the commit at the given timestamp via - * [[DeltaSnapshotManager.getActiveCommitAtTime]], computes the starting version using + * [[DeltaV2SnapshotManager.getActiveCommitAtTime]], computes the starting version using * [[DeltaStreamUtils.getStartingVersionFromCommitAtTimestamp]], and validates the protocol at the * returned version. */ private static long getStartingVersionFromTimestamp( SparkSession spark, - DeltaSnapshotManager snapshotManager, + DeltaV2SnapshotManager snapshotManager, Engine engine, Timestamp timestamp, boolean canExceedLatest) { @@ -865,7 +780,7 @@ private static long getStartingVersionFromTimestamp( /* canReturnLastCommit= */ true, /* mustBeRecreatable= */ false, /* canReturnEarliestCommit= */ true); - long latestVersion = snapshotManager.loadLatestSnapshot().getVersion(); + long latestVersion = snapshotManager.loadLatestSnapshot().version(); long startingVersion = DeltaStreamUtils.getStartingVersionFromCommitAtTimestamp( /* timeZone= */ spark.sessionState().conf().sessionLocalTimeZone(), @@ -891,7 +806,7 @@ private static long getStartingVersionFromTimestamp( *

Returns true when the validation was performed and succeeded. */ private static boolean validateProtocolAt( - SparkSession spark, DeltaSnapshotManager snapshotManager, Engine engine, long version) { + SparkSession spark, DeltaV2SnapshotManager snapshotManager, Engine engine, long version) { boolean alwaysValidateProtocol = (Boolean) spark @@ -982,19 +897,10 @@ CloseableIterator getFileChanges( if (isInitialSnapshot) { // Lazily combine snapshot files with delta logs starting from fromVersion + 1. // filterDeltaLogs handles the case when no commits exist after fromVersion. - // TODO: Unify behind SnapshotFileSource interface; extend distributed path to CDC. - CloseableIterator snapshotFiles; - if (useDistributedInitialSnapshot) { - snapshotFiles = getSnapshotFilesViaDataFrame(fromVersion, fromIndex); - } else { - InitialSnapshotCache snapshot = getSnapshotFiles(fromVersion); - snapshotFiles = Utils.toCloseableIterator(snapshot.files.iterator()); - } + CloseableIterator snapshotFiles = getSnapshotAt(fromVersion); CloseableIterator deltaChanges = filterDeltaLogs(fromVersion + 1, endOffset); result = snapshotFiles.combine(deltaChanges); } else { - // Release cached DataFrame — initial snapshot processing complete. - invalidateDataFrameCache(); result = filterDeltaLogs(fromVersion, endOffset); } @@ -1021,10 +927,8 @@ CloseableIterator getFileChangesWithRateLimitForCDC( validateCDFEnabledOnTable(fromVersion); CloseableIterator result; if (isInitialSnapshot) { - InitialSnapshotCache snapshot = getSnapshotFiles(fromVersion); - long commitTimestamp = snapshot.commitTimestamp; - CloseableIterator snapshotFiles = - Utils.toCloseableIterator(snapshot.files.iterator()); + CloseableIterator snapshotFiles = getSnapshotAt(fromVersion); + long commitTimestamp = initialSnapshot.snapshot().timestamp(); snapshotFiles = snapshotFiles.map( file -> @@ -1130,7 +1034,9 @@ void validateCDFEnabledOnTable(long startVersion) { startSnapshot = snapshotAtSourceInit; } else { try { - startSnapshot = (SnapshotImpl) snapshotManager.loadSnapshotAt(startVersion); + startSnapshot = + DeltaV2Snapshot$.MODULE$.getKernelSnapshot( + snapshotManager.loadSnapshotAt(startVersion)); } catch (io.delta.kernel.exceptions.KernelException e) { // startVersion may not yet exist (e.g. startingVersion=latest resolves to latest+1). // TODO(#6745): narrow this catch once kernel exposes a specific exception subclass @@ -1192,7 +1098,7 @@ private CloseableIterator getCommitsFromRange( // 2. buildOffsetFromIndexedFile bumps the version up by one when we hit the END_INDEX. // TODO(#5318): consider caching the latest version to avoid loading a new snapshot. // TODO(#5318): kernel should ideally relax this constraint. - endVersionOpt = Optional.of(snapshotManager.loadLatestSnapshot().getVersion()); + endVersionOpt = Optional.of(snapshotManager.loadLatestSnapshot().version()); } // After capping, check if startVersion is beyond the endVersion. @@ -1205,7 +1111,7 @@ private CloseableIterator getCommitsFromRange( // When endOffset is empty (offset discovery), check if startVersion exceeds the current // latest version. We must load the current latest (not snapshotAtSourceInit) because new // commits may have arrived since stream initialization. - long currentLatestVersion = snapshotManager.loadLatestSnapshot().getVersion(); + long currentLatestVersion = snapshotManager.loadLatestSnapshot().version(); if (startVersion > currentLatestVersion) { return Utils.toCloseableIterator(Collections.emptyIterator()); } @@ -1698,7 +1604,9 @@ private void checkReadIncompatibleSchemaChangeOnStreamStartOnce( SnapshotImpl startVersionSnapshot = null; Exception err = null; try { - startVersionSnapshot = (SnapshotImpl) snapshotManager.loadSnapshotAt(batchStartVersion); + startVersionSnapshot = + DeltaV2Snapshot$.MODULE$.getKernelSnapshot( + snapshotManager.loadSnapshotAt(batchStartVersion)); } catch (Exception e) { err = e; } @@ -1826,198 +1734,72 @@ private long addIndexedFilesAndReturnNextIndex( } /** - * Get all files from a snapshot at the specified version, sorted by modificationTime and path, - * with indices assigned sequentially, and wrapped with BEGIN/END sentinels. - * - *

Mimics DeltaSourceSnapshot in DSv1. - * - * @param version The snapshot version to read - * @return The cached or newly loaded initial snapshot + * Adds dummy BEGIN_INDEX and END_INDEX IndexedFiles for {@code version} before and after the + * contents of the iterator. The contents of the iterator must be the IndexedFiles that correspond + * to this version. */ - private InitialSnapshotCache getSnapshotFiles(long version) { - InitialSnapshotCache cache = cachedInitialSnapshot.get(); - - if (cache != null && cache.version != null && cache.version == version) { - return cache; - } - - InitialSnapshotCache newCache = loadAndValidateSnapshot(version); - cachedInitialSnapshot.set(newCache); - - return newCache; + private static CloseableIterator addBeginAndEndIndexOffsetsForVersion( + long version, CloseableIterator files) { + return Utils.singletonCloseableIterator( + IndexedFile.sentinel(version, DeltaSourceOffset.BASE_INDEX())) + .combine(files) + .combine( + Utils.singletonCloseableIterator( + IndexedFile.sentinel(version, DeltaSourceOffset.END_INDEX()))); } - private CloseableIterator getSnapshotFilesViaDataFrame( - long version, long fromIndex) { - DataFrameSnapshotCache dfCache = cachedDataFrameSnapshot.get(); - if (dfCache == null || dfCache.getVersion() != version) { - invalidateDataFrameCache(); - dfCache = buildDataFrameSnapshotCache(version); - cachedDataFrameSnapshot.set(dfCache); - } - - Dataset df = dfCache.getSortedAddFiles(); - - // Push filtering to executors: O(remaining) per batch instead of O(N) full-scan. - if (fromIndex > DeltaSourceOffset.BASE_INDEX()) { - df = df.where(functions.col(FILE_IDX_COL).gt(fromIndex)); - } - - return dataFrameToIndexedFiles(df, version); - } - - /** Builds a new persisted DataFrame cache for the initial snapshot at the given version. */ - private DataFrameSnapshotCache buildDataFrameSnapshotCache(long version) { - // May differ from snapshotAtSourceInit on checkpoint restart. loadSnapshotAt is - // metadata-only on driver; log replay runs on executors via ScanFileRDD. - SnapshotImpl snapshot = (SnapshotImpl) snapshotManager.loadSnapshotAt(version); - SerializableReadOnlySnapshot serSnapshot = - SerializableReadOnlySnapshot.fromSnapshot(snapshot, hadoopConf); - - ScanFileRDD rdd = new ScanFileRDD(spark.sparkContext(), serSnapshot); - Dataset sorted = - spark.createDataFrame(rdd, ScanFileRDD.SPARK_SCHEMA).orderBy("modificationTime", "path"); - - // zipWithIndex for contiguous 0-based indices — monotonically_increasing_id() is - // non-contiguous across partitions and breaks checkpoint compatibility. - int numFields = ScanFileRDD.SPARK_SCHEMA.size(); - StructType schemaWithIdx = - ScanFileRDD.SPARK_SCHEMA.add(FILE_IDX_COL, DataTypes.LongType, false); - JavaRDD indexedRDD = - sorted - .javaRDD() - .zipWithIndex() - .map( - (Tuple2 tuple) -> { - Row row = tuple._1(); - long idx = tuple._2(); - Object[] fields = new Object[numFields + 1]; - for (int i = 0; i < numFields; i++) { - fields[i] = row.get(i); - } - fields[numFields] = idx; - return RowFactory.create(fields); - }); - - Dataset df = - spark.createDataFrame(indexedRDD, schemaWithIdx).persist(snapshotCacheStorageLevel); - - return new DataFrameSnapshotCache(version, df); - } - - /** Converts sorted AddFile DataFrame to lazy IndexedFile iterator with BEGIN/END sentinels. */ - private static CloseableIterator dataFrameToIndexedFiles( - Dataset df, long version) { - - int fileIdxOrdinal = df.schema().fieldIndex(FILE_IDX_COL); - java.util.Iterator localIter = df.toLocalIterator(); - - return new CloseableIterator() { - private boolean sentBegin = false; - private boolean sentEnd = false; - - @Override - public boolean hasNext() { - return !sentEnd; - } - - @Override - public IndexedFile next() { - if (!sentBegin) { - sentBegin = true; - return IndexedFile.sentinel(version, DeltaSourceOffset.BASE_INDEX()); - } - - if (localIter.hasNext()) { - Row sparkRow = localIter.next(); - long fileIdx = sparkRow.getLong(fileIdxOrdinal); - // Extra _file_idx column at ordinal N is safely ignored (ordinal-based access up to - // schema size). - io.delta.kernel.data.Row kernelRow = - new SparkRowToKernelRow(sparkRow, AddFile.SCHEMA_WITHOUT_STATS); - return IndexedFile.addFile(version, fileIdx, new AddFile(kernelRow)); - } - - sentEnd = true; - return IndexedFile.sentinel(version, DeltaSourceOffset.END_INDEX()); - } - - @Override - public void close() throws IOException { - // toLocalIterator() resources are managed by Spark - } - }; - } - - /** Loads snapshot files at the specified version. */ - private InitialSnapshotCache loadAndValidateSnapshot(long version) { - SnapshotImpl snapshot = (SnapshotImpl) snapshotManager.loadSnapshotAt(version); - // If schema tracking is already active and the initial snapshot has advanced since the tracked - // read snapshot, replace the tracked metadata/protocol before reading snapshot files. - if (metadataEvolutionHandler.shouldTrackMetadataChange() - && snapshot.getVersion() >= readSnapshotAtSourceInit.getVersion()) { - metadataEvolutionHandler.updateMetadataTrackingLogAndFailTheStreamIfNeeded( - snapshot.getMetadata(), - snapshot.getProtocol(), - snapshot.getVersion(), - /* replace= */ true); - } - long commitTimestamp = snapshot.getTimestamp(engine); - - Scan scan = snapshot.getScanBuilder().build(); - - List addFiles = new ArrayList<>(); - try (CloseableIterator filesIter = scan.getScanFiles(engine)) { - while (filesIter.hasNext()) { - FilteredColumnarBatch filteredBatch = filesIter.next(); - - // Get all AddFiles from the batch. Include both dataChange=true and dataChange=false - // (checkpoint files) files. StreamingHelper.getAddFile respects the selection vector - // to filter out duplicate files (e.g., stats re-collection re-adds files with updated - // stats). - for (int rowId = 0; rowId < filteredBatch.getData().getSize(); rowId++) { - Optional addOpt = StreamingHelper.getAddFile(filteredBatch, rowId); - if (addOpt.isPresent()) { - addFiles.add(addOpt.get()); - - // Basic memory protection: each IndexedFile is ~1-2KB (path, stats, partition values, - // etc.). - // This limit aims to prevent OOM for large tables. - // TODO(#5318): support large tables and remove this limit. - if (addFiles.size() > maxInitialSnapshotFiles) { - throw (RuntimeException) - DeltaErrors.initialSnapshotTooLargeForStreaming( - version, addFiles.size(), maxInitialSnapshotFiles, tablePath); - } - } - } + /** + * This method computes the initial snapshot to read when Delta Source was initialized on a fresh + * stream. + * + * @return an iterator of IndexedFiles for the initial snapshot + */ + private CloseableIterator getSnapshotAt(long version) { + if (initialSnapshot == null || version != initialSnapshotVersion) { + cleanUpSnapshotResources(); + Snapshot snapshot = snapshotManager.loadSnapshotAt(version); + + initialSnapshot = new DeltaSourceSnapshot(spark, snapshot, Seq$.MODULE$.empty()); + initialSnapshotVersion = version; + + SnapshotImpl kernelSnapshot = DeltaV2Snapshot$.MODULE$.getKernelSnapshot(snapshot); + + // This handles a special case for schema tracking log when it's initialized but the initial + // snapshot's schema has changed, suppose: + // 1. The stream starts and looks at the initial snapshot to compute the starting offset, say + // at version 0 with schema + // 2. User renames a column, creates version 1 with schema + // 3. The read compatibility check fails during scanning version 1, initializes schema log + // using the initial snapshot's schema (, because that's the safest thing to do as we + // have not served any data from initial snapshot yet) and exits stream. + // 4. Stream restarts, since no starting offset was generated, it will retry loading the + // initial snapshot, which is now at version 1, but the tracked schema is now different + // from the "new" initial snapshot schema! Worse, since schema tracking ignores any schema + // changes inside initial snapshot, we will then be reading the files using a wrong schema! + // The below logic allows us to detect any discrepancies when reading initial snapshot using + // a tracked schema, and reinitialize the log if needed. + if (metadataEvolutionHandler.shouldTrackMetadataChange() + && initialSnapshot.snapshot().version() >= readSnapshotAtSourceInit.getVersion()) { + metadataEvolutionHandler.updateMetadataTrackingLogAndFailTheStreamIfNeeded( + kernelSnapshot.getMetadata(), + kernelSnapshot.getProtocol(), + initialSnapshot.snapshot().version(), + // The new schema should replace the previous initialized schema for initial snapshot + /* replace= */ true); } - } catch (IOException e) { - throw new RuntimeException( - String.format("Failed to read snapshot files at version %d", version), e); } - // TODO(#5318): For large snapshots, consider external sorting. - // CRITICAL: Sort by modificationTime, then path for deterministic ordering - addFiles.sort( - Comparator.comparing(AddFile::getModificationTime).thenComparing(AddFile::getPath)); - - // Build IndexedFile list with sentinels - List indexedFiles = new ArrayList<>(); - - // Add BEGIN sentinel - indexedFiles.add(IndexedFile.sentinel(version, DeltaSourceOffset.BASE_INDEX())); - - // Add data files with sequential indices starting from 0 - for (int i = 0; i < addFiles.size(); i++) { - indexedFiles.add(IndexedFile.addFile(version, /* index= */ i, addFiles.get(i))); - } - - // Add END sentinel - indexedFiles.add(IndexedFile.sentinel(version, DeltaSourceOffset.END_INDEX())); - - return new InitialSnapshotCache( - version, Collections.unmodifiableList(indexedFiles), commitTimestamp); + // TODO: Add range-aware iteration to DeltaSourceSnapshot so V1 and V2 can filter the + // already-processed prefix before decoding files, and V2 avoids converting skipped AddFiles. + CloseableIterator snapshotFiles = + Utils.toCloseableIterator(CollectionConverters.asJava(initialSnapshot.iterator())) + .map( + file -> + IndexedFile.addFile( + file.version(), + file.index(), + KernelActionUtils$.MODULE$.toKernelScanAddFile(file.add()))); + return addBeginAndEndIndexOffsetsForVersion(version, snapshotFiles); } /** diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/DeltaV2Scan.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/DeltaV2Scan.java index 4947c50863a..bc5330eda98 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/DeltaV2Scan.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/DeltaV2Scan.java @@ -17,26 +17,18 @@ import static io.delta.spark.internal.v2.utils.ExpressionUtils.dsv2PredicateToCatalystExpression; -import io.delta.kernel.Snapshot; -import io.delta.kernel.data.FilteredColumnarBatch; -import io.delta.kernel.data.Row; -import io.delta.kernel.defaults.engine.DefaultEngine; -import io.delta.kernel.engine.Engine; import io.delta.kernel.expressions.Predicate; -import io.delta.kernel.internal.ScanImpl; -import io.delta.kernel.internal.actions.AddFile; -import io.delta.kernel.internal.actions.DeletionVectorDescriptor; -import io.delta.kernel.internal.data.ScanStateRow; -import io.delta.kernel.utils.CloseableIterator; +import io.delta.kernel.internal.SnapshotImpl; +import io.delta.spark.internal.v2.DeltaV2JavaLogging; +import io.delta.spark.internal.v2.kernel.KernelEngineFactory; import io.delta.spark.internal.v2.read.cdc.CDCSchemaContext; import io.delta.spark.internal.v2.read.deletionvector.DeletionVectorSchemaContext; -import io.delta.spark.internal.v2.snapshot.DeltaSnapshotManager; import io.delta.spark.internal.v2.utils.PartitionUtils; import io.delta.spark.internal.v2.utils.ScalaUtils; import io.delta.spark.internal.v2.utils.SchemaUtils; -import java.io.IOException; import java.time.ZoneId; import java.util.*; +import java.util.function.Supplier; import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; import org.apache.spark.sql.SparkSession; @@ -49,16 +41,22 @@ import org.apache.spark.sql.connector.read.colstats.ColumnStatistics; import org.apache.spark.sql.connector.read.streaming.MicroBatchStream; import org.apache.spark.sql.delta.DeltaOptions; +import org.apache.spark.sql.delta.Snapshot; +import org.apache.spark.sql.delta.actions.AddFile; import org.apache.spark.sql.delta.sources.DeltaSourceMetadataTrackingLog; +import org.apache.spark.sql.delta.stats.DataSize; +import org.apache.spark.sql.delta.stats.DeltaScan; +import org.apache.spark.sql.delta.v2.interop.DeltaV2Snapshot$; +import org.apache.spark.sql.delta.v2.interop.DeltaV2SnapshotManager; import org.apache.spark.sql.execution.datasources.*; import org.apache.spark.sql.execution.datasources.parquet.ParquetUtils; +import org.apache.spark.sql.execution.datasources.v2.DeltaV2FilterTranslator; import org.apache.spark.sql.internal.SQLConf; import org.apache.spark.sql.sources.Filter; -import org.apache.spark.sql.types.StringType; -import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.types.StructType; import org.apache.spark.sql.util.CaseInsensitiveStringMap; import scala.Option; +import scala.collection.immutable.Seq; /** * Package-private scan implementation for Delta's Spark DataSource V2 read path. @@ -66,22 +64,28 @@ *

This class must remain package-private so callers outside {@code v2.read} depend only on * Spark's public connector interfaces instead of coupling to Delta's internal V2 implementation. */ -class DeltaV2Scan implements Scan, SupportsReportStatistics, SupportsRuntimeV2Filtering { +class DeltaV2Scan extends DeltaV2JavaLogging + implements Scan, SupportsReportStatistics, SupportsRuntimeV2Filtering { - private final DeltaSnapshotManager snapshotManager; + private final DeltaV2SnapshotManager snapshotManager; private final Snapshot initialSnapshot; private final StructType readDataSchema; private final StructType dataSchema; private final StructType partitionSchema; private final StructType ddlOrderedReadOutputSchema; - private final Predicate[] pushedToKernelFilters; - private final Filter[] dataFilters; + // Produces the V1 DeltaScan via DeltaV2Snapshot.filesForScan. Kept lazy because streaming scans + // never consume batch-selected files and must not run the batch data-skipping path during + // ScanBuilder.build(). + private final Supplier deltaScanSupplier; + // Pushed data filters (min/max skipping), as Catalyst expressions, for explain and scan equality. + private final Expression[] dataFilters; + // Pushed partition filters (row-exact), as Catalyst expressions, for explain and scan equality. + private final Expression[] partitionFilters; + private final Set partitionFiltersSet; // Derived Sets used only for equals/hashCode: filters are AND-ed at evaluation time, // so list order has no semantic meaning and two scans with the same filter set in // different orders should compare equal. - private final Set pushedToKernelFiltersSet; - private final Set dataFiltersSet; - private final io.delta.kernel.Scan kernelScan; + private final Set dataFiltersSet; private final Optional catalogStats; private final Configuration hadoopConf; private final boolean isCDCRead; @@ -93,20 +97,9 @@ class DeltaV2Scan implements Scan, SupportsReportStatistics, SupportsRuntimeV2Fi private final DeltaOptions deltaOptions; private final ZoneId zoneId; - // Planned input files and the corresponding selected AddFile actions. - private List partitionedFiles = new ArrayList<>(); - // Per-file row counts, parallel to partitionedFiles. Populated only while rowCountKnown is - // true; cleared if any AddFile lacks numRecords. Retained so totalRows can be recomputed - // after runtime partition filtering prunes files, instead of invalidating the count. - private List perFileRowCounts = new ArrayList<>(); - private List selectedFiles = new ArrayList<>(); - private long totalBytes = 0L; - private long totalRows = 0L; - // true iff every AddFile in the scan had numRecords in its stats JSON. - private boolean rowCountKnown = false; - // Estimated size in bytes accounting for column projection, used for query optimizer cost - // estimation - private long estimatedSizeInBytes = 0L; + // Initialized lazily and shared with batches. Runtime pruning, when supported, replaces this + // reference without modifying an existing selection. + private DeltaScan preparedScan; private volatile boolean planned = false; // Runtime predicates applied after planning (using Set for order-independent comparison) @@ -115,15 +108,15 @@ class DeltaV2Scan implements Scan, SupportsReportStatistics, SupportsRuntimeV2Fi // TODO(#6743): bundle scan-level schemas into a single ScanSchemaContext. public DeltaV2Scan( - DeltaSnapshotManager snapshotManager, + DeltaV2SnapshotManager snapshotManager, Snapshot initialSnapshot, StructType tableSchema, StructType dataSchema, StructType partitionSchema, StructType readDataSchema, - Predicate[] pushedToKernelFilters, - Filter[] dataFilters, - io.delta.kernel.Scan kernelScan, + Supplier scanSupplier, + Expression[] dataFilters, + Expression[] partitionFilters, Optional catalogStats, CaseInsensitiveStringMap options, OptionalInt pushedLimit) { @@ -133,12 +126,13 @@ public DeltaV2Scan( this.dataSchema = Objects.requireNonNull(dataSchema, "dataSchema is null"); this.partitionSchema = Objects.requireNonNull(partitionSchema, "partitionSchema is null"); this.readDataSchema = Objects.requireNonNull(readDataSchema, "readDataSchema is null"); - this.pushedToKernelFilters = - pushedToKernelFilters == null ? new Predicate[0] : pushedToKernelFilters.clone(); - this.dataFilters = dataFilters == null ? new Filter[0] : dataFilters.clone(); - this.pushedToKernelFiltersSet = Set.copyOf(Arrays.asList(this.pushedToKernelFilters)); - this.dataFiltersSet = Set.copyOf(Arrays.asList(this.dataFilters)); - this.kernelScan = Objects.requireNonNull(kernelScan, "kernelScan is null"); + this.deltaScanSupplier = Objects.requireNonNull(scanSupplier, "deltaScanSupplier is null"); + this.dataFilters = dataFilters == null ? new Expression[0] : dataFilters.clone(); + this.partitionFilters = partitionFilters == null ? new Expression[0] : partitionFilters.clone(); + this.partitionFiltersSet = canonicalizedSet(this.partitionFilters); + // Canonicalize so equals/hashCode ignore exprId/ordering: two scans of the same table with the + // same logical filters compare equal even when the filter expressions carry different exprIds. + this.dataFiltersSet = canonicalizedSet(this.dataFilters); this.catalogStats = Objects.requireNonNull(catalogStats, "catalogStats is null"); this.options = Objects.requireNonNull(options, "options is null"); this.pushedLimit = Objects.requireNonNull(pushedLimit, "pushedLimit is null"); @@ -154,6 +148,14 @@ public DeltaV2Scan( isCDCRead ? CDCSchemaContext.appendCDCColumns(ddlOrdered) : ddlOrdered; } + /** + * Returns the Kernel snapshot wrapped by {@link #initialSnapshot} for operations that have not + * migrated to the shared snapshot facade yet. + */ + private SnapshotImpl kernelSnapshot() { + return DeltaV2Snapshot$.MODULE$.getKernelSnapshot(initialSnapshot); + } + /** Read schema for the scan, in the table's DDL column order. */ @Override public StructType readSchema() { @@ -192,7 +194,7 @@ public Scan.ColumnarSupportMode columnarSupportMode() { if (isCDCRead) { schemaForBatchCheck = CDCSchemaContext.appendCDCColumns(schemaForBatchCheck); } - if (PartitionUtils.tableSupportsDeletionVectors(initialSnapshot)) { + if (PartitionUtils.tableSupportsDeletionVectors(kernelSnapshot())) { schemaForBatchCheck = new DeletionVectorSchemaContext(schemaForBatchCheck, partitionSchema) .getSchemaWithDvColumn(); @@ -203,24 +205,49 @@ public Scan.ColumnarSupportMode columnarSupportMode() { : Scan.ColumnarSupportMode.UNSUPPORTED; } + // TODO: Drop toBatch (and the filter translation it needs) once this scan implements the + // file-source Scan interface. DataSourceV2Strategy already routes a Scan that implements + // org.apache.spark.sql.connector.read.FileScan through FileScanBridge to FileSourceScanExec, + // ahead of the plain DataSourceV2ScanRelation case, so toBatch stops being reachable at that + // point and the Batch-only bookkeeping below goes away with it. @Override public Batch toBatch() { + return recordFrameProfileValue("batchScan.toBatch", this::createBatch); + } + + /** Constructs the fallback batch without putting its multi-step body inside a Java lambda. */ + private Batch createBatch() { if (isCDCRead) { throw new UnsupportedOperationException( "Batch reads with CDC (readChangeFeed / readChangeData) are not supported in the V2 " + "connector. Either remove the CDC read option or use a streaming read."); } - ensurePlanned(); + final DeltaScan deltaScan = preparedScan(); + // File selection is done by V1 data skipping, so no kernel predicates are + // pushed to the batch. Keep two distinct filter sets: partition + data filters distinguish + // batches that select different files under otherwise-equal state, while only data filters may + // reach the Parquet reader. Partition columns are materialized from the file path rather than + // stored in Parquet, so passing a partition predicate to Parquet can incorrectly reject every + // row. Order-insensitive equality is handled by DeltaV2Batch. + final Expression[] pushedCatalystFilters = + new Expression[partitionFilters.length + dataFilters.length]; + System.arraycopy(partitionFilters, 0, pushedCatalystFilters, 0, partitionFilters.length); + System.arraycopy( + dataFilters, 0, pushedCatalystFilters, partitionFilters.length, dataFilters.length); + final Filter[] batchPushedFilters = DeltaV2FilterTranslator.translate(pushedCatalystFilters); + final Filter[] batchDataFilters = DeltaV2FilterTranslator.translate(dataFilters); return new DeltaV2Batch( initialSnapshot, dataSchema, partitionSchema, readDataSchema, ddlOrderedReadOutputSchema, - partitionedFiles, - pushedToKernelFilters, - dataFilters, - totalBytes, + deltaScan, + getTablePath(), + zoneId, + new Predicate[0], + batchDataFilters, + batchPushedFilters, scalaOptions, hadoopConf); } @@ -240,10 +267,10 @@ public MicroBatchStream toMicroBatchStream(String checkpointLocation) { Option metadataTrackingLog = MetadataEvolutionHandler.getMetadataTrackingLogForMicroBatchStream( spark, - (io.delta.kernel.internal.SnapshotImpl) latestSnapshot, + latestSnapshot, options, snapshotManager, - DefaultEngine.create(hadoopConf), + KernelEngineFactory.createDefaultEngine(hadoopConf), Option.apply(checkpointLocation), /* mergeConsecutiveSchemaChanges= */ false); @@ -258,7 +285,7 @@ public MicroBatchStream toMicroBatchStream(String checkpointLocation) { partitionSchema, readDataSchema, ddlOrderedReadOutputSchema, - dataFilters != null ? dataFilters : new Filter[0], + new Filter[0], scalaOptions != null ? scalaOptions : scala.collection.immutable.Map$.MODULE$.empty(), metadataTrackingLog, checkpointLocation); @@ -267,9 +294,7 @@ public MicroBatchStream toMicroBatchStream(String checkpointLocation) { @Override public String description() { final String pushed = - Arrays.stream(pushedToKernelFilters) - .map(Object::toString) - .collect(Collectors.joining(", ")); + Arrays.stream(partitionFilters).map(Object::toString).collect(Collectors.joining(", ")); final String data = Arrays.stream(dataFilters).map(Object::toString).collect(Collectors.joining(", ")); final StringBuilder description = @@ -280,152 +305,120 @@ public String description() { return description.toString(); } + /** + * Returns the catalog size when available, or the compression-adjusted post-pruning file size, + * without constructing row or column statistics. + * + *

Intentionally no {@code @Override}: Delta also compiles this source against supported Spark + * versions whose {@link SupportsReportStatistics} interface predates this optional method. The + * method still overrides the interface default when Delta is built against a Spark version that + * provides it. + */ + public OptionalLong estimateSizeInBytes() { + return estimateSizeInBytesInternal(); + } + + /** Package-private entry point for Scala wrappers compiled together with this Java source. */ + OptionalLong estimateSizeInBytesInternal() { + if (isZeroLimit()) { + return OptionalLong.of(0L); + } + if (catalogStats.isPresent()) { + return catalogStats.get().sizeInBytes(); + } + return estimateSelectedFileSizeInBytes(); + } + @Override public Statistics estimateStatistics() { - ensurePlanned(); - // Capture mutable scan state as final locals so the returned Statistics object reflects a - // consistent snapshot. A subsequent filter() call mutates rowCountKnown and totalRows - // (see ensurePlanned(runtimePredicates)), and we don't want those mutations to leak into a - // Statistics instance the caller is still holding. - final long plannedBytes = estimatedSizeInBytes; - final boolean rowCountKnownSnapshot = rowCountKnown; - final long totalRowsSnapshot = totalRows; - - // When catalog stats are available and CBO is enabled, combine table-level stats - // (for columnStats) with planned file stats (for sizeInBytes and numRows). - // This mirrors V1's LogicalRelation.computeStats() which gates column stats on - // conf.cboEnabled || conf.planStatsEnabled. - if (arePlanStatsEnabled() && catalogStats.isPresent()) { + if (isZeroLimit()) { + return statistics(OptionalLong.of(0L), OptionalLong.of(0L), Collections.emptyMap()); + } + if (catalogHasNumRows()) { final Statistics stats = catalogStats.get(); - return new Statistics() { - @Override - public OptionalLong sizeInBytes() { - // Planned file size is authoritative (even if 0 for an empty table) - return OptionalLong.of(plannedBytes); - } - - @Override - public OptionalLong numRows() { - // Prefer per-file (post-prune) numRows when known. Fall back to catalog numRows - // when per-file is unavailable (e.g. some AddFile lacks numRecords). The catalog - // value is table-level and may be stale, but it's better than reporting unknown. - if (rowCountKnownSnapshot) { - return OptionalLong.of(totalRowsSnapshot); - } - return stats.numRows(); - } + return statistics(OptionalLong.empty(), stats.numRows(), stats.columnStats()); + } + return statistics( + estimateSelectedFileSizeInBytes(), OptionalLong.empty(), Collections.emptyMap()); + } - @Override - public Map columnStats() { - // TODO: After partition pruning, column stats (e.g. min, max, nullCount, - // distinctCount) could be tightened based on the pruned file-level stats. - return stats.columnStats(); - } - }; + /** + * Sum of the selected files' on-disk sizes. Used as the scanned-bytes fallback when the DeltaScan + * carries no {@code scanned.bytesCompressed} aggregate (for example, a table without file + * statistics). {@code AddFile.size()} is always populated, so this reads only lightweight file + * metadata and never constructs physical input files. + */ + private static long selectedFileSizeInBytes(DeltaScan deltaScan) { + long totalBytes = 0L; + for (AddFile addFile : scala.jdk.javaapi.CollectionConverters.asJava(deltaScan.files())) { + totalBytes += addFile.size(); } + return totalBytes; + } + + private OptionalLong estimateSelectedFileSizeInBytes() { + // Like PreparedDeltaFileIndex.sizeInBytes, this is the selected scan's size, not the + // full-snapshot checksum size. Reading it may trigger file selection, but not input-file + // construction. + final DeltaScan deltaScan = preparedScan(); + final long totalBytes = + toOptionalLong(deltaScan.scanned().bytesCompressed()) + .orElseGet(() -> selectedFileSizeInBytes(deltaScan)); + // Do not scale the selected-file bytes by readSchema. Delta returns false from + // reflectsFullyPushedDownFilters(), so Spark re-adds fully pushed filters when adjusting + // statistics. Delta's scan builder retains filter-only columns in readSchema; when that makes + // the scan output wider than the query output, Spark also leaves the Project that restores the + // query output above the relation. The Filter and Project in Spark's logical plan, rather than + // this scan, own those statistics adjustments. + return OptionalLong.of(totalBytes); + } + + private boolean isZeroLimit() { + return pushedLimit.isPresent() && pushedLimit.getAsInt() == 0; + } - // No catalog stats available or CBO disabled — return stats from planned files only + private static OptionalLong toOptionalLong(scala.Option value) { + return value.isDefined() + ? OptionalLong.of(((Number) value.get()).longValue()) + : OptionalLong.empty(); + } + + private Statistics statistics( + OptionalLong sizeInBytes, + OptionalLong numRows, + Map columnStats) { return new Statistics() { @Override public OptionalLong sizeInBytes() { - return OptionalLong.of(plannedBytes); + return sizeInBytes; } @Override public OptionalLong numRows() { - return rowCountKnownSnapshot ? OptionalLong.of(totalRowsSnapshot) : OptionalLong.empty(); + return numRows; + } + + @Override + public Map columnStats() { + return columnStats; } }; } /** - * Computes the estimated size in bytes accounting for column projection. - * - *

This mirrors what {@code SizeInBytesOnlyStatsPlanVisitor.visitUnaryNode} (from Spark code) - * would compute for a {@code Project} over a {@code LogicalRelation}: {@code sizeInBytes = - * childSizeInBytes * outputRowSize / childRowSize} - * - *

Where: - * - *

    - *
  • childRowSize = {@code ROW_OVERHEAD + dataSchema + partitionSchema} (equivalent to - * LogicalRelation output) - *
  • outputRowSize = {@code ROW_OVERHEAD + readDataSchema + partitionSchema} - * (equivalent to Project output) - *
+ * Delta requires Spark to retain fully pushed predicates when adjusting scan statistics. In + * particular, catalog statistics describe the unfiltered table and must not be treated as if + * every connector-pushed predicate were already reflected. * - *

When catalog column stats are available, uses per-column {@code avgLen} instead of {@code - * defaultSize()} for more accurate size estimation, mirroring {@code - * EstimationUtils.getSizePerRow()} behavior. - * - *

This provides consistent statistics with the v1 code path (LogicalRelation + visitUnaryNode - * from Spark code directory). - * - * @param totalBytes the total size in bytes of the planned files (raw physical size) - * @return the estimated size in bytes after accounting for column projection - */ - private long computeEstimatedSizeWithColumnProjection(long totalBytes) { - if (totalBytes <= 0) { - return totalBytes; - } - - // Row overhead constant, matching EstimationUtils.getSizePerRow (from Spark) - final int ROW_OVERHEAD = 8; - - // Use avgLen from catalog column stats when available for more accurate estimation - Map avgLenByColumn = getAvgLenByColumn(); - - final long fullSchemaRowSize = - ROW_OVERHEAD - + getSchemaSize(dataSchema, avgLenByColumn) - + getSchemaSize(partitionSchema, avgLenByColumn); - final long outputRowSize = ROW_OVERHEAD + getSchemaSize(readSchema(), avgLenByColumn); - - long estimatedBytes = (totalBytes * outputRowSize) / fullSchemaRowSize; - - return Math.max(1L, estimatedBytes); - } - - /** - * Returns a map of column name to avgLen from catalog column stats, used to improve size - * estimation accuracy when catalog stats are available. + *

Intentionally no {@code @Override}; see {@link #estimateSizeInBytes()}. */ - private Map getAvgLenByColumn() { - if (!catalogStats.isPresent()) { - return Collections.emptyMap(); - } - Map colStats = catalogStats.get().columnStats(); - if (colStats.isEmpty()) { - return Collections.emptyMap(); - } - Map result = new HashMap<>(); - for (Map.Entry entry : colStats.entrySet()) { - result.put(entry.getKey().fieldNames()[0], entry.getValue().avgLen()); - } - return result; + public boolean reflectsFullyPushedDownFilters() { + return reflectsFullyPushedDownFiltersInternal(); } - /** - * Computes the estimated in-memory size for a schema, using avgLen from catalog stats when - * available, falling back to defaultSize(). Mirrors EstimationUtils.getSizePerRow(). For - * StringType columns with avgLen, adds UTF8String overhead (base + offset + numBytes = 12 bytes). - */ - private static long getSchemaSize(StructType schema, Map avgLenByColumn) { - long size = 0; - for (StructField field : schema.fields()) { - OptionalLong avgLen = avgLenByColumn.getOrDefault(field.name(), OptionalLong.empty()); - if (avgLen.isPresent()) { - if (field.dataType() instanceof StringType) { - // UTF8String: base + offset + numBytes (matching EstimationUtils.getSizePerRow) - size += avgLen.getAsLong() + 8 + 4; - } else { - size += avgLen.getAsLong(); - } - } else { - size += field.dataType().defaultSize(); - } - } - return size; + /** Package-private entry point for Scala wrappers compiled together with this Java source. */ + boolean reflectsFullyPushedDownFiltersInternal() { + return false; } /** @@ -434,215 +427,69 @@ private static long getSchemaSize(StructType schema, Map a * @return the table path with trailing slash */ public String getTablePath() { - final Engine tableEngine = DefaultEngine.create(hadoopConf); - final Row scanState = kernelScan.getScanState(tableEngine); - final String tableRoot = ScanStateRow.getTableRoot(scanState).toUri().toString(); + // PartitionUtils passes the resolved path to SparkPath.fromUrlString, so the table root must be + // URL-encoded (for example, spaces and literal '%' characters). + final String tableRoot = initialSnapshot.dataPath().toUri().toString(); return tableRoot.endsWith("/") ? tableRoot : tableRoot + "/"; } - /** - * Plan the files to scan by materializing {@link PartitionedFile}s and aggregating size stats. - * Ensures all iterators are closed to avoid resource leaks. - * - *

When a limit is pushed (via {@link SupportsPushDownLimit}), per-file {@code numRecords} - * (minus deletion-vector cardinality) are used to stop adding files once enough logical rows have - * been accumulated to satisfy the limit. Files that lack statistics are added but do not count - * toward the limit. - */ + /** Select files lazily and reuse the statistics computed by V1 file selection. */ private void planScanFiles() { - // Short-circuit LIMIT 0. When plan stats are enabled, mark the row count as known because it is - // derived directly from the limit, rather than falling back to catalog stats. - if (isLimitPushed() && pushedLimit.getAsInt() == 0) { - estimatedSizeInBytes = 0; - rowCountKnown = arePlanStatsEnabled(); - totalRows = 0; - return; - } - - final Engine tableEngine = DefaultEngine.create(hadoopConf); - final String tablePath = getTablePath(); - - // TODO: Promote getScanFiles(Engine, boolean includeStats) to the public Scan interface to - // avoid coupling to the kernel-internal ScanImpl class. Until that API is available, this - // instanceof check is the only way to request per-file statistics from the kernel. - // - // Read per-file stats JSON when either: - // - the optimizer will use numRows (CBO or planStats enabled), matching V1's behavior - // (LogicalRelation.computeStats()), or - // - a limit is pushed, in which case we need numRecords to decide when to stop planning. - // Both require the kernel scan to be a ScanImpl (the only path that supports includeStats). - final boolean scanIsStatsCapable = kernelScan instanceof ScanImpl; - final boolean includeStats = scanIsStatsCapable && (arePlanStatsEnabled() || isLimitPushed()); - final CloseableIterator scanFileBatches; - if (includeStats) { - scanFileBatches = ((ScanImpl) kernelScan).getScanFiles(tableEngine, true /* includeStats */); - // Only participate in CBO numRows reporting when the optimizer actually consumes it, matching - // V1's LogicalRelation.computeStats() gate. A limit alone forces stats to be read (for - // termination via logicalRowCount) but must not start surfacing numRows when CBO is off. - rowCountKnown = arePlanStatsEnabled(); // assume all files have stats; cleared on first miss - } else { - rowCountKnown = false; - scanFileBatches = kernelScan.getScanFiles(tableEngine); - } - - // Tracks logical (surviving) rows accumulated so far for limit pushdown. This differs from - // totalRows, which is the physical numRecords sum used for CBO and stops being tracked once any - // file lacks stats. accumulatedLimitRows always drives limit termination when a limit is - // pushed. - long accumulatedLimitRows = 0L; - try (scanFileBatches) { - // Check the limit before hasNext() to avoid any unnecessary kernel iterator I/O. - while (!isLimitReached(accumulatedLimitRows) && scanFileBatches.hasNext()) { - final FilteredColumnarBatch batch = scanFileBatches.next(); - - try (CloseableIterator addFileRowIter = batch.getRows()) { - while (!isLimitReached(accumulatedLimitRows) && addFileRowIter.hasNext()) { - final Row row = addFileRowIter.next(); - final AddFile addFile = new AddFile(row.getStruct(0)); - - final PartitionedFile partitionedFile = - PartitionUtils.buildPartitionedFile(addFile, partitionSchema, tablePath, zoneId); - - totalBytes += addFile.getSize(); - partitionedFiles.add(partitionedFile); - selectedFiles.add(new DeltaScanFile(addFile)); - - Optional numRecords = - rowCountKnown || isLimitPushed() ? addFile.getNumRecords() : Optional.empty(); - if (rowCountKnown) { - if (numRecords.isPresent()) { - totalRows += numRecords.get(); - perFileRowCounts.add(numRecords.get()); - } else { - // This file has no numRecords — row count is unknowable for the whole scan. - // Clear partial state and stop accumulating for all subsequent files. - rowCountKnown = false; - totalRows = 0; - perFileRowCounts.clear(); - } - } - - if (isLimitPushed()) { - accumulatedLimitRows += logicalRowCount(addFile, numRecords); - } - } - } - } - } catch (IOException e) { - throw new RuntimeException(e); - } - - // Pre-compute estimated size accounting for column projection - estimatedSizeInBytes = computeEstimatedSizeWithColumnProjection(totalBytes); + // Select files lazily via V1 data skipping over the Kernel-backed snapshot, which + // DataSkippingDeltaV2SnapshotSuite validates against the V1 oracle. Keeping this work behind + // batch planning avoids running an unused batch scan for MicroBatchStream. When a limit is + // pushed the supplier routes selection through V1's limit-aware filesForScan. + final Supplier selectFiles = + () -> Objects.requireNonNull(deltaScanSupplier.get(), "deltaScanSupplier returned null"); + final DeltaScan deltaScan = recordFrameProfileValue("scan.awaitFileSelection", selectFiles); + final Runnable materializeFiles = () -> materializeSelectedFiles(deltaScan); + recordFrameProfileAction("scan.materializeSelectedFiles", materializeFiles); } - /** Returns true iff a limit was pushed down from Spark. */ - private boolean isLimitPushed() { - return pushedLimit.isPresent(); - } - - /** - * Returns true iff a limit is pushed and the accumulated logical row count already satisfies it. - */ - private boolean isLimitReached(long accumulatedLimitRows) { - return isLimitPushed() && accumulatedLimitRows >= pushedLimit.getAsInt(); + /** Retains scan metadata without constructing physical input files. */ + private void materializeSelectedFiles(DeltaScan deltaScan) { + Objects.requireNonNull(deltaScan.scannedSnapshot(), "deltaScan.scannedSnapshot returned null"); + // Preserve V1's aggregates, including unknown values. Stats access must not traverse AddFiles + // to recompute bytes or per-file row counts. + preparedScan = deltaScan; } - /** - * Returns the surviving row count contributed by a file toward a pushed limit: the physical - * {@code numRecords} minus deletion-vector cardinality. Returns 0 when statistics are missing, so - * a file without stats is planned but never counted toward the limit. - */ - private static long logicalRowCount(AddFile addFile, Optional numRecords) { - if (!numRecords.isPresent()) { - return 0L; - } - long dvCardinality = - addFile.getDeletionVector().map(DeletionVectorDescriptor::getCardinality).orElse(0L); - return Math.max(0L, numRecords.get() - dvCardinality); + /** Copies stats and optionally files, preserving snapshot and filter metadata. */ + private static DeltaScan withFilesAndStats( + DeltaScan deltaScan, Optional> replacementFiles, DataSize scanned) { + return deltaScan.copy( + deltaScan.version(), + replacementFiles.isPresent() ? replacementFiles.get() : deltaScan.files(), + deltaScan.total(), + deltaScan.partition(), + scanned, + deltaScan.scannedSnapshot(), + deltaScan.partitionFilters(), + deltaScan.dataFilters(), + deltaScan.partitionLikeDataFilters(), + deltaScan.rewrittenPartitionLikeDataFilters(), + deltaScan.unusedFilters(), + deltaScan.scanDurationMs(), + deltaScan.scanDurationNs(), + deltaScan.dataSkippingType()); } /** - * Ensure the scan is planned exactly once in a thread-safe manner, optionally applying runtime - * filters. + * Returns the lazily selected scan without constructing physical input files. Initialization is + * guarded by this scan's monitor; planned is published only after success. */ - private synchronized void ensurePlanned(List runtimePredicates) { - // First, ensure planning is done + synchronized DeltaScan preparedScan() { if (!planned) { - planScanFiles(); + recordFrameProfileAction("scan.planFiles", this::planScanFiles); planned = true; } - - // Then apply runtime predicates if provided - if (runtimePredicates != null && !runtimePredicates.isEmpty()) { - // Record the applied predicates for equals/hashCode comparison - for (RuntimePredicate filter : runtimePredicates) { - appliedRuntimePredicates.add(filter.predicate); - } - - List runtimeFilteredPartitionedFiles = new ArrayList<>(); - List runtimeFilteredFiles = new ArrayList<>(); - // Parallel to runtimeFilteredPartitionedFiles; only used when rowCountKnown is true. - List filteredRowCounts = rowCountKnown ? new ArrayList<>() : null; - long newTotalRows = 0L; - for (int i = 0; i < this.partitionedFiles.size(); i++) { - PartitionedFile pf = this.partitionedFiles.get(i); - InternalRow partitionValues = pf.partitionValues(); - boolean allMatch = - runtimePredicates.stream() - .allMatch(predicate -> predicate.evaluator.eval(partitionValues)); - if (allMatch) { - runtimeFilteredPartitionedFiles.add(pf); - runtimeFilteredFiles.add(this.selectedFiles.get(i)); - if (rowCountKnown) { - long rc = this.perFileRowCounts.get(i); - filteredRowCounts.add(rc); - newTotalRows += rc; - } - } - } - - // Update partitionedFiles, totalBytes, totalRows, and estimatedSizeInBytes if any partition - // is filtered out - if (runtimeFilteredPartitionedFiles.size() < this.partitionedFiles.size()) { - this.partitionedFiles = runtimeFilteredPartitionedFiles; - this.selectedFiles = runtimeFilteredFiles; - this.totalBytes = - runtimeFilteredPartitionedFiles.stream().mapToLong(PartitionedFile::fileSize).sum(); - this.estimatedSizeInBytes = computeEstimatedSizeWithColumnProjection(this.totalBytes); - if (rowCountKnown) { - // Recompute totalRows from per-file counts of files that survived pruning so - // numRows() reports the post-prune count rather than a stale pre-filter value. - this.perFileRowCounts = filteredRowCounts; - this.totalRows = newTotalRows; - } - } - } - } - - /** Ensure the scan is planned exactly once in a thread-safe manner. */ - private void ensurePlanned() { - // Pass null to indicate no runtime predicate should be applied - just perform the scan planning - ensurePlanned(null); + return preparedScan; } public StructType getDataSchema() { return dataSchema; } - /** - * Returns the Delta files selected by this scan after pushdown and runtime filtering. - * - *

The returned descriptors preserve only the metadata needed by row-level ReplaceData commits - * to construct matching RemoveFile actions. - * - * @apiNote Internal API for the DSv2 DML write path (see {@code DeltaReplaceDataBatchWrite}). - */ - public List getSelectedFiles() { - ensurePlanned(); - return Collections.unmodifiableList(selectedFiles); - } - public StructType getPartitionSchema() { return partitionSchema; } @@ -664,6 +511,57 @@ OptionalInt getPushedLimit() { return pushedLimit; } + private synchronized void applyRuntimePredicates(List runtimePredicates) { + preparedScan(); + // Record the applied predicates for equals/hashCode comparison + for (RuntimePredicate filter : runtimePredicates) { + appliedRuntimePredicates.add(filter.predicate); + } + + List runtimeFilteredAddFiles = new ArrayList<>(); + long newTotalBytes = 0L; + boolean newRowCountKnown = preparedScan.scanned().rows().isDefined(); + long newTotalRows = 0L; + long originalFileCount = 0L; + for (AddFile addFile : scala.jdk.javaapi.CollectionConverters.asJava(preparedScan.files())) { + originalFileCount++; + InternalRow partitionValues = + PartitionUtils.getPartitionRow(addFile.partitionValues(), partitionSchema, zoneId); + boolean allMatch = + runtimePredicates.stream() + .allMatch(predicate -> predicate.evaluator.eval(partitionValues)); + if (newRowCountKnown) { + scala.Option numRecords = addFile.numPhysicalRecords(); + if (numRecords.isEmpty()) { + newRowCountKnown = false; + } else if (allMatch) { + newTotalRows += ((Number) numRecords.get()).longValue(); + } + } + if (allMatch) { + runtimeFilteredAddFiles.add(addFile); + newTotalBytes += addFile.size(); + } + } + + // A no-op filter preserves aggregate-only statistics. If files were pruned, only complete + // per-file counts can describe the survivors (limit selection can omit those counts). + if (runtimeFilteredAddFiles.size() < originalFileCount) { + DataSize scanned = + new DataSize( + Option.apply(newTotalBytes), + newRowCountKnown ? Option.apply(newTotalRows) : Option.empty(), + Option.apply((long) runtimeFilteredAddFiles.size()), + Option.empty()); + preparedScan = + withFilesAndStats( + preparedScan, + Optional.of( + scala.jdk.javaapi.CollectionConverters.asScala(runtimeFilteredAddFiles).toList()), + scanned); + } + } + @Override public NamedReference[] filterAttributes() { return Arrays.stream(partitionSchema.fields()) @@ -702,30 +600,14 @@ public void filter(org.apache.spark.sql.connector.expressions.filter.Predicate[] } if (!runtimePredicates.isEmpty()) { - // Apply runtime predicates within the synchronized ensurePlanned method - ensurePlanned(runtimePredicates); + // Apply runtime predicates while holding this scan's monitor + applyRuntimePredicates(runtimePredicates); } } - /** - * Returns whether plan-time statistics should be collected and reported. Matches V1 behavior: - * {@code LogicalRelation.computeStats()} only surfaces stats when {@code spark.sql.cbo.enabled} - * or {@code spark.sql.cbo.planStats.enabled} is true. - * - *

Despite the row-count-flavored framing in V1, this gate controls multiple things in the V2 - * scan path: - * - *

    - *
  • whether {@code numRows()} is reported in {@link #estimateStatistics()} - *
  • whether the catalog-stats branch is entered (which also governs {@code columnStats} - * propagation from the catalog) - *
  • whether per-file stats JSON is parsed in {@link #planScanFiles()} - *
- * - * Future readers touching any of these paths should treat this helper as load-bearing. - */ - private boolean arePlanStatsEnabled() { - return sqlConf.cboEnabled() || sqlConf.planStatsEnabled(); + /** Returns whether the catalog-provided statistics include a numRows value. */ + private boolean catalogHasNumRows() { + return catalogStats.isPresent() && catalogStats.get().numRows().isPresent(); } @Override @@ -737,35 +619,46 @@ public boolean equals(Object o) { return false; } DeltaV2Scan that = (DeltaV2Scan) o; - return Objects.equals(initialSnapshot.getPath(), that.initialSnapshot.getPath()) - && initialSnapshot.getVersion() == that.initialSnapshot.getVersion() + return Objects.equals(initialSnapshot.path(), that.initialSnapshot.path()) + && initialSnapshot.version() == that.initialSnapshot.version() && Objects.equals(dataSchema, that.dataSchema) && Objects.equals(partitionSchema, that.partitionSchema) && Objects.equals(readDataSchema, that.readDataSchema) - && Objects.equals(pushedToKernelFiltersSet, that.pushedToKernelFiltersSet) + && Objects.equals(partitionFiltersSet, that.partitionFiltersSet) && Objects.equals(dataFiltersSet, that.dataFiltersSet) - // ignoring kernelScan because it is derived from Snapshot which is created from tablePath, - // with pushed down filters that are also recorded in `pushedToKernelFilters` && Objects.equals(options, that.options) && Objects.equals(pushedLimit, that.pushedLimit) && Objects.equals(appliedRuntimePredicates, that.appliedRuntimePredicates) && Objects.equals(catalogStats, that.catalogStats); } + /** Set of exprId-normalized filter expressions (exprId- and order-insensitive) for equals. */ + private static Set canonicalizedSet(Expression[] filters) { + Set set = new HashSet<>(); + for (Expression f : filters) { + set.add(DeltaV2ScanBuilder.normalizeForEquality(f)); + } + return set; + } + @Override public int hashCode() { - return Objects.hash( - catalogStats, - initialSnapshot.getPath(), - initialSnapshot.getVersion(), - dataSchema, - partitionSchema, - readDataSchema, - options, - pushedLimit, - appliedRuntimePredicates, - pushedToKernelFiltersSet, - dataFiltersSet); + int result = + Objects.hash( + catalogStats, + initialSnapshot.path(), + initialSnapshot.version(), + dataSchema, + partitionSchema, + readDataSchema, + options, + pushedLimit, + appliedRuntimePredicates, + dataFiltersSet); + // Fold in the partition-filter set so scans that differ only by pushed partition filters hash + // differently, matching equals()'s partitionFiltersSet check. + result = 31 * result + Objects.hashCode(partitionFiltersSet); + return result; } /** diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/DeltaV2ScanUtils.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/DeltaV2ScanUtils.java index 0fa4147f586..a5da4b2f2fb 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/DeltaV2ScanUtils.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/DeltaV2ScanUtils.java @@ -15,13 +15,15 @@ */ package io.delta.spark.internal.v2.read; -import io.delta.kernel.Snapshot; -import io.delta.spark.internal.v2.snapshot.DeltaSnapshotManager; +import io.delta.kernel.engine.Engine; import java.util.Optional; import org.apache.spark.sql.catalyst.InternalRow; +import org.apache.spark.sql.catalyst.catalog.CatalogTable; import org.apache.spark.sql.connector.read.PartitionReaderFactory; import org.apache.spark.sql.connector.read.ScanBuilder; import org.apache.spark.sql.connector.read.Statistics; +import org.apache.spark.sql.delta.Snapshot; +import org.apache.spark.sql.delta.v2.interop.DeltaV2SnapshotManager; import org.apache.spark.sql.execution.datasources.PartitionedFile; import org.apache.spark.sql.types.StructType; import org.apache.spark.sql.util.CaseInsensitiveStringMap; @@ -41,7 +43,9 @@ public static PartitionReaderFactory newReaderFactory( public static ScanBuilder newScanBuilder( String tableName, Snapshot initialSnapshot, - DeltaSnapshotManager snapshotManager, + Engine kernelEngine, + Optional catalogTable, + DeltaV2SnapshotManager snapshotManager, StructType dataSchema, StructType partitionSchema, StructType tableSchema, @@ -50,6 +54,8 @@ public static ScanBuilder newScanBuilder( return DeltaV2ScanBuilder.create( tableName, initialSnapshot, + kernelEngine, + catalogTable, snapshotManager, dataSchema, partitionSchema, diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/MetadataEvolutionHandler.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/MetadataEvolutionHandler.java index d169ae6de8d..49f27da068c 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/MetadataEvolutionHandler.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/MetadataEvolutionHandler.java @@ -37,7 +37,6 @@ import io.delta.kernel.utils.CloseableIterator.BreakableFilterResult; import io.delta.spark.internal.v2.adapters.KernelMetadataAdapter; import io.delta.spark.internal.v2.adapters.KernelProtocolAdapter; -import io.delta.spark.internal.v2.snapshot.DeltaSnapshotManager; import io.delta.spark.internal.v2.utils.ScalaUtils; import io.delta.spark.internal.v2.utils.SchemaUtils; import io.delta.spark.internal.v2.utils.StreamingHelper; @@ -67,6 +66,8 @@ import org.apache.spark.sql.delta.sources.PersistedMetadata; import org.apache.spark.sql.delta.v2.interop.AbstractMetadata; import org.apache.spark.sql.delta.v2.interop.AbstractProtocol; +import org.apache.spark.sql.delta.v2.interop.DeltaV2Snapshot$; +import org.apache.spark.sql.delta.v2.interop.DeltaV2SnapshotManager; import org.apache.spark.sql.util.CaseInsensitiveStringMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -102,7 +103,7 @@ public class MetadataEvolutionHandler { private final SparkSession spark; private final String tableId; private final String tablePath; - private final DeltaSnapshotManager snapshotManager; + private final DeltaV2SnapshotManager snapshotManager; private final Engine engine; private final DeltaOptions options; private final DeltaStreamUtils.SchemaReadOptions schemaReadOptions; @@ -120,7 +121,7 @@ public MetadataEvolutionHandler( SparkSession spark, String tableId, String tablePath, - DeltaSnapshotManager snapshotManager, + DeltaV2SnapshotManager snapshotManager, Engine engine, DeltaOptions options, DeltaStreamUtils.SchemaReadOptions schemaReadOptions, @@ -340,7 +341,9 @@ public void initializeMetadataTrackingAndExitStream( metadata = validated.metadata; protocol = validated.protocol; } else { - SnapshotImpl snapshot = (SnapshotImpl) snapshotManager.loadSnapshotAt(batchStartVersion); + SnapshotImpl snapshot = + DeltaV2Snapshot$.MODULE$.getKernelSnapshot( + snapshotManager.loadSnapshotAt(batchStartVersion)); version = snapshot.getVersion(); metadata = snapshot.getMetadata(); protocol = snapshot.getProtocol(); @@ -425,11 +428,14 @@ private Map collectProtocolActions(long startVersion, long endVe *

V2 port of V1's {@code * DeltaSourceMetadataEvolutionSupport.validateAndResolveMetadataForLogInitialization}. */ + // TODO(kernel-table-manager): Remove getKernelSnapshot here once collectMetadataActions + // and collectProtocolActions return AbstractMetadata/AbstractProtocol instead of Kernel types. private ValidatedMetadataAndProtocol validateAndResolveMetadataForLogInitialization( long startVersion, long endVersion) { List metadataChanges = new ArrayList<>(collectMetadataActions(startVersion, endVersion).values()); - SnapshotImpl startSnapshot = (SnapshotImpl) snapshotManager.loadSnapshotAt(startVersion); + SnapshotImpl startSnapshot = + DeltaV2Snapshot$.MODULE$.getKernelSnapshot(snapshotManager.loadSnapshotAt(startVersion)); Metadata startMetadata = startSnapshot.getMetadata(); // Try to find rename or drop columns in between, or nullability/datatype changes by using @@ -510,9 +516,9 @@ private static class ValidatedMetadataAndProtocol { */ public static Optional getPersistedMetadataForMicroBatchStream( SparkSession spark, - SnapshotImpl snapshot, + org.apache.spark.sql.delta.Snapshot snapshot, Map options, - DeltaSnapshotManager snapshotManager, + DeltaV2SnapshotManager snapshotManager, Engine engine) { boolean mergeConsecutiveSchemaChanges = (boolean) @@ -568,7 +574,7 @@ public static boolean shouldPropagateSchemaTrackingToTable( * and {@code lazyCrcInfo} fields throw on access. Callers must only read {@code metadata}, {@code * protocol}, {@code schema}, {@code version}, and {@code dataPath} from the returned snapshot; * anything that triggers log replay (e.g. {@code getCurrentCrcInfo}, {@code getScanBuilder}) must - * call {@link DeltaSnapshotManager#loadSnapshotAt} instead. + * call {@link DeltaV2SnapshotManager#loadSnapshotAt} instead. */ public static SnapshotImpl buildReadSnapshotFromPersistedMetadata( SnapshotImpl snapshotAtSourceInit, Engine engine, PersistedMetadata customMetadata) { @@ -663,9 +669,9 @@ private static Protocol toKernelProtocol( */ public static Option getMetadataTrackingLogForMicroBatchStream( SparkSession spark, - SnapshotImpl snapshot, + org.apache.spark.sql.delta.Snapshot snapshot, Map options, - DeltaSnapshotManager snapshotManager, + DeltaV2SnapshotManager snapshotManager, Engine engine, Option sourceMetadataPathOpt, boolean mergeConsecutiveSchemaChanges) { @@ -685,12 +691,12 @@ public static Option getMetadataTrackingLogForMi "Schema tracking location is not supported for Delta streaming source"); } - String tablePath = snapshot.getPath(); + String tablePath = snapshot.dataPath().toString(); return Option.apply( DeltaSourceMetadataTrackingLog.create( spark, location, - snapshot.getMetadata().getId(), + snapshot.metadata().getId(), tablePath, ScalaUtils.toScalaMap(options), sourceMetadataPathOpt, @@ -705,7 +711,7 @@ public static Option getMetadataTrackingLogForMi /** V2 port of {@code DeltaSourceMetadataEvolutionSupport.getMergedConsecutiveMetadataChanges}. */ public static Option getMergedConsecutiveMetadataChanges( PersistedMetadata currentMetadata, - DeltaSnapshotManager snapshotManager, + DeltaV2SnapshotManager snapshotManager, Engine engine, String tablePath) { final long currentMetadataVersion = currentMetadata.deltaCommitVersion(); diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/ScanFileRDD.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/ScanFileRDD.java index fd7fa3ca82b..3c00e0a9f22 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/ScanFileRDD.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/ScanFileRDD.java @@ -83,7 +83,7 @@ public Partition[] getPartitions() { @Override public scala.collection.Iterator compute(Partition split, TaskContext context) { Engine engine = DefaultEngine.create(serializableSnapshot.getHadoopConf()); - Scan scan = serializableSnapshot.toScan(); + Scan scan = serializableSnapshot.toScan(engine); CloseableIterator batchIter; try { diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/snapshot/DeltaSnapshotManager.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/snapshot/DeltaSnapshotManager.java deleted file mode 100644 index 3cab6be2b9e..00000000000 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/snapshot/DeltaSnapshotManager.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright (2025) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.delta.spark.internal.v2.snapshot; - -import io.delta.kernel.CommitRange; -import io.delta.kernel.Snapshot; -import io.delta.kernel.engine.Engine; -import io.delta.kernel.internal.DeltaHistoryManager; -import io.delta.spark.internal.v2.exception.VersionNotFoundException; -import java.util.Optional; -import org.apache.spark.annotation.Experimental; - -/** - * Interface for managing Delta table snapshots. - * - *

This interface provides methods for loading, caching, and querying Delta table snapshots. It - * supports both current snapshot access and historical snapshot queries based on version or - * timestamp. - * - *

Implementations of this interface are responsible for managing snapshot lifecycle, including - * loading snapshots from storage and maintaining any necessary caching. - */ -@Experimental -public interface DeltaSnapshotManager { - - /** - * Loads and returns the latest snapshot of the Delta table. - * - * @return the latest snapshot of the Delta table - */ - Snapshot loadLatestSnapshot(); - - /** - * Loads and returns a snapshot at a specific version of the Delta table. - * - * @param version the version number to load (must be >= 0) - * @return the snapshot at the specified version - * @throws io.delta.kernel.exceptions.KernelException if the version cannot be loaded - */ - Snapshot loadSnapshotAt(long version); - - /** - * Finds and returns the commit that was active at a specific timestamp. - * - * @param timestampMillis the timestamp in milliseconds since epoch (UTC) - * @param canReturnLastCommit if true, returns the last commit if the timestamp is after all - * commits; if false, throws an exception - * @param mustBeRecreatable if true, only considers commits that can be fully recreated from - * available log files; if false, considers all commits - * @param canReturnEarliestCommit if true, returns the earliest commit if the timestamp is before - * all commits; if false, throws an exception - * @return the commit that was active at the specified timestamp - * @throws io.delta.kernel.exceptions.KernelException if no suitable commit is found based on the - * provided flags - */ - DeltaHistoryManager.Commit getActiveCommitAtTime( - long timestampMillis, - boolean canReturnLastCommit, - boolean mustBeRecreatable, - boolean canReturnEarliestCommit); - - /** - * Checks if a specific version of the Delta table exists and is accessible. - * - * @param version the version to check - * @param mustBeRecreatable if true, requires that the version can be fully recreated from - * available log files; if false, only requires that the version's log file exists - * @param allowOutOfRange if true, allows versions greater than the latest version without - * throwing an exception; if false, throws exception for out-of-range versions - * @throws VersionNotFoundException if the version is not available or does not meet the specified - * criteria - */ - void checkVersionExists(long version, boolean mustBeRecreatable, boolean allowOutOfRange) - throws VersionNotFoundException; - - /** - * Gets a range of table changes (commits) between start and end versions. - * - *

Expected Behavior: - * - *

    - *
  • Returns a {@link io.delta.kernel.CommitRange} representing all commits from the start - * version (inclusive) to the end version (inclusive if provided) - *
  • If endVersion is not provided, the range extends to the latest available version - *
  • The returned CommitRange can be used to iterate through actions in the version range - *
  • This is typically used for streaming and incremental processing scenarios - *
- * - *

Use Case: Use this method for streaming queries, incremental processing, or CDC - * scenarios where you need to process changes between versions. - * - * @param engine the engine implementation for executing operations - * @param startVersion the starting version (inclusive) - * @param endVersion optional ending version (inclusive); if not provided, extends to latest - * @return a CommitRange representing the specified range of commits - */ - CommitRange getTableChanges(Engine engine, long startVersion, Optional endVersion); -} diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/snapshot/PathBasedSnapshotManager.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/snapshot/PathBasedSnapshotManager.java index 0dd4845b00f..249f47f031d 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/snapshot/PathBasedSnapshotManager.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/snapshot/PathBasedSnapshotManager.java @@ -19,27 +19,31 @@ import io.delta.kernel.CommitRange; import io.delta.kernel.CommitRangeBuilder; -import io.delta.kernel.Snapshot; import io.delta.kernel.TableManager; -import io.delta.kernel.defaults.engine.DefaultEngine; import io.delta.kernel.engine.Engine; import io.delta.kernel.internal.DeltaHistoryManager; import io.delta.kernel.internal.SnapshotImpl; import io.delta.spark.internal.v2.exception.VersionNotFoundException; +import io.delta.spark.internal.v2.kernel.KernelEngineFactory; import java.util.ArrayList; import java.util.Optional; import org.apache.hadoop.conf.Configuration; import org.apache.spark.annotation.Experimental; +import org.apache.spark.sql.delta.Snapshot; +import org.apache.spark.sql.delta.v2.interop.DeltaV2SnapshotManager; +import org.apache.spark.sql.delta.v2.interop.DeltaV2SnapshotManager$; -/** Implementation of DeltaSnapshotManager for managing Delta snapshots for Path-based Table. */ +/** Implementation of DeltaV2SnapshotManager for managing Delta snapshots for Path-based Table. */ @Experimental -public class PathBasedSnapshotManager implements DeltaSnapshotManager { +public class PathBasedSnapshotManager implements DeltaV2SnapshotManager { private final String tablePath; private final Engine kernelEngine; public PathBasedSnapshotManager(String tablePath, Configuration hadoopConf) { - this(tablePath, DefaultEngine.create(requireNonNull(hadoopConf, "hadoopConf is null"))); + this( + tablePath, + KernelEngineFactory.createDefaultEngine(requireNonNull(hadoopConf, "hadoopConf is null"))); } public PathBasedSnapshotManager(String tablePath, Engine kernelEngine) { @@ -54,7 +58,8 @@ public PathBasedSnapshotManager(String tablePath, Engine kernelEngine) { */ @Override public Snapshot loadLatestSnapshot() { - return TableManager.loadSnapshot(tablePath).build(kernelEngine); + return DeltaV2SnapshotManager$.MODULE$.wrapKernelSnapshot( + loadLatestKernelSnapshot(), tablePath); } /** @@ -65,7 +70,17 @@ public Snapshot loadLatestSnapshot() { */ @Override public Snapshot loadSnapshotAt(long version) { - return TableManager.loadSnapshot(tablePath).atVersion(version).build(kernelEngine); + return DeltaV2SnapshotManager$.MODULE$.wrapKernelSnapshot( + loadKernelSnapshotAt(version), tablePath); + } + + private SnapshotImpl loadLatestKernelSnapshot() { + return (SnapshotImpl) TableManager.loadSnapshot(tablePath).build(kernelEngine); + } + + private SnapshotImpl loadKernelSnapshotAt(long version) { + return (SnapshotImpl) + TableManager.loadSnapshot(tablePath).atVersion(version).build(kernelEngine); } /** @@ -89,7 +104,7 @@ public DeltaHistoryManager.Commit getActiveCommitAtTime( boolean canReturnLastCommit, boolean mustBeRecreatable, boolean canReturnEarliestCommit) { - SnapshotImpl snapshot = (SnapshotImpl) loadLatestSnapshot(); + SnapshotImpl snapshot = loadLatestKernelSnapshot(); return DeltaHistoryManager.getActiveCommitAtTimestamp( kernelEngine, snapshot, @@ -114,7 +129,7 @@ public DeltaHistoryManager.Commit getActiveCommitAtTime( @Override public void checkVersionExists(long version, boolean mustBeRecreatable, boolean allowOutOfRange) throws VersionNotFoundException { - SnapshotImpl snapshot = (SnapshotImpl) loadLatestSnapshot(); + SnapshotImpl snapshot = loadLatestKernelSnapshot(); long earliest = mustBeRecreatable ? DeltaHistoryManager.getEarliestRecreatableCommit( diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/snapshot/SnapshotManagerFactory.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/snapshot/SnapshotManagerFactory.java index 0cc4860d3e3..acff35ec2bf 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/snapshot/SnapshotManagerFactory.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/snapshot/SnapshotManagerFactory.java @@ -29,9 +29,10 @@ import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.catalyst.catalog.CatalogTable; import org.apache.spark.sql.delta.coordinatedcommits.UCTokenBasedRestClientFactory$; +import org.apache.spark.sql.delta.v2.interop.DeltaV2SnapshotManager; /** - * Factory for creating {@link DeltaSnapshotManager} instances. + * Factory for creating {@link DeltaV2SnapshotManager} instances. * *

This factory determines the appropriate snapshot manager based on the table configuration: * @@ -52,9 +53,9 @@ private SnapshotManagerFactory() {} * @param tablePath the filesystem path to the Delta table * @param kernelEngine the pre-configured Kernel {@link Engine} to use for table operations * @param catalogTable optional Spark catalog table metadata - * @return a {@link DeltaSnapshotManager} appropriate for the table type + * @return a {@link DeltaV2SnapshotManager} appropriate for the table type */ - public static DeltaSnapshotManager create( + public static DeltaV2SnapshotManager create( String tablePath, Engine kernelEngine, Optional catalogTable) { if (catalogTable.isPresent()) { diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/snapshot/unitycatalog/UCManagedTableSnapshotManager.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/snapshot/unitycatalog/UCManagedTableSnapshotManager.java index 1b2a4622440..f1393a9521e 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/snapshot/unitycatalog/UCManagedTableSnapshotManager.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/snapshot/unitycatalog/UCManagedTableSnapshotManager.java @@ -18,7 +18,6 @@ import static java.util.Objects.requireNonNull; import io.delta.kernel.CommitRange; -import io.delta.kernel.Snapshot; import io.delta.kernel.engine.Engine; import io.delta.kernel.internal.DeltaHistoryManager; import io.delta.kernel.internal.SnapshotImpl; @@ -26,9 +25,11 @@ import io.delta.kernel.unitycatalog.UCCatalogManagedClient; import io.delta.kernel.unitycatalog.UCTableIdentifier; import io.delta.spark.internal.v2.exception.VersionNotFoundException; -import io.delta.spark.internal.v2.snapshot.DeltaSnapshotManager; import java.util.List; import java.util.Optional; +import org.apache.spark.sql.delta.Snapshot; +import org.apache.spark.sql.delta.v2.interop.DeltaV2SnapshotManager; +import org.apache.spark.sql.delta.v2.interop.DeltaV2SnapshotManager$; /** * Snapshot manager for Unity Catalog managed tables. @@ -36,7 +37,7 @@ *

Used for tables with the catalog-managed commit feature enabled. Unity Catalog serves as the * source of truth for the table's commit history. */ -public class UCManagedTableSnapshotManager implements DeltaSnapshotManager { +public class UCManagedTableSnapshotManager implements DeltaV2SnapshotManager { private final UCCatalogManagedClient ucCatalogManagedClient; private final String tableId; @@ -69,22 +70,25 @@ public UCManagedTableSnapshotManager( */ @Override public Snapshot loadLatestSnapshot() { - return loadSnapshot(Optional.empty() /* versionOpt */); + return DeltaV2SnapshotManager$.MODULE$.wrapKernelSnapshot( + loadKernelSnapshot(Optional.empty()), tablePath); } @Override public Snapshot loadSnapshotAt(long version) { - return loadSnapshot(Optional.of(version)); + return DeltaV2SnapshotManager$.MODULE$.wrapKernelSnapshot( + loadKernelSnapshot(Optional.of(version)), tablePath); } - private Snapshot loadSnapshot(Optional versionOpt) { - return ucCatalogManagedClient.loadSnapshot( - engine, - tableId, - tablePath, - tableIdentifier, - versionOpt, - Optional.empty() /* timestampOpt */); + private SnapshotImpl loadKernelSnapshot(Optional versionOpt) { + return (SnapshotImpl) + ucCatalogManagedClient.loadSnapshot( + engine, + tableId, + tablePath, + tableIdentifier, + versionOpt, + Optional.empty() /* timestampOpt */); } /** @@ -108,7 +112,7 @@ public DeltaHistoryManager.Commit getActiveCommitAtTime( boolean canReturnLastCommit, boolean mustBeRecreatable, boolean canReturnEarliestCommit) { - SnapshotImpl snapshot = (SnapshotImpl) loadLatestSnapshot(); + SnapshotImpl snapshot = loadKernelSnapshot(Optional.empty()); List catalogCommits = snapshot.getLogSegment().getAllCatalogCommits(); return DeltaHistoryManager.getActiveCommitAtTimestamp( engine, @@ -138,7 +142,7 @@ public DeltaHistoryManager.Commit getActiveCommitAtTime( public void checkVersionExists(long version, boolean mustBeRecreatable, boolean allowOutOfRange) throws VersionNotFoundException { // Load latest to get the current version bounds - SnapshotImpl snapshot = (SnapshotImpl) loadLatestSnapshot(); + SnapshotImpl snapshot = loadKernelSnapshot(Optional.empty()); // Latest version visible in this UC-managed snapshot. long latestSnapshotVersion = snapshot.getVersion(); diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/PartitionUtils.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/PartitionUtils.java index a84ebdbca0d..9dae5581a06 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/PartitionUtils.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/PartitionUtils.java @@ -35,7 +35,9 @@ import io.delta.spark.internal.v2.read.metadata.MetadataStructReadFunction; import io.delta.spark.internal.v2.read.metadata.MetadataStructSchemaContext; import java.time.ZoneId; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; @@ -57,6 +59,7 @@ import org.apache.spark.sql.delta.DeltaParquetFileFormat; import org.apache.spark.sql.delta.RowId$; import org.apache.spark.sql.delta.RowIndexFilterType; +import org.apache.spark.sql.delta.stats.DeltaScan; import org.apache.spark.sql.execution.datasources.FileFormat$; import org.apache.spark.sql.execution.datasources.FilePartition; import org.apache.spark.sql.execution.datasources.FilePartition$; @@ -133,6 +136,30 @@ public static long calculateMaxSplitBytes( return Math.min(defaultMaxSplitBytes, Math.max(openCostInBytes, bytesPerCore)); } + /** + * Materializes selected AddFiles only when a batch needs physical input partitions. Optimizer + * statistics and validation consumers retain the AddFiles without constructing partition rows, + * resolving paths, or serializing DV metadata. + */ + public static InputPartition[] planInputPartitions( + SparkSession sparkSession, + DeltaScan deltaScan, + StructType partitionSchema, + String tablePath, + ZoneId zoneId, + Configuration hadoopConf, + SQLConf sqlConf) { + List selectedFiles = + CollectionConverters.asJava(deltaScan.files()); + List partitionedFiles = new ArrayList<>(selectedFiles.size()); + long totalBytes = 0L; + for (org.apache.spark.sql.delta.actions.AddFile addFile : selectedFiles) { + partitionedFiles.add(buildPartitionedFile(addFile, partitionSchema, tablePath, zoneId)); + totalBytes += addFile.size(); + } + return planInputPartitions(sparkSession, partitionedFiles, totalBytes, hadoopConf, sqlConf); + } + /** * Plan input partitions by bin-packing a list of {@link PartitionedFile}s into {@link * FilePartition}s. @@ -162,41 +189,60 @@ public static InputPartition[] planInputPartitions( */ public static InternalRow getPartitionRow( MapValue partitionValues, StructType partitionSchema, ZoneId zoneId) { + return getPartitionRow(buildPartitionValuesMap(partitionValues), partitionSchema, zoneId); + } + + /** + * Copies partition values out of a potentially batch-backed Kernel map. + * + *

The returned map remains valid after the Kernel batch is closed. + */ + public static Map buildPartitionValuesMap(MapValue partitionValues) { + Objects.requireNonNull(partitionValues, "partitionValues is null"); + Map result = new LinkedHashMap<>(partitionValues.getSize()); + for (int index = 0; index < partitionValues.getSize(); index++) { + String key = partitionValues.getKeys().getString(index); + String value = + partitionValues.getValues().isNullAt(index) + ? null + : partitionValues.getValues().getString(index); + result.put(key, value); + } + return Collections.unmodifiableMap(result); + } + + /** Returns an immutable copy of already materialized partition values. */ + public static Map buildPartitionValuesMap(Map partitionValues) { + Objects.requireNonNull(partitionValues, "partitionValues is null"); + return Collections.unmodifiableMap(new LinkedHashMap<>(partitionValues)); + } + + /** + * Build the partition {@link InternalRow} from durable partition values copied out of a Kernel + * row. This overload is for consumers that outlive the Kernel batch backing {@link MapValue}. + */ + public static InternalRow getPartitionRow( + Map partitionValues, StructType partitionSchema, ZoneId zoneId) { final int numPartCols = partitionSchema.fields().length; - assert partitionValues.getSize() == numPartCols + assert partitionValues.size() == numPartCols : String.format( java.util.Locale.ROOT, "Partition values size from add file %d != partition columns size %d", - partitionValues.getSize(), + partitionValues.size(), numPartCols); final Object[] values = new Object[numPartCols]; - - // Build physical name -> index map once - // Partition values use physical names as keys when column mapping is enabled - final Map physicalNameToIndex = new HashMap<>(numPartCols); - for (int i = 0; i < numPartCols; i++) { - StructField field = partitionSchema.fields()[i]; - String physicalName = DeltaColumnMapping.getPhysicalName(field); - physicalNameToIndex.put(physicalName, i); - values[i] = null; - } - - // Fill values in a single pass over partitionValues - for (int idx = 0; idx < partitionValues.getSize(); idx++) { - final String key = partitionValues.getKeys().getString(idx); - final String strVal = partitionValues.getValues().getString(idx); - final Integer pos = physicalNameToIndex.get(key); - if (pos != null) { - final StructField field = partitionSchema.fields()[pos]; - if (strVal == null) { - values[pos] = null; - } else if (field.dataType() instanceof StringType) { - values[pos] = UTF8String.fromString(strVal); - } else { - values[pos] = - PartitioningUtils.castPartValueToDesiredType(field.dataType(), strVal, zoneId); - } + for (int index = 0; index < numPartCols; index++) { + final StructField field = partitionSchema.fields()[index]; + final String physicalName = DeltaColumnMapping.getPhysicalName(field); + final String strVal = partitionValues.get(physicalName); + if (strVal == null) { + values[index] = null; + } else if (field.dataType() instanceof StringType) { + values[index] = UTF8String.fromString(strVal); + } else { + values[index] = + PartitioningUtils.castPartValueToDesiredType(field.dataType(), strVal, zoneId); } } return new GenericInternalRow(values); @@ -207,8 +253,8 @@ public static InternalRow getPartitionRow( * columns of {@code row}, as required by {@code Transaction.getWriteContext}. Insertion order * follows {@code partitionSchema}. * - *

TODO(#7140): key by physical name to support column mapping (safe today as build() rejects - * column-mapped tables). + *

Keys are the logical partition column names. For a column-mapped table they are translated + * to the physical names and for a non-column-mapped table they are left unchanged. * * @param row the full write row Spark hands the writer * @param partitionSchema the partition columns (in partition order) @@ -339,18 +385,107 @@ private static Literal convertRowValueToKernelLiteral( */ public static PartitionedFile buildPartitionedFile( AddFile addFile, StructType partitionSchema, String tablePath, ZoneId zoneId) { - scala.collection.immutable.Map metadata = - mergeIntoScalaMap( - buildDvMetadataScala(addFile.getDeletionVector()), - buildRowTrackingMetadata(addFile.getBaseRowId(), addFile.getDefaultRowCommitVersion())); - return makePartitionedFile( - new Path(tablePath, addFile.getPath()).toString(), + return buildPartitionedFile( + addFile.getPath(), addFile.getSize(), addFile.getModificationTime(), + addFile.getDeletionVector(), + addFile.getBaseRowId(), + addFile.getDefaultRowCommitVersion(), getPartitionRow(addFile.getPartitionValues(), partitionSchema, zoneId), + tablePath); + } + + private static PartitionedFile buildPartitionedFile( + String path, + long size, + long modificationTime, + Optional deletionVector, + Optional baseRowId, + Optional defaultRowCommitVersion, + InternalRow partitionRow, + String tablePath) { + scala.collection.immutable.Map metadata = + mergeIntoScalaMap( + buildDvMetadataScala(deletionVector), + buildRowTrackingMetadata(baseRowId, defaultRowCommitVersion)); + return makePartitionedFile( + new Path(tablePath, path).toString(), size, modificationTime, partitionRow, metadata); + } + + /** + * Build a PartitionedFile from a V1 AddFile produced by {@code DeltaV2Snapshot.filesForScan}. The + * V1 file selection reuses the same file records the Kernel path stores (see {@code + * KernelSnapshotUtils.toV1AddFile}), so this mirrors the Kernel {@link #buildPartitionedFile} + * overload field-for-field: same path resolution, same DV / row-tracking metadata keys, same + * partition-row construction. Unlike the Kernel overload it does not assert the partition-value + * count, because V1 AddFile may omit unset partition keys. + */ + public static PartitionedFile buildPartitionedFile( + org.apache.spark.sql.delta.actions.AddFile v1AddFile, + StructType partitionSchema, + String tablePath, + ZoneId zoneId) { + final org.apache.spark.sql.delta.actions.DeletionVectorDescriptor dv = + v1AddFile.deletionVector(); + final scala.collection.immutable.Map dvMetadata = + dv == null + ? emptyScalaMap() + : buildDvMetadata(dv.serializeToBase64(), RowIndexFilterType.IF_CONTAINED); + final scala.collection.immutable.Map metadata = + mergeIntoScalaMap( + dvMetadata, + buildRowTrackingMetadata( + toJavaLong(v1AddFile.baseRowId()), + toJavaLong(v1AddFile.defaultRowCommitVersion()))); + return makePartitionedFile( + new Path(tablePath, v1AddFile.path()).toString(), + v1AddFile.size(), + v1AddFile.modificationTime(), + getPartitionRow(v1AddFile.partitionValues(), partitionSchema, zoneId), metadata); } + /** Converts a Scala {@code Option[Long]} to a Java {@code Optional}. */ + private static Optional toJavaLong(scala.Option opt) { + return opt.isDefined() ? Optional.of((Long) opt.get()) : Optional.empty(); + } + + /** Returns an empty typed Scala map, matching the shape the metadata builders return. */ + @SuppressWarnings("unchecked") + private static scala.collection.immutable.Map emptyScalaMap() { + return (scala.collection.immutable.Map) + (scala.collection.immutable.Map) scala.collection.immutable.Map$.MODULE$.empty(); + } + + /** + * Builds the partition {@link InternalRow} from a V1 AddFile's partition-value map. Mirrors the + * {@link MapValue} overload (physical-name keying, timezone-aware casting) but reads a Scala + * {@code Map[String, String]} and tolerates missing keys (leaving those columns null) rather than + * asserting the value count, because V1 AddFile may omit unset partition keys. + */ + public static InternalRow getPartitionRow( + scala.collection.immutable.Map partitionValues, + StructType partitionSchema, + ZoneId zoneId) { + final int numPartCols = partitionSchema.fields().length; + final Object[] values = new Object[numPartCols]; + final Map javaPartitionValues = CollectionConverters.asJava(partitionValues); + for (int i = 0; i < numPartCols; i++) { + final StructField field = partitionSchema.fields()[i]; + final String physicalName = DeltaColumnMapping.getPhysicalName(field); + final String strVal = javaPartitionValues.get(physicalName); + if (strVal == null) { + values[i] = null; + } else if (field.dataType() instanceof StringType) { + values[i] = UTF8String.fromString(strVal); + } else { + values[i] = PartitioningUtils.castPartValueToDesiredType(field.dataType(), strVal, zoneId); + } + } + return new GenericInternalRow(values); + } + /** * Build a PartitionedFile from a CDCDataFile with CDC constants in * otherConstantMetadataColumnValues for the {@link CDCReadFunction} to null-coalesce. @@ -502,9 +637,8 @@ public static PartitionReaderFactory createDeltaParquetReaderFactory( // For write-time CDF reads (streaming with readChangeFeed=true), build the schema context // and augment readDataSchema with CDC tail columns before DV wrapping so that DV column - // indices account for them. Read-time CDF (via DeltaChangelogBatch) does not go - // through this path: DeltaChangelogBatch's outer CDCPartitionReaderFactory injects the - // tail columns as per-partition constants instead. + // indices account for them. Read-time CDF (Auto-CDF) does not go through this path: its + // partition reader factory injects the tail columns as per-partition constants instead. Optional cdcSchemaContext = isWriteTimeCDCRead ? Optional.of(new CDCSchemaContext(readDataSchema, partitionSchema)) diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/ScalaUtils.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/ScalaUtils.java index 5bb3bf3898c..12d452296c6 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/ScalaUtils.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/ScalaUtils.java @@ -19,6 +19,7 @@ import java.util.Collections; import java.util.Map; import java.util.Optional; +import java.util.OptionalLong; import scala.Option; import scala.Tuple2; import scala.collection.JavaConverters; @@ -78,4 +79,11 @@ public static Option toScalaOption(Optional optional) { public static Optional toJavaOptional(Option option) { return option.isDefined() ? Optional.of(option.get()) : Optional.empty(); } + + /** Converts a Scala {@link Option} containing a long to a Java {@link OptionalLong}. */ + public static OptionalLong toJavaOptionalLong(Option option) { + return option.isDefined() + ? OptionalLong.of(((Number) option.get()).longValue()) + : OptionalLong.empty(); + } } diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/SerializableReadOnlySnapshot.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/SerializableReadOnlySnapshot.java index 7ba7dc83f94..6cc060d3374 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/SerializableReadOnlySnapshot.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/SerializableReadOnlySnapshot.java @@ -119,7 +119,7 @@ public static SerializableReadOnlySnapshot fromSnapshot( * returned {@code Scan} only exposes {@code getScanFiles()} — no write-path or commit APIs. */ public Scan toScan(Configuration hadoopConfOverride) { - return buildScan(hadoopConfOverride); + return buildScan(DefaultEngine.create(hadoopConfOverride)); } /** @@ -127,22 +127,26 @@ public Scan toScan(Configuration hadoopConfOverride) { * no conf override is needed. */ public Scan toScan() { - return buildScan(hadoopConf.value()); + return buildScan(DefaultEngine.create(hadoopConf.value())); + } + + /** Reconstructs a read-only scan using the caller-owned imperative engine. */ + public Scan toScan(Engine engine) { + return buildScan(engine); } public long getVersion() { return version; } - /** Returns the serialized Hadoop configuration for creating an Engine on the executor. */ + /** Returns the serialized Hadoop configuration for creating an Engine on the Spark executor. */ public Configuration getHadoopConf() { return hadoopConf.value(); } // ---- internal reconstruction ---- - private Scan buildScan(Configuration conf) { - Engine engine = DefaultEngine.create(conf); + private Scan buildScan(Engine engine) { io.delta.kernel.internal.fs.Path kernelDataPath = new io.delta.kernel.internal.fs.Path(dataPath); io.delta.kernel.internal.fs.Path kernelLogPath = new io.delta.kernel.internal.fs.Path(logPath); diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/StreamingHelper.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/StreamingHelper.java index 8bdae5c5a6f..20774b060bd 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/StreamingHelper.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/StreamingHelper.java @@ -35,10 +35,10 @@ import io.delta.kernel.internal.util.Preconditions; import io.delta.kernel.utils.CloseableIterator; import io.delta.spark.internal.v2.read.CDCDataFile; -import io.delta.spark.internal.v2.snapshot.DeltaSnapshotManager; import java.io.IOException; import java.util.*; import org.apache.spark.annotation.Experimental; +import org.apache.spark.sql.delta.v2.interop.DeltaV2SnapshotManager; /** * Helper class providing utilities for working with Delta table data in streaming scenarios. @@ -220,7 +220,7 @@ public static CloseableIterator getCommitActionsFromRangeUnsafe( public static Map collectMetadataActionsFromRangeUnsafe( long startVersion, Optional endVersionOpt, - DeltaSnapshotManager snapshotManager, + DeltaV2SnapshotManager snapshotManager, Engine engine, String tablePath) { return collectActionsFromRangeUnsafe( @@ -256,7 +256,7 @@ public static Map collectMetadataActionsFromRangeUnsafe( public static Map collectProtocolActionsFromRangeUnsafe( long startVersion, Optional endVersionOpt, - DeltaSnapshotManager snapshotManager, + DeltaV2SnapshotManager snapshotManager, Engine engine, String tablePath) { return collectActionsFromRangeUnsafe( @@ -283,7 +283,7 @@ private interface RowExtractor { private static Map collectActionsFromRangeUnsafe( long startVersion, Optional endVersionOpt, - DeltaSnapshotManager snapshotManager, + DeltaV2SnapshotManager snapshotManager, Engine engine, String tablePath, DeltaLogActionUtils.DeltaAction actionType, diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/write/DeltaRowLevelOperationBuilder.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/write/DeltaRowLevelOperationBuilder.java index a8f87f83a2b..e61f7c20e34 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/write/DeltaRowLevelOperationBuilder.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/write/DeltaRowLevelOperationBuilder.java @@ -17,7 +17,6 @@ import static java.util.Objects.requireNonNull; -import io.delta.kernel.Snapshot; import io.delta.kernel.engine.Engine; import io.delta.spark.internal.v2.catalog.DeltaV2Table; import org.apache.hadoop.conf.Configuration; @@ -29,6 +28,7 @@ import org.apache.spark.sql.connector.write.RowLevelOperationBuilder; import org.apache.spark.sql.connector.write.RowLevelOperationInfo; import org.apache.spark.sql.connector.write.WriteBuilder; +import org.apache.spark.sql.delta.Snapshot; import org.apache.spark.sql.execution.datasources.FileFormat$; import org.apache.spark.sql.util.CaseInsensitiveStringMap; diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/write/DeltaV2BatchWrite.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/write/DeltaV2BatchWrite.java index 3a4655dd94a..997573afc4d 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/write/DeltaV2BatchWrite.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/write/DeltaV2BatchWrite.java @@ -15,7 +15,6 @@ */ package io.delta.spark.internal.v2.write; -import io.delta.kernel.Snapshot; import io.delta.kernel.Transaction; import io.delta.kernel.TransactionCommitResult; import io.delta.kernel.data.Row; @@ -32,6 +31,7 @@ import org.apache.spark.sql.connector.write.PhysicalWriteInfo; import org.apache.spark.sql.connector.write.Write; import org.apache.spark.sql.connector.write.WriterCommitMessage; +import org.apache.spark.sql.delta.Snapshot; import org.apache.spark.sql.types.StructType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -64,10 +64,18 @@ static String getEngineInfo() { Snapshot initialSnapshot, StructType dataSchema, StructType partitionSchema, - LogicalWriteInfo writeInfo) { + LogicalWriteInfo writeInfo, + boolean variantShreddingEnabled) { this.context = DeltaV2BatchWriteContext.create( - engine, hadoopConf, tablePath, initialSnapshot, dataSchema, partitionSchema, writeInfo); + engine, + hadoopConf, + tablePath, + initialSnapshot, + dataSchema, + partitionSchema, + writeInfo, + variantShreddingEnabled); } @Override diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/write/DeltaV2BatchWriteContext.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/write/DeltaV2BatchWriteContext.java index 0e2aacb4640..8b86fdfd1fe 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/write/DeltaV2BatchWriteContext.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/write/DeltaV2BatchWriteContext.java @@ -16,7 +16,6 @@ package io.delta.spark.internal.v2.write; import io.delta.kernel.Operation; -import io.delta.kernel.Snapshot; import io.delta.kernel.Transaction; import io.delta.kernel.data.Row; import io.delta.kernel.engine.Engine; @@ -25,6 +24,8 @@ import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.spark.sql.connector.write.LogicalWriteInfo; +import org.apache.spark.sql.delta.Snapshot; +import org.apache.spark.sql.delta.v2.interop.DeltaV2Snapshot$; import org.apache.spark.sql.types.StructType; /** @@ -46,9 +47,17 @@ static DeltaV2BatchWriteContext create( Snapshot initialSnapshot, StructType dataSchema, StructType partitionSchema, - LogicalWriteInfo writeInfo) { + LogicalWriteInfo writeInfo, + boolean variantShreddingEnabled) { return new DeltaV2BatchWriteContext( - engine, hadoopConf, tablePath, initialSnapshot, dataSchema, partitionSchema, writeInfo); + engine, + hadoopConf, + tablePath, + initialSnapshot, + dataSchema, + partitionSchema, + writeInfo, + variantShreddingEnabled); } private DeltaV2BatchWriteContext( @@ -58,10 +67,20 @@ private DeltaV2BatchWriteContext( Snapshot initialSnapshot, StructType dataSchema, StructType partitionSchema, - LogicalWriteInfo writeInfo) { - super(engine, hadoopConf, tablePath, initialSnapshot, dataSchema, partitionSchema, writeInfo); + LogicalWriteInfo writeInfo, + boolean variantShreddingEnabled) { + super( + engine, + hadoopConf, + tablePath, + initialSnapshot, + dataSchema, + partitionSchema, + writeInfo, + variantShreddingEnabled); this.transaction = - initialSnapshot + DeltaV2Snapshot$.MODULE$ + .getKernelSnapshot(initialSnapshot) .buildUpdateTableTransaction(getEngineInfo(), Operation.WRITE) .build(getEngine()); Row txnState = transaction.getTransactionState(getEngine()); diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/write/DeltaV2StreamingWrite.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/write/DeltaV2StreamingWrite.java index 7ca3fce8588..7a9f68a0fc1 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/write/DeltaV2StreamingWrite.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/write/DeltaV2StreamingWrite.java @@ -18,21 +18,23 @@ import static java.util.Objects.requireNonNull; import io.delta.kernel.Operation; -import io.delta.kernel.Snapshot; import io.delta.kernel.Transaction; import io.delta.kernel.data.Row; import io.delta.kernel.engine.Engine; import io.delta.kernel.exceptions.ConcurrentTransactionException; import io.delta.kernel.internal.SnapshotImpl; -import io.delta.kernel.internal.actions.Protocol; -import io.delta.kernel.types.StructType; import io.delta.kernel.utils.CloseableIterable; -import io.delta.spark.internal.v2.snapshot.DeltaSnapshotManager; import java.util.function.Function; import org.apache.spark.sql.connector.write.PhysicalWriteInfo; import org.apache.spark.sql.connector.write.WriterCommitMessage; import org.apache.spark.sql.connector.write.streaming.StreamingDataWriterFactory; import org.apache.spark.sql.connector.write.streaming.StreamingWrite; +import org.apache.spark.sql.delta.DeltaConfigs; +import org.apache.spark.sql.delta.Snapshot; +import org.apache.spark.sql.delta.actions.Protocol; +import org.apache.spark.sql.delta.v2.interop.DeltaV2Snapshot$; +import org.apache.spark.sql.delta.v2.interop.DeltaV2SnapshotManager; +import org.apache.spark.sql.types.StructType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -50,49 +52,66 @@ * replay, as V1 ({@code txn.txnVersion}) does. A concurrent same-epoch commit that races the * pre-check is still caught as {@link ConcurrentTransactionException} and skipped. * - *

Schema/protocol guard: {@link #commit} fails the query if the reloaded snapshot's - * schema or protocol has diverged from the write state's, since Kernel does not re-validate the - * executor-written files at commit. TODO(#7140): rebuild the write state against the new - * schema/protocol so a compatible change (e.g. an added nullable column) is tolerated instead of - * forcing a query restart. + *

Layout guard: {@link #commit} fails the query if the reloaded snapshot's schema or + * protocol has diverged from the write state's, since Kernel does not re-validate the + * executor-written files at commit. The variant shredding property is checked alongside them: it + * decides the file layout but changes neither schema nor protocol, so the other two checks do not + * see it. TODO(#7140): rebuild the write state against the new schema/protocol so a compatible + * change (e.g. an added nullable column) is tolerated instead of forcing a query restart. */ class DeltaV2StreamingWrite implements StreamingWrite { private static final Logger logger = LoggerFactory.getLogger(DeltaV2StreamingWrite.class); private final Engine engine; - private final DeltaSnapshotManager snapshotManager; + private final DeltaV2SnapshotManager snapshotManager; private final String queryId; private final DeltaV2DataWriterFactory dataWriterFactory; // The write state's schema/protocol baseline; the per-epoch guard fails if the table diverges. private final StructType writeSchema; private final Protocol writeProtocol; + // Same idea for the shredding layout: the executor-side Parquet writer is built once from this + // value, so an epoch committed after the property changed would carry the stale layout. + private final boolean writeVariantShreddingEnabled; + // Whether the property could actually change this write's layout (version supports shredding, + // kill switch on, schema has a variant column). When false, a property change cannot affect the + // files, so the guard below must not act on it. Frozen with the write state. + private final boolean variantLayoutFollowsProperty; /** * @param engine Kernel engine (driver-only) * @param initialSnapshot the batch's planned snapshot; write-state source and guard baseline * @param snapshotManager reloads the latest snapshot per epoch (see {@link #commit}) * @param queryId streaming query id; the transaction application id for cross-restart idempotency + * @param variantShreddingEnabled the table's shredding property as the write state was built with + * it; the per-epoch guard baseline + * @param variantLayoutFollowsProperty whether that property can actually change this write's + * layout; when false the guard ignores a change to it * @param dataWriterFactoryBuilder builds the executor write state; supplied by {@link * DeltaV2Write} to share construction with the batch path */ DeltaV2StreamingWrite( Engine engine, Snapshot initialSnapshot, - DeltaSnapshotManager snapshotManager, + DeltaV2SnapshotManager snapshotManager, String queryId, + boolean variantShreddingEnabled, + boolean variantLayoutFollowsProperty, Function dataWriterFactoryBuilder) { this.engine = requireNonNull(engine, "engine is null"); requireNonNull(initialSnapshot, "initialSnapshot is null"); this.snapshotManager = requireNonNull(snapshotManager, "snapshotManager is null"); this.queryId = requireNonNull(queryId, "queryId is null"); requireNonNull(dataWriterFactoryBuilder, "dataWriterFactoryBuilder is null"); - this.writeSchema = initialSnapshot.getSchema(); - this.writeProtocol = ((SnapshotImpl) initialSnapshot).getProtocol(); + this.writeSchema = initialSnapshot.schema(); + this.writeProtocol = initialSnapshot.protocol(); + this.writeVariantShreddingEnabled = variantShreddingEnabled; + this.variantLayoutFollowsProperty = variantLayoutFollowsProperty; // We only need this transaction's serialized write context for the factory, not the commit // (commit() builds its own per epoch). Transaction stateTxn = - initialSnapshot + DeltaV2Snapshot$.MODULE$ + .getKernelSnapshot(initialSnapshot) .buildUpdateTableTransaction(DeltaV2Write.getEngineInfo(), Operation.STREAMING_UPDATE) .build(engine); this.dataWriterFactory = dataWriterFactoryBuilder.apply(stateTxn); @@ -110,27 +129,39 @@ public boolean useCommitCoordinator() { @Override public void commit(long epochId, WriterCommitMessage[] messages) { - // Refresh so this epoch commits at latest+1. + // TODO: Expose streaming transaction construction and latest transaction-version lookup + // through the snapshot facade so this path does not depend on SnapshotImpl. + // Kernel-only: needs SnapshotImpl.buildUpdateTableTransaction + // (TransactionBuilder) for the streaming commit, and + // getLatestTransactionVersion for the epoch-skip check. + // One reload, so the skip check, guards, and the transaction below all judge the same snapshot. Snapshot latestSnapshot = snapshotManager.loadLatestSnapshot(); + SnapshotImpl kernelLatestSnapshot = DeltaV2Snapshot$.MODULE$.getKernelSnapshot(latestSnapshot); + + // Skip an already-committed epoch before any guard runs. StreamingWrite.commit may be called + // more than once for one epoch and must be idempotent, so a repeated commit of a committed + // epoch is an unconditional no-op -- its data already landed under the layout in force then, + // and a table change since must not turn that success into a failure. Its executor-written + // files for this repeat are orphaned (VACUUM'd). The next uncommitted epoch still hits the + // guards. + long committedEpoch = + kernelLatestSnapshot.getLatestTransactionVersion(engine, queryId).orElse(-1L); + if (committedEpoch >= epochId) { + logger.info("Skipping already committed epoch {} for query {}", epochId, queryId); + return; + } // TODO(#7140): no implicit type cast and mergeSchema. Fail loudly on a concurrent // schema/protocol change. assertSchemaAndProtocolUnchanged(latestSnapshot); + assertVariantShreddingUnchanged(DeltaV2WriteBuilder.isVariantShreddingEnabled(latestSnapshot)); // TODO(#7140): no self-scan guard. A stream reading and writing the same table commits // as a blind append, skipping the conflict check V1 gets via readWholeTable(). - // Skip an already-committed epoch. Its executor-written files are then orphaned (VACUUM'd). - long committedEpoch = - ((SnapshotImpl) latestSnapshot).getLatestTransactionVersion(engine, queryId).orElse(-1L); - if (committedEpoch >= epochId) { - logger.info("Skipping already committed epoch {} for query {}", epochId, queryId); - return; - } - try { Transaction txn = - latestSnapshot + kernelLatestSnapshot .buildUpdateTableTransaction(DeltaV2Write.getEngineInfo(), Operation.STREAMING_UPDATE) .withTransactionId(queryId, epochId) .build(engine); @@ -149,14 +180,14 @@ public void commit(long epochId, WriterCommitMessage[] messages) { /** Fails the epoch if the fresh snapshot's schema/protocol diverged from the write's baseline. */ private void assertSchemaAndProtocolUnchanged(Snapshot latestSnapshot) { - if (!writeSchema.equals(latestSnapshot.getSchema())) { + if (!writeSchema.equals(latestSnapshot.schema())) { throw new IllegalStateException( "DSv2 streaming write to query " + queryId + " cannot continue: the table schema changed after the stream started. Restart the " + "query to pick up the new schema."); } - if (!writeProtocol.equals(((SnapshotImpl) latestSnapshot).getProtocol())) { + if (!writeProtocol.equals(latestSnapshot.protocol())) { throw new IllegalStateException( "DSv2 streaming write to query " + queryId @@ -165,6 +196,34 @@ private void assertSchemaAndProtocolUnchanged(Snapshot latestSnapshot) { } } + /** + * Fails the epoch if the table's variant shredding property diverged from the write state's. + * + *

Neither of the checks above catches this. Turning shredding off -- by unsetting the property + * or through {@code REORG ... APPLY (UNSHRED VARIANT)} -- leaves the schema untouched and leaves + * the {@code variantShredding} feature in the protocol, so the epoch would otherwise commit files + * in a layout the table no longer asks for, on top of an explicit opt-out. + * + *

Only acts when {@link #variantLayoutFollowsProperty} holds: where the property cannot change + * the file layout (no variant column, kill switch off, or a Spark version without shredding) a + * change to it is irrelevant and must not fail the epoch. + */ + private void assertVariantShreddingUnchanged(boolean latestVariantShreddingEnabled) { + if (variantLayoutFollowsProperty + && latestVariantShreddingEnabled != writeVariantShreddingEnabled) { + throw new IllegalStateException( + "DSv2 streaming write to query " + + queryId + + " cannot continue: the table property " + + DeltaConfigs.ENABLE_VARIANT_SHREDDING().key() + + " changed from " + + writeVariantShreddingEnabled + + " to " + + latestVariantShreddingEnabled + + " after the stream started. Restart the query to write the new layout."); + } + } + @Override public void abort(long epochId, WriterCommitMessage[] messages) { logger.warn( diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/write/DeltaV2Write.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/write/DeltaV2Write.java index aaaec6cb6c6..8f513385242 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/write/DeltaV2Write.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/write/DeltaV2Write.java @@ -18,9 +18,7 @@ import static java.util.Objects.requireNonNull; import com.google.common.annotations.VisibleForTesting; -import io.delta.kernel.Snapshot; import io.delta.kernel.engine.Engine; -import io.delta.spark.internal.v2.snapshot.DeltaSnapshotManager; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -36,6 +34,8 @@ import org.apache.spark.sql.connector.write.Write; import org.apache.spark.sql.connector.write.streaming.StreamingWrite; import org.apache.spark.sql.delta.DeltaOptions; +import org.apache.spark.sql.delta.Snapshot; +import org.apache.spark.sql.delta.v2.interop.DeltaV2SnapshotManager; import org.apache.spark.sql.types.StructType; import org.apache.spark.sql.util.CaseInsensitiveStringMap; @@ -80,10 +80,17 @@ class DeltaV2Write implements Write, RequiresDistributionAndOrdering { private final Configuration hadoopConf; private final String tablePath; private final Snapshot initialSnapshot; - private final DeltaSnapshotManager snapshotManager; + private final DeltaV2SnapshotManager snapshotManager; private final StructType dataSchema; private final StructType partitionSchema; private final String queryId; + /** + * Whether the table opted into shredded variant writes. Resolved by the write builder, which + * holds the snapshot facade that exposes Delta metadata, and carried here so both write modes + * configure the Parquet writer identically. + */ + private final boolean variantShreddingEnabled; + private final LogicalWriteInfo writeInfo; /** @@ -99,10 +106,11 @@ class DeltaV2Write implements Write, RequiresDistributionAndOrdering { Configuration hadoopConf, String tablePath, Snapshot initialSnapshot, - DeltaSnapshotManager snapshotManager, + DeltaV2SnapshotManager snapshotManager, StructType dataSchema, StructType partitionSchema, - LogicalWriteInfo writeInfo) { + LogicalWriteInfo writeInfo, + boolean variantShreddingEnabled) { this.engine = requireNonNull(engine, "engine is null"); this.hadoopConf = requireNonNull(hadoopConf, "hadoopConf is null"); this.tablePath = requireNonNull(tablePath, "tablePath is null"); @@ -112,6 +120,7 @@ class DeltaV2Write implements Write, RequiresDistributionAndOrdering { this.partitionSchema = requireNonNull(partitionSchema, "partitionSchema is null"); this.writeInfo = requireNonNull(writeInfo, "writeInfo is null"); this.queryId = requireNonNull(writeInfo.queryId(), "queryId is null"); + this.variantShreddingEnabled = variantShreddingEnabled; } /** @@ -127,20 +136,42 @@ public BatchWrite toBatch() { // The batch path builds its own driver-side context (DeltaV2BatchWriteContext) and commits a // single transaction off initialSnapshot. return new DeltaV2BatchWrite( - engine, hadoopConf, tablePath, initialSnapshot, dataSchema, partitionSchema, writeInfo); + engine, + hadoopConf, + tablePath, + initialSnapshot, + dataSchema, + partitionSchema, + writeInfo, + variantShreddingEnabled); } @Override public StreamingWrite toStreaming() { rejectUnsupportedStreamingOptions(); - // Build the operation-independent write context once; the streaming write drives its own - // per-epoch transactions (Operation.STREAMING_UPDATE) and reuses buildDataWriterFactory to - // produce the executor write state -- the same setup the batch path uses, no duplication. + // Build the write context once and reuse buildDataWriterFactory for each epoch. That freezes + // the layout it derives from initialSnapshot -- including variant shredding -- which does not + // advance while the query runs, so the streaming commit re-reads the shredding property off + // the reloaded snapshot and fails the epoch on divergence, as it already does for schema and + // protocol. DeltaV2WriteContext context = DeltaV2WriteContext.create( - engine, hadoopConf, tablePath, initialSnapshot, dataSchema, partitionSchema, writeInfo); + engine, + hadoopConf, + tablePath, + initialSnapshot, + dataSchema, + partitionSchema, + writeInfo, + variantShreddingEnabled); return new DeltaV2StreamingWrite( - engine, initialSnapshot, snapshotManager, queryId, context::buildDataWriterFactory); + engine, + initialSnapshot, + snapshotManager, + queryId, + variantShreddingEnabled, + context.variantLayoutFollowsProperty(), + context::buildDataWriterFactory); } /** diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/write/DeltaV2WriteBuilder.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/write/DeltaV2WriteBuilder.java index d5cc5138ded..cb01ceafa18 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/write/DeltaV2WriteBuilder.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/write/DeltaV2WriteBuilder.java @@ -17,19 +17,18 @@ import static java.util.Objects.requireNonNull; -import io.delta.kernel.Snapshot; import io.delta.kernel.engine.Engine; -import io.delta.kernel.internal.util.ColumnMapping; -import io.delta.kernel.internal.util.ColumnMapping.ColumnMappingMode; -import io.delta.spark.internal.v2.snapshot.DeltaSnapshotManager; -import io.delta.spark.internal.v2.utils.SchemaUtils; +import io.delta.kernel.internal.TableConfig; import org.apache.hadoop.conf.Configuration; import org.apache.spark.sql.connector.write.LogicalWriteInfo; import org.apache.spark.sql.connector.write.Write; import org.apache.spark.sql.connector.write.WriteBuilder; import org.apache.spark.sql.delta.DeltaColumnMapping; +import org.apache.spark.sql.delta.DeltaConfigs; +import org.apache.spark.sql.delta.Snapshot; import org.apache.spark.sql.delta.TypeWideningMode; import org.apache.spark.sql.delta.schema.SchemaMergingUtils; +import org.apache.spark.sql.delta.v2.interop.DeltaV2SnapshotManager; import org.apache.spark.sql.types.StructType; /** @@ -47,7 +46,7 @@ public class DeltaV2WriteBuilder implements WriteBuilder { private final String tablePath; private final Configuration hadoopConf; private final Snapshot initialSnapshot; - private final DeltaSnapshotManager snapshotManager; + private final DeltaV2SnapshotManager snapshotManager; private final StructType dataSchema; private final StructType partitionSchema; private final LogicalWriteInfo writeInfo; @@ -56,7 +55,7 @@ public class DeltaV2WriteBuilder implements WriteBuilder { * @param engine Kernel engine (persisted in DeltaV2Table, shared across operations) * @param tablePath filesystem path to the Delta table root * @param hadoopConf Hadoop configuration (with merged table options) - * @param initialSnapshot Kernel snapshot loaded at table construction time + * @param initialSnapshot snapshot loaded at table construction time * @param snapshotManager reloads the latest snapshot; used by the streaming write to build each * epoch's commit against the current table state (see {@link DeltaV2StreamingWrite}) * @param dataSchema the table's data (non-partition) schema, from DeltaV2Table's SchemaProvider @@ -69,7 +68,7 @@ public DeltaV2WriteBuilder( String tablePath, Configuration hadoopConf, Snapshot initialSnapshot, - DeltaSnapshotManager snapshotManager, + DeltaV2SnapshotManager snapshotManager, StructType dataSchema, StructType partitionSchema, LogicalWriteInfo writeInfo) { @@ -87,20 +86,31 @@ public DeltaV2WriteBuilder( public Write build() { validateDataSchema(initialSnapshot, writeInfo.schema()); - // TODO(#7140): support partitioned writes to column-mapped tables. Unpartitioned is supported; - // partitioned writes need partition values keyed by physical name, so reject for now. + // TODO: support partitioned IcebergCompat / materializePartitionColumns writes. if (!partitionSchema.isEmpty()) { - ColumnMappingMode cmMode = - ColumnMapping.getColumnMappingMode(initialSnapshot.getTableProperties()); - if (cmMode != ColumnMappingMode.NONE) { + boolean icebergCompat = + TableConfig.ICEBERG_COMPAT_V2_ENABLED.fromMetadata( + initialSnapshot.metadata().getConfiguration()) + || TableConfig.ICEBERG_COMPAT_V3_ENABLED.fromMetadata( + initialSnapshot.metadata().getConfiguration()); + // Detect the materializePartitionColumns writer feature by its protocol name. + boolean materializePartitionColumns = + initialSnapshot.protocol().getWriterFeatures() != null + && initialSnapshot + .protocol() + .getWriterFeatures() + .contains("materializePartitionColumns"); + if (icebergCompat || materializePartitionColumns) { throw new UnsupportedOperationException( - "DSv2 partitioned writes are not supported on column-mapped Delta tables " - + "(delta.columnMapping.mode = " - + cmMode - + "). Use the V1 write path (format(\"delta\").write()) instead."); + "DSv2 partitioned writes are not supported on tables that materialize partition " + + "columns (IcebergCompat or the materializePartitionColumns feature). Use the V1 " + + "write path (format(\"delta\").write()) instead."); } } + // Resolved once here through the same V1 accessor (alternate keys and default included) and + // passed down as a boolean, so the batch and streaming writes share one decision. + boolean variantShreddingEnabled = isVariantShreddingEnabled(initialSnapshot); // Returns a mode-dispatching Write: toBatch() -> DeltaV2BatchWrite (batch commit off // initialSnapshot), toStreaming() -> DeltaV2StreamingWrite (per-epoch commit off the latest // snapshot via snapshotManager). Both modes share the executor-side write-state construction. @@ -112,14 +122,25 @@ public Write build() { snapshotManager, dataSchema, partitionSchema, - writeInfo); + writeInfo, + variantShreddingEnabled); + } + + /** + * Whether {@code snapshot} opted into shredded variant writes. + * + *

Read through the V1 {@link DeltaConfigs} accessor so alternate property keys and the + * property default are handled exactly as the V1 write path handles them. Package-visible because + * the streaming commit re-reads it off a reloaded snapshot to detect a mid-query change. + */ + static boolean isVariantShreddingEnabled(Snapshot snapshot) { + return (Boolean) DeltaConfigs.ENABLE_VARIANT_SHREDDING().fromMetaData(snapshot.metadata()); } static void validateDataSchema(Snapshot initialSnapshot, StructType dataSchema) { // Validate data schema against table schema using the same utility as V1 // (ImplicitMetadataOperation.updateMetadata -> SchemaMergingUtils.mergeSchemas). - StructType tableSchema = - SchemaUtils.convertKernelSchemaToSparkSchema(initialSnapshot.getSchema()); + StructType tableSchema = initialSnapshot.schema(); // Strip column mapping metadata (physical names, IDs) so mergeSchemas compares // only logical types - matches V1's dropColumnMappingMetadata call. StructType cleanTableSchema = diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/write/DeltaV2WriteContext.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/write/DeltaV2WriteContext.java index 0a57e104c35..e84f9669b54 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/write/DeltaV2WriteContext.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/write/DeltaV2WriteContext.java @@ -15,7 +15,6 @@ */ package io.delta.spark.internal.v2.write; -import io.delta.kernel.Snapshot; import io.delta.kernel.Transaction; import io.delta.kernel.data.Row; import io.delta.kernel.engine.Engine; @@ -34,7 +33,11 @@ import org.apache.hadoop.mapreduce.Job; import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.connector.write.LogicalWriteInfo; +import org.apache.spark.sql.delta.Snapshot; +import org.apache.spark.sql.delta.shims.VariantShreddingShims; +import org.apache.spark.sql.delta.v2.interop.DeltaV2Snapshot$; import org.apache.spark.sql.execution.datasources.OutputWriterFactory; +import org.apache.spark.sql.internal.SQLConf; import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.types.StructType; import org.apache.spark.util.SerializableConfiguration; @@ -72,10 +75,17 @@ static String getEngineInfo() { // The full incoming write-row schema (data + partition columns), used to compute the ordinals of // the data / partition columns within each row for the executor writer. private final StructType writeSchema; - private final io.delta.kernel.types.StructType kernelTableSchema; private final OutputWriterFactory outputWriterFactory; private final SerializableConfiguration serializableHadoopConf; private final String sessionTimeZone; + /** + * Whether the variant shredding layout of this write actually tracks {@code + * delta.enableVariantShredding}. It does only when this Spark version can shred, the write + * shredding kill switch is on, and the data schema contains a variant column; otherwise the + * property cannot change the files, so the streaming guard must not act on a change to it. Frozen + * here because the writer factory froze the same inputs. + */ + private final boolean variantLayoutFollowsProperty; static DeltaV2WriteContext create( Engine engine, @@ -84,9 +94,17 @@ static DeltaV2WriteContext create( Snapshot initialSnapshot, StructType dataSchema, StructType partitionSchema, - LogicalWriteInfo writeInfo) { + LogicalWriteInfo writeInfo, + boolean variantShreddingEnabled) { return new DeltaV2WriteContext( - engine, hadoopConf, tablePath, initialSnapshot, dataSchema, partitionSchema, writeInfo); + engine, + hadoopConf, + tablePath, + initialSnapshot, + dataSchema, + partitionSchema, + writeInfo, + variantShreddingEnabled); } private static void verifySchemaForWrite(DeltaParquetFileFormatV2 format, StructType dataSchema) { @@ -140,9 +158,9 @@ protected DeltaV2WriteContext( Snapshot initialSnapshot, StructType dataSchema, StructType partitionSchema, - LogicalWriteInfo writeInfo) { + LogicalWriteInfo writeInfo, + boolean variantShreddingEnabled) { this.engine = engine; - this.kernelTableSchema = initialSnapshot.getSchema(); SparkSession session = SparkSession.getActiveSession() @@ -161,7 +179,7 @@ protected DeltaV2WriteContext( } DeltaParquetFileFormatV2 format = PartitionUtils.createDeltaParquetFileFormat( - initialSnapshot, + DeltaV2Snapshot$.MODULE$.getKernelSnapshot(initialSnapshot), tablePath, /* optimizationsEnabled */ true, /* useMetadataRowIndex */ Option.empty(), @@ -173,11 +191,89 @@ protected DeltaV2WriteContext( verifySchemaForWrite(format, this.dataSchema); org.apache.spark.sql.execution.datasources.DataSourceUtils.checkFieldNames( format, this.dataSchema); - Map options = writeInfo.options().asCaseSensitiveMap(); + Map parquetOptions = + mergeVariantShreddingOptions( + writeInfo.options().asCaseSensitiveMap(), variantShreddingEnabled); scala.collection.immutable.Map scalaOpts = - ScalaUtils.toScalaMap(options != null ? options : Collections.emptyMap()); + ScalaUtils.toScalaMap(parquetOptions); this.outputWriterFactory = format.prepareWrite(session, job, scalaOpts, this.dataSchema); this.serializableHadoopConf = new SerializableConfiguration(job.getConfiguration()); + this.variantLayoutFollowsProperty = + computeVariantLayoutFollowsProperty(session, this.dataSchema); + } + + /** + * Returns {@code callerOptions} with the table-derived variant shredding options applied, so the + * Parquet writer shreds exactly when the table opted in. + * + *

The option comes from {@link VariantShreddingShims} because the underlying conf exists only + * in newer Spark versions while this class compiles against every supported one. The shim yields + * no entry where the conf is absent, leaving the writer on its default behavior there. + * + *

Any caller spelling that differs from a shim key only in case is dropped first. {@code + * ParquetOptions} reads these through a case-insensitive map, which collapses such keys in map + * iteration order, so leaving both in place would make the effective value depend on iteration + * order rather than on the table property. + * + *

Package-visible so the merge can be asserted directly: the resulting file layout cannot + * distinguish which of two colliding spellings won. + */ + static Map mergeVariantShreddingOptions( + Map callerOptions, boolean variantShreddingEnabled) { + Map merged = + new HashMap<>(callerOptions != null ? callerOptions : Collections.emptyMap()); + Map shreddingOptions = + ScalaUtils.toJavaMap( + VariantShreddingShims.getVariantInferShreddingSchemaOptions(variantShreddingEnabled)); + merged + .keySet() + .removeIf(key -> shreddingOptions.keySet().stream().anyMatch(key::equalsIgnoreCase)); + merged.putAll(shreddingOptions); + return merged; + } + + /** + * Whether a later change to {@code delta.enableVariantShredding} could change this write's file + * layout. Mirrors the three conditions Parquet actually gates the shredded writer on ({@code + * ParquetOptions#inferShreddingForVariant} and {@code ParquetUtils#needShreddingInference}): the + * version shim must supply the inference option (absent on Spark 4.0, where the map is empty), + * the {@code spark.sql.variant.writeShredding.enabled} kill switch must be on, and the data + * schema must have a shreddable variant column. When any is false the property cannot flip the + * layout, so the streaming guard must ignore a change to it rather than fail an epoch that could + * not have shredded anyway. + */ + private static boolean computeVariantLayoutFollowsProperty( + SparkSession session, StructType dataSchema) { + boolean shreddingSupported = + !VariantShreddingShims.getVariantInferShreddingSchemaOptions(true).isEmpty(); + boolean killSwitchOn = + (boolean) session.sessionState().conf().getConf(SQLConf.VARIANT_WRITE_SHREDDING_ENABLED()); + return shreddingSupported && killSwitchOn && hasShreddableVariant(dataSchema); + } + + /** + * Whether {@code schema} has a variant column the shredding writer would actually shred. Matches + * {@code InferVariantShreddingSchema.getPathsToVariant}: a variant at the top level or nested + * only through structs. Variants reached through an array or map element are not shredded, so a + * shredding-property change leaves their files unchanged and must not be treated as layout- + * sensitive. + */ + private static boolean hasShreddableVariant(StructType schema) { + for (StructField field : schema.fields()) { + if (field.dataType() instanceof org.apache.spark.sql.types.VariantType) { + return true; + } + if (field.dataType() instanceof StructType + && hasShreddableVariant((StructType) field.dataType())) { + return true; + } + } + return false; + } + + /** See {@link #variantLayoutFollowsProperty}. */ + boolean variantLayoutFollowsProperty() { + return variantLayoutFollowsProperty; } /** @@ -246,10 +342,6 @@ StructType getPartitionSchema() { return partitionSchema; } - io.delta.kernel.types.StructType getKernelTableSchema() { - return kernelTableSchema; - } - OutputWriterFactory getOutputWriterFactory() { return outputWriterFactory; } diff --git a/spark/v2/src/main/scala/io/delta/spark/internal/v2/DeltaV2Logging.scala b/spark/v2/src/main/scala/io/delta/spark/internal/v2/DeltaV2Logging.scala new file mode 100644 index 00000000000..b6823084270 --- /dev/null +++ b/spark/v2/src/main/scala/io/delta/spark/internal/v2/DeltaV2Logging.scala @@ -0,0 +1,46 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.spark.internal.v2 + +import java.util.function.Supplier + +import org.apache.spark.sql.delta.metering.DeltaLogging + +/** + * Delta V2 specialization of [[DeltaLogging]]. Frame-name suffixes passed here must be fixed + * system-code literals; this trait supplies the shared `Delta` group and `v2.` prefix. + */ +private[v2] trait DeltaV2Logging extends DeltaLogging { + private def deltaV2FrameName(nameSuffix: String): String = "v2." + nameSuffix + + protected final def recordFrameProfile[T]( + nameSuffix: String)(thunk: => T): T = { + super.recordFrameProfile("Delta", deltaV2FrameName(nameSuffix))(thunk) + } + + protected final def recordFrameProfileValue[T]( + nameSuffix: String, + body: Supplier[T]): T = + super.recordFrameProfileValue("Delta", deltaV2FrameName(nameSuffix), body) + + protected final def recordFrameProfileAction( + nameSuffix: String, + body: Runnable): Unit = + super.recordFrameProfileAction("Delta", deltaV2FrameName(nameSuffix), body) +} + +/** Java inheritance bridge for Delta V2 classes that do not already have a superclass. */ +private[v2] abstract class DeltaV2JavaLogging extends DeltaV2Logging diff --git a/spark/v2/src/main/scala/io/delta/spark/internal/v2/catalog/DeltaV2TableShimsWithLogging.scala b/spark/v2/src/main/scala/io/delta/spark/internal/v2/catalog/DeltaV2TableShimsWithLogging.scala new file mode 100644 index 00000000000..75c9582ebbd --- /dev/null +++ b/spark/v2/src/main/scala/io/delta/spark/internal/v2/catalog/DeltaV2TableShimsWithLogging.scala @@ -0,0 +1,26 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.spark.internal.v2.catalog + +import io.delta.spark.internal.v2.DeltaV2Logging + +/** + * Scala inheritance bridge that combines the Java table's Spark-version shims with scoped Delta + * V2 logging. + */ +private[catalog] abstract class DeltaV2TableShimsWithLogging + extends DeltaV2TableShims + with DeltaV2Logging diff --git a/spark/v2/src/main/scala/io/delta/spark/internal/v2/kernel/KernelContext.scala b/spark/v2/src/main/scala/io/delta/spark/internal/v2/kernel/KernelContext.scala new file mode 100644 index 00000000000..1e0c5d3778d --- /dev/null +++ b/spark/v2/src/main/scala/io/delta/spark/internal/v2/kernel/KernelContext.scala @@ -0,0 +1,57 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.spark.internal.v2.kernel + +import org.apache.spark.sql.delta.storage.LogStore +import io.delta.kernel.defaults.engine.{DefaultEngine => KernelDefaultEngine} +import io.delta.kernel.engine.{Engine => KernelEngine} + +import org.apache.spark.sql.SparkSession + +/** + * Table-scoped owner of Kernel resources and deferred Hadoop configuration materialization. + * + * It retains session-invariant filesystem options and the table manager's LogStore. When + * filesystem I/O begins, it binds those options to the active Spark session so session-derived + * settings and credentials are not retained by reusable connector state. Its LogStore and + * lazily-created Engine remain stable across Spark sessions. + */ +private[v2] final class KernelContext( + val sessionInvariantFsOptions: Map[String, String], + val logStore: LogStore) { + require(sessionInvariantFsOptions != null, "sessionInvariantFsOptions must not be null") + require(logStore != null, "logStore must not be null") + + private[v2] def materializeHadoopConf() = + SparkSession.active.sessionState.newHadoopConfWithOptions(sessionInvariantFsOptions) + + private def createDefaultEngine(): KernelEngine = { + val hadoopConf = materializeHadoopConf() + KernelDefaultEngine.create(hadoopConf) + } + + private lazy val kernelDefaultEngine = createDefaultEngine() + + private[v2] def getDefaultEngine(): KernelEngine = kernelDefaultEngine +} + +private[v2] object KernelContext { + def empty(logStore: LogStore): KernelContext = new KernelContext(Map.empty, logStore) + + def apply(sessionInvariantFsOptions: Map[String, String], logStore: LogStore): KernelContext = + new KernelContext(sessionInvariantFsOptions, logStore) +} diff --git a/spark/v2/src/main/scala/io/delta/spark/internal/v2/kernel/KernelEngineFactory.scala b/spark/v2/src/main/scala/io/delta/spark/internal/v2/kernel/KernelEngineFactory.scala new file mode 100644 index 00000000000..3a368df57db --- /dev/null +++ b/spark/v2/src/main/scala/io/delta/spark/internal/v2/kernel/KernelEngineFactory.scala @@ -0,0 +1,30 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.spark.internal.v2.kernel + +import org.apache.hadoop.conf.Configuration +import io.delta.kernel.defaults.engine.{DefaultEngine => KernelDefaultEngine} +import io.delta.kernel.engine.{Engine => KernelEngine} + +/** Factory for creating the default Kernel [[KernelEngine]] used by the DSv2 connector. */ +object KernelEngineFactory { + + /** Builds the backend-appropriate default engine. */ + def createDefaultEngine(hadoopConf: Configuration): KernelEngine = { + KernelDefaultEngine.create(hadoopConf) + } +} diff --git a/spark/v2/src/main/scala/io/delta/spark/internal/v2/read/DeltaV2Batch.scala b/spark/v2/src/main/scala/io/delta/spark/internal/v2/read/DeltaV2Batch.scala new file mode 100644 index 00000000000..7a5b6a8b0ed --- /dev/null +++ b/spark/v2/src/main/scala/io/delta/spark/internal/v2/read/DeltaV2Batch.scala @@ -0,0 +1,144 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.spark.internal.v2.read + +import java.time.ZoneId +import java.util.Objects + +import org.apache.spark.sql.delta.Snapshot +import org.apache.spark.sql.delta.stats.DeltaScan +import io.delta.spark.internal.v2.DeltaV2Logging +import org.apache.spark.sql.delta.v2.interop.DeltaV2Snapshot +import io.delta.spark.internal.v2.utils.PartitionUtils +import org.apache.hadoop.conf.Configuration +import io.delta.kernel.expressions.{Predicate => KernelPredicate} +import io.delta.kernel.internal.{SnapshotImpl => KernelSnapshot} + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.connector.read.{Batch, InputPartition, PartitionReaderFactory} +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.sources.Filter +import org.apache.spark.sql.types.StructType + +/** + * Package-private batch implementation for Delta's Spark DataSource V2 read path. + * + * This class must remain package-private so callers outside `v2.read` depend only on Spark's + * public connector interfaces instead of coupling to Delta's internal V2 implementation. + */ +// Fields read across instances in equals/hashCode are `private val`, since object-private +// `private[this]` cannot read `that.field`. Other retained fields are `private[this]`; the two +// filter-array params are consumed only to derive the equality sets, so they take no `val`. +private[read] final class DeltaV2Batch( + private val initialSnapshot: Snapshot, + private val dataSchema: StructType, + private val partitionSchema: StructType, + private val readDataSchema: StructType, + private[this] val ddlOrderedReadOutputSchema: StructType, + private val deltaScan: DeltaScan, + private[this] val tablePath: String, + private[this] val zoneId: ZoneId, + kernelPushedFilters: Array[KernelPredicate], + // Data-column filters only: the sole filters passed to the Parquet reader factory. Partition + // columns are not stored in Parquet; their values come from PartitionedFile.partitionValues. + private[this] val dataFilters: Array[Filter], + // Full filter set (partition + data). Used only to build the equals/hashCode identity set so + // batches selecting different file sets stay distinct; never passed to the reader factory. + allFilters: Array[Filter], + private[this] val readerOptions: Map[String, String], + private[this] val hadoopConf: Configuration) + extends Batch + with DeltaV2Logging { + + // Use Objects.requireNonNull (throws NullPointerException) rather than Scala's require (throws + // IllegalArgumentException) to preserve the exact null-check behavior of the original Java class. + Objects.requireNonNull(initialSnapshot, "initialSnapshot is null") + Objects.requireNonNull(dataSchema, "dataSchema is null") + Objects.requireNonNull(partitionSchema, "partitionSchema is null") + Objects.requireNonNull(readDataSchema, "readDataSchema is null") + Objects.requireNonNull(ddlOrderedReadOutputSchema, "ddlOrderedReadOutputSchema is null") + Objects.requireNonNull(deltaScan, "deltaScan is null") + Objects.requireNonNull(tablePath, "tablePath is null") + Objects.requireNonNull(zoneId, "zoneId is null") + Objects.requireNonNull(kernelPushedFilters, "kernelPushedFilters is null") + Objects.requireNonNull(dataFilters, "dataFilters is null") + Objects.requireNonNull(allFilters, "allFilters is null") + Objects.requireNonNull(readerOptions, "readerOptions is null") + Objects.requireNonNull(hadoopConf, "hadoopConf is null") + + // Order-insensitive equality keys. Filters are AND-ed, so order does not affect the scan. + private val kernelPushedFilterSet = kernelPushedFilters.toSet + private val filterSet = allFilters.toSet + private[this] val sqlConf: SQLConf = SQLConf.get + + private def kernelSnapshot: KernelSnapshot = + DeltaV2Snapshot.getKernelSnapshot(initialSnapshot) + + override def planInputPartitions(): Array[InputPartition] = { + recordFrameProfile("batchScan.planInputPartitions") { + PartitionUtils.planInputPartitions( + SparkSession.active, + deltaScan, + partitionSchema, + tablePath, + zoneId, + hadoopConf, + sqlConf) + } + } + + override def createReaderFactory(): PartitionReaderFactory = { + // Non-CDC plain table scan. Write-time CDF streaming reads route through + // DeltaV2MicroBatchStream; read-time CDF batch reads use a dedicated batch implementation. + recordFrameProfile("batchScan.createReaderFactory") { + PartitionUtils.createDeltaParquetReaderFactory( + kernelSnapshot, + dataSchema, + partitionSchema, + readDataSchema, + ddlOrderedReadOutputSchema, + dataFilters, + readerOptions, + hadoopConf, + sqlConf) + } + } + + override def equals(obj: Any): Boolean = obj match { + case that: DeltaV2Batch => + (this eq that) || ( + kernelSnapshot == that.kernelSnapshot && + readDataSchema == that.readDataSchema && + dataSchema == that.dataSchema && + partitionSchema == that.partitionSchema && + kernelPushedFilterSet == that.kernelPushedFilterSet && + filterSet == that.filterSet && + deltaScan.files.size == that.deltaScan.files.size + ) + case _ => false + } + + override def hashCode(): Int = + Seq( + kernelSnapshot, + readDataSchema, + dataSchema, + partitionSchema, + kernelPushedFilterSet, + filterSet, + deltaScan.files.size).hashCode() +} diff --git a/spark/v2/src/main/scala/io/delta/spark/internal/v2/read/DeltaV2ScanBuilder.scala b/spark/v2/src/main/scala/io/delta/spark/internal/v2/read/DeltaV2ScanBuilder.scala index e535fdc0852..47338223380 100644 --- a/spark/v2/src/main/scala/io/delta/spark/internal/v2/read/DeltaV2ScanBuilder.scala +++ b/spark/v2/src/main/scala/io/delta/spark/internal/v2/read/DeltaV2ScanBuilder.scala @@ -17,28 +17,42 @@ package io.delta.spark.internal.v2.read import java.util.{Locale, Objects, Optional, OptionalInt} +import java.util.function.Supplier -import scala.jdk.CollectionConverters._ - +import io.delta.kernel.engine.Engine import io.delta.spark.internal.v2.read.cdc.CDCSchemaContext -import io.delta.spark.internal.v2.snapshot.DeltaSnapshotManager -import io.delta.spark.internal.v2.utils.ExpressionUtils -import io.delta.kernel.Snapshot -import io.delta.kernel.expressions.{And, Predicate} -import org.apache.spark.sql.connector.read.{Scan, ScanBuilder, Statistics, SupportsPushDownFilters, SupportsPushDownLimit, SupportsPushDownRequiredColumns} -import org.apache.spark.sql.sources.Filter +import org.apache.spark.sql.delta.Snapshot +import org.apache.spark.sql.delta.stats.DeltaScan +import io.delta.spark.internal.v2.DeltaV2Logging +import org.apache.spark.sql.delta.v2.interop.DeltaV2Snapshot +import org.apache.spark.sql.delta.v2.interop.DeltaV2SnapshotManager + +import org.apache.spark.sql.catalyst.catalog.CatalogTable +import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression, ExprId} +import org.apache.spark.sql.connector.expressions.filter.Predicate +import org.apache.spark.sql.connector.read.{Scan, ScanBuilder, Statistics, SupportsPushDownLimit, SupportsPushDownRequiredColumns} +import org.apache.spark.sql.execution.datasources.DataSourceUtils +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.internal.connector.SupportsPushDownCatalystFilters import org.apache.spark.sql.types.StructType import org.apache.spark.sql.util.CaseInsensitiveStringMap /** - * Package-private scan builder implementation for Delta's Spark DataSource V2 read path. + * A Spark ScanBuilder for the Delta v2 connector. In [[build]] it captures the inputs for a + * Kernel-backed [[DeltaV2Snapshot]] and hands `DeltaV2Scan` a lazy [[DeltaScan]] supplier. Batch + * planning invokes the supplier to run V1 data skipping; streaming never needs that batch work. * - * This class must remain package-private so callers outside `v2.read` depend only on Spark's - * public connector interfaces instead of coupling to Delta's internal V2 implementation. + * Filter pushdown implements [[SupportsPushDownCatalystFilters]]: the trait's `Seq[Expression]` + * signature erases to a version-specific `Seq` type across Scala 2.12 / 2.13, which a single Java + * override cannot satisfy, so this builder is Scala. Pushed filters are split into partition and + * data filters and kept as Catalyst expressions (fed to `DeltaV2Snapshot.filesForScan`); they are + * not translated to data source predicates, so `pushedFilters` reports none. * - * @param tableName the name of the table - * @param initialSnapshot Snapshot created during connector setup + * @param tableName the name of the table (used only for identification) + * @param initialSnapshot Kernel snapshot created during connector setup + * @param kernelEngine the Kernel engine used to read scan files + * @param catalogTable the catalog table this scan resolved from, if any * @param snapshotManager the snapshot manager for this table * @param dataSchema the data schema (non-partition columns) * @param partitionSchema the partition schema @@ -49,20 +63,25 @@ import org.apache.spark.sql.util.CaseInsensitiveStringMap private[read] class DeltaV2ScanBuilder( tableName: String, initialSnapshot: Snapshot, - snapshotManager: DeltaSnapshotManager, + kernelEngine: Engine, + catalogTable: Optional[CatalogTable], + snapshotManager: DeltaV2SnapshotManager, dataSchema: StructType, partitionSchema: StructType, tableSchema: StructType, catalogStats: Optional[Statistics], options: CaseInsensitiveStringMap) - extends ScanBuilder - with SupportsPushDownRequiredColumns - with SupportsPushDownFilters - with SupportsPushDownLimit { + extends ScanBuilder + with SupportsPushDownRequiredColumns + with SupportsPushDownCatalystFilters + with SupportsPushDownLimit + with DeltaV2Logging { // Use Objects.requireNonNull (throws NullPointerException) rather than Scala's require (throws // IllegalArgumentException) to preserve the exact null-check behavior of the original Java class. Objects.requireNonNull(initialSnapshot, "initialSnapshot is null") + Objects.requireNonNull(kernelEngine, "kernelEngine is null") + Objects.requireNonNull(catalogTable, "catalogTable is null") Objects.requireNonNull(snapshotManager, "snapshotManager is null") Objects.requireNonNull(dataSchema, "dataSchema is null") Objects.requireNonNull(partitionSchema, "partitionSchema is null") @@ -70,103 +89,56 @@ private[read] class DeltaV2ScanBuilder( Objects.requireNonNull(catalogStats, "catalogStats is null") Objects.requireNonNull(options, "options is null") - private var kernelScanBuilder: io.delta.kernel.ScanBuilder = - initialSnapshot.getScanBuilder - private val partitionColumnSet: Set[String] = partitionSchema.fields.map(_.name.toLowerCase(Locale.ROOT)).toSet private var requiredDataSchema: StructType = dataSchema - // pushedKernelPredicates: Predicates that have been pushed down to the Delta Kernel for - // evaluation. - // pushedSparkFilters: The same pushed predicates, but represented using Spark's Filter API - // (needed because Spark operates on Filter objects while the Kernel uses Predicate). - private var pushedKernelPredicates: Array[Predicate] = Array.empty - private var pushedSparkFilters: Array[Filter] = Array.empty - private var dataFilters: Array[Filter] = Array.empty - // Tracks whether any filter still needs to be applied after the scan. + // Pushed partition + data catalyst filters (their union drives V1 filesForScan skipping). + private var partitionCatalystFilters: Array[Expression] = Array.empty + private var dataCatalystFilters: Array[Expression] = Array.empty + // Tracks whether any filter still needs to be applied after the scan. Data filters are not + // row-exact (min/max skipping only), so they leave a post-scan residual; partition filters are + // exact and leave none. Used to decide when a pushed LIMIT is safe (see build()). private var hasPostScanResidualFilters: Boolean = false private var pushedLimit: OptionalInt = OptionalInt.empty() - override def pruneColumns(requiredSchema: StructType): Unit = { - Objects.requireNonNull(requiredSchema, "requiredSchema is null") - // CDC columns are injected later by CDCReadFunction, so strip them here. - requiredDataSchema = new StructType( - requiredSchema.fields.filter { f => - val name = f.name.toLowerCase(Locale.ROOT) - !partitionColumnSet.contains(name) && !CDCSchemaContext.isCDCColumn(name) - }) - } - - override def pushFilters(filters: Array[Filter]): Array[Filter] = { - val kernelSupportedFilters = new java.util.ArrayList[Filter]() - val convertedKernelPredicates = new java.util.ArrayList[Predicate]() - val dataFilterList = new java.util.ArrayList[Filter]() - val postScanFilters = new java.util.ArrayList[Filter]() - - filters.foreach { filter => - val classification = - ExpressionUtils.classifyFilter(filter, partitionColumnSet.asJava, tableSchema) - // Collect kernel predicates if supported. - if (classification.isKernelSupported) { - convertedKernelPredicates.add(classification.kernelPredicate.get()) - if (!classification.isPartialConversion) { - // Add filter to kernelSupportedFilters if it is fully converted. - // TODO: add partially converted Spark filter as well; right now we only have the - // partially converted kernel predicate. - kernelSupportedFilters.add(filter) - } - } - - // Collect data filters. - if (classification.isDataFilter) { - dataFilterList.add(filter) - } - - // Collect post-scan filters. Filters with the following characteristics need to be evaluated - // after the delta kernel scan: - // 1. filters that are not supported by delta kernel, thus kernel cannot apply them during - // scan; - // 2. filters that are not fully converted to kernel predicates, thus the unconverted part - // needs to be evaluated after scan; - // 3. filters that are data filters, as kernel only evaluates data filters based on min/max - // stats, thus need to be evaluated with actual data after scan. - // - // Fully converted partition filters are used to prune partitions during scan. Only the - // partitions that satisfy the filters will be scanned, so no need for post-scan evaluation. - if (!classification.isKernelSupported - || classification.isPartialConversion - || classification.isDataFilter) { - postScanFilters.add(filter) - } + override def pushFilters(filters: Seq[Expression]): Seq[Expression] = + recordFrameProfile("scanBuilder.pushFilters") { + val (partitionFilters, dataFilters) = + DataSourceUtils.getPartitionFiltersAndDataFilters(partitionSchema, filters) + partitionCatalystFilters = partitionFilters.toArray + dataCatalystFilters = dataFilters.toArray + // Data filters need post-scan evaluation (min/max skipping is not row-exact); partition + // filters are exact and need no re-evaluation. ScanBuilder mutations can be cumulative, so a + // later pushFilters call must not make an earlier residual safe to ignore. + hasPostScanResidualFilters |= dataFilters.nonEmpty + dataFilters } - pushedSparkFilters = kernelSupportedFilters.toArray(new Array[Filter](0)) - pushedKernelPredicates = convertedKernelPredicates.toArray(new Array[Predicate](0)) - if (pushedKernelPredicates.nonEmpty) { - val kernelAnd = pushedKernelPredicates.reduce((left, right) => new And(left, right)) - kernelScanBuilder = kernelScanBuilder.withFilter(kernelAnd) - } - dataFilters = dataFilterList.toArray(new Array[Filter](0)) - val postScan = postScanFilters.toArray(new Array[Filter](0)) - // ScanBuilder mutations can be cumulative, so a later pushFilters call should not make an - // earlier residual safe to ignore. - hasPostScanResidualFilters |= postScan.length > 0 - postScan - } + // Filters are kept as Catalyst expressions and are not translated to data source predicates. + override def pushedFilters: Array[Predicate] = Array.empty - override def pushedFilters(): Array[Filter] = pushedSparkFilters + override def pruneColumns(requiredSchema: StructType): Unit = + recordFrameProfile("scanBuilder.pruneColumns") { + Objects.requireNonNull(requiredSchema, "requiredSchema is null") + // CDC columns are injected later by CDCReadFunction, so strip them here. + requiredDataSchema = new StructType( + requiredSchema.fields.filter { f => + val name = f.name.toLowerCase(Locale.ROOT) + !partitionColumnSet.contains(name) && !CDCSchemaContext.isCDCColumn(name) + }) + } /** * Accepts a LIMIT hint from Spark's optimizer. * - * Always returns `true`: the connector treats the limit as a best-effort hint, using per-file - * `numRecords` statistics to stop adding files to the scan plan once enough rows have been - * accumulated (see [[DeltaV2Scan]]). Because pruning happens at file granularity, the planned - * scan may still return more rows than requested (for example, a single file with 1,000 rows for - * LIMIT 5), so `isPartiallyPushed()` is left at its default of `true` and Spark keeps its limit - * operators as a backstop. + * Always returns `true`: the connector treats the limit as a best-effort hint. When the limit is + * kept (see [[build]]), file selection routes through V1's limit-aware `filesForScan`, which + * stops adding files once enough logical rows have accumulated. Because pruning happens at file + * granularity, the planned scan may still return more rows than requested (for example, a single + * file with 1,000 rows for LIMIT 5), so `isPartiallyPushed()` is left at its default of `true` + * and Spark keeps its limit operators as a backstop. * * @param limit the row limit requested by Spark which must be non-negative. */ @@ -181,26 +153,69 @@ private[read] class DeltaV2ScanBuilder( // isPartiallyPushed() intentionally uses the interface default (true). Because pruning happens at // file granularity, the scan may produce more rows than requested, so Spark must reapply LIMIT. - override def build(): Scan = { - // Spark's V2ScanRelationPushDown only pushes a limit when no post-scan residual remains. It - // matches PhysicalOperation(_, Nil, sHolder). Retain this defensive check for direct callers - // that may invoke the ScanBuilder methods in a different order. - val effectiveLimit = if (hasPostScanResidualFilters) OptionalInt.empty() else pushedLimit + override def build(): Scan = + recordFrameProfile("scanBuilder.build") { + // Capture the planning inputs here, but defer constructing the Kernel-backed V1 snapshot and + // running filesForScan until DeltaV2Scan actually plans a batch. A MicroBatchStream performs + // its own snapshot and commit-range reads and never consumes batch-selected files. + // + // Ask for per-file record counts exactly when DeltaV2Scan will consume them for scan + // metadata. + // This must stay in sync with DeltaV2Scan.arePlanStatsEnabled, which gates the reading side. + // Note this only affects the no-limit branch below -- V1's limit-aware filesForScan takes no + // keepNumRecords and always drops per-file stats, so DeltaV2Scan falls back to + // DeltaScan.scanned.rows there. + val sqlConf = SQLConf.get + val keepNumRecords = sqlConf.cboEnabled || sqlConf.planStatsEnabled + val partitionFiltersForScan = partitionCatalystFilters.toIndexedSeq + val catalystFilters = partitionFiltersForScan ++ dataCatalystFilters - new DeltaV2Scan( - snapshotManager, - initialSnapshot, - tableSchema, - dataSchema, - partitionSchema, - requiredDataSchema, - pushedKernelPredicates, - dataFilters, - kernelScanBuilder.build(), - catalogStats, - options, - effectiveLimit) - } + // Spark's V2ScanRelationPushDown only pushes a limit when no post-scan residual remains (it + // matches PhysicalOperation(_, Nil, sHolder)), so an effective limit implies only exact + // partition filters are present. Retain this residual check for direct callers that may + // invoke the ScanBuilder methods in a different order. + val effectiveLimit = if (hasPostScanResidualFilters) OptionalInt.empty() else pushedLimit + + val deltaScanSupplier = new Supplier[DeltaScan] { + override def get(): DeltaScan = { + val snapshot = initialSnapshot.asInstanceOf[DeltaV2Snapshot] + + // When a limit is pushed, route selection through V1's limit-aware filesForScan. It + // requires partition-only filters (guaranteed above) and prunes files by accumulating + // record counts until the limit is satisfied. + def selectFiles(): DeltaScan = + if (effectiveLimit.isPresent) { + recordFrameProfile("filesForScan.limitAndFilters") { + snapshot.filesForScan( + effectiveLimit.getAsInt.toLong, + partitionFiltersForScan) + } + } else { + recordFrameProfile("filesForScan.filters") { + snapshot.filesForScan(catalystFilters, keepNumRecords) + } + } + + // Select files inline on this path. + selectFiles() + } + } + + val scan = new DeltaV2Scan( + snapshotManager, + initialSnapshot, + tableSchema, + dataSchema, + partitionSchema, + requiredDataSchema, + deltaScanSupplier, + dataCatalystFilters, + partitionCatalystFilters, + catalogStats, + options, + effectiveLimit) + scan + } private[read] def getOptions: CaseInsensitiveStringMap = options @@ -214,15 +229,23 @@ private[read] class DeltaV2ScanBuilder( private[read] object DeltaV2ScanBuilder { /** - * Factory used by the Java `DeltaV2ScanUtils`. A companion-object static forwarder (rather than a - * direct `new` from Java) so the OSS sbt/zinc mixed Java+Scala compilation in the spark/v2 module - * can resolve the reference: Java code calling a Scala `object` method compiles cleanly, whereas - * Java `new`-ing this Scala class -- which itself references same-module Java -- does not stage. + * Normalizes attribute references so scan equality compares logical filters across analyses. + * Resets both the ExprId and the qualifier: the same table column can surface with different + * qualifiers (e.g. a CTE alias vs the fully-qualified table name) yet be logically identical, + * and leaving the qualifier in place would make two otherwise-equal scans compare unequal and + * defeat exchange/subquery reuse. */ + def normalizeForEquality(expr: Expression): Expression = expr.transformUp { + case a: AttributeReference => a.withExprId(ExprId(0L)).withQualifier(Nil) + } + + /** Factory used by the Java DeltaV2ScanUtils. */ def create( tableName: String, initialSnapshot: Snapshot, - snapshotManager: DeltaSnapshotManager, + kernelEngine: Engine, + catalogTable: Optional[CatalogTable], + snapshotManager: DeltaV2SnapshotManager, dataSchema: StructType, partitionSchema: StructType, tableSchema: StructType, @@ -231,10 +254,59 @@ private[read] object DeltaV2ScanBuilder { new DeltaV2ScanBuilder( tableName, initialSnapshot, + kernelEngine, + catalogTable, snapshotManager, dataSchema, partitionSchema, tableSchema, catalogStats, options) + + /** + * Adapter for callers that hold a Kernel snapshot but push no filters -- currently Delta Sharing + * DSv2, whose batch scan builds a full-table read (filter pushdown is a follow-up milestone). + * + * Rather than duplicate the DeltaScan-production logic (which must construct the `private[v2]` + * [[DeltaV2Snapshot]] and run the V1 data-skipping path), this reuses the normal build path with + * no pushed filters: an empty filter set makes `filesForScan` take its no-filter fast path and + * return a full-table [[DeltaScan]]. Column pruning is applied so the returned scan reads only + * `requiredSchema`. + * + * @param tableName the table name (used only for identification) + * @param snapshot the Kernel snapshot to read + * @param kernelEngine the Kernel engine + * @param snapshotManager the snapshot manager for this table + * @param dataSchema the data schema (non-partition columns) + * @param partitionSchema the partition schema + * @param tableSchema the full table schema + * @param requiredSchema the columns actually needed (partition/CDC columns are stripped) + * @param catalogStats optional V2 Statistics converted from catalog stats + * @return a `DeltaV2Scan` over the full table (no data skipping) + */ + def forUnfilteredScan( + tableName: String, + snapshot: Snapshot, + kernelEngine: Engine, + snapshotManager: DeltaV2SnapshotManager, + dataSchema: StructType, + partitionSchema: StructType, + tableSchema: StructType, + requiredSchema: StructType, + catalogStats: Optional[Statistics]): Scan = { + // No CatalogTable: a full-table, no-filter scan does not use catalogTable (it only feeds the + val builder = new DeltaV2ScanBuilder( + tableName, + snapshot, + kernelEngine, + Optional.empty[CatalogTable](), + snapshotManager, + dataSchema, + partitionSchema, + tableSchema, + catalogStats, + CaseInsensitiveStringMap.empty()) + builder.pruneColumns(requiredSchema) + builder.build() + } } diff --git a/spark/v2/src/main/scala/io/delta/spark/internal/v2/tablemanager/CachedSnapshotManager.scala b/spark/v2/src/main/scala/io/delta/spark/internal/v2/tablemanager/CachedSnapshotManager.scala new file mode 100644 index 00000000000..458bfb1caf8 --- /dev/null +++ b/spark/v2/src/main/scala/io/delta/spark/internal/v2/tablemanager/CachedSnapshotManager.scala @@ -0,0 +1,302 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.spark.internal.v2.tablemanager + +import java.util.{Objects, Optional} +import java.util.concurrent.atomic.AtomicReference +import java.util.concurrent.locks.ReentrantLock + +import scala.jdk.OptionConverters._ + +import org.apache.spark.sql.delta.Snapshot +import org.apache.spark.sql.delta.sources.DeltaSQLConf +import io.delta.spark.internal.v2.DeltaV2Logging +import org.apache.spark.sql.delta.v2.interop.{ + DeltaV2QueryContext, + DeltaV2SnapshotManager +} +import io.delta.spark.internal.v2.kernel.KernelContext +import io.delta.spark.internal.v2.snapshot.SnapshotManagerFactory +import org.apache.hadoop.fs.Path +import io.delta.kernel.{CommitRange => KernelCommitRange} +import io.delta.kernel.engine.{Engine => KernelEngine} +import io.delta.kernel.internal.{ + DeltaHistoryManager => KernelDeltaHistoryManager +} + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.catalog.CatalogTable + +/** + * Table-scoped snapshot manager that caches the [[DeltaV2Snapshot]] + * and serves it to every operation on the same table. + * + * [[currentSnapshot]] remains `null` until the first successful load. The first installed snapshot + * records its table identity. A subsequent table identity replaces the cached snapshot and is + * logged, matching DeltaLog's drop-and-recreate behavior. Stale entries refresh through an uncached + * snapshot manager; dependent modules may layer incremental refresh strategies on this base. + */ +private[tablemanager] class CachedSnapshotManager( + tablePath: Path, + kernelContext: KernelContext, + latestCatalogTable: AtomicReference[CatalogTable]) + extends DeltaV2SnapshotManager + with DeltaV2Logging { + + private case class CachedSnapshot(snapshot: Snapshot, validatedAtMs: Long) + + private val snapshotLock = new ReentrantLock() + @volatile private var currentSnapshot: CachedSnapshot = _ + + // === DeltaV2SnapshotManager implementation ================================ + + override def loadLatestSnapshot(): Snapshot = { + recordFrameProfile("cachedSnapshotManager.loadLatestSnapshot") { + loadLatestSnapshotInternal(Option(latestCatalogTable.get())) + } + } + + override def loadLatestSnapshot(queryContext: DeltaV2QueryContext): Snapshot = { + Objects.requireNonNull(queryContext, "queryContext is null") + recordFrameProfile("cachedSnapshotManager.loadLatestSnapshot") { + loadLatestSnapshotInternal(queryContext.catalogTableOpt) + } + } + + override def loadSnapshotAt(version: Long): Snapshot = { + recordFrameProfile("cachedSnapshotManager.loadSnapshotAt") { + loadSnapshotAtInternal(version, Option(latestCatalogTable.get())) + } + } + + override def loadSnapshotAt( + version: Long, + queryContext: DeltaV2QueryContext): Snapshot = { + Objects.requireNonNull(queryContext, "queryContext is null") + recordFrameProfile("cachedSnapshotManager.loadSnapshotAt") { + loadSnapshotAtInternal(version, queryContext.catalogTableOpt) + } + } + + override def getActiveCommitAtTime( + timestampMillis: Long, + canReturnLastCommit: Boolean, + mustBeRecreatable: Boolean, + canReturnEarliestCommit: Boolean): KernelDeltaHistoryManager.Commit = + withUncachedSnapshotManager(Option(latestCatalogTable.get()))( + _.getActiveCommitAtTime( + timestampMillis, + canReturnLastCommit, + mustBeRecreatable, + canReturnEarliestCommit)) + + override def getActiveCommitAtTime( + timestampMillis: Long, + canReturnLastCommit: Boolean, + mustBeRecreatable: Boolean, + canReturnEarliestCommit: Boolean, + queryContext: DeltaV2QueryContext): KernelDeltaHistoryManager.Commit = { + Objects.requireNonNull(queryContext, "queryContext is null") + withUncachedSnapshotManager(queryContext.catalogTableOpt)( + _.getActiveCommitAtTime( + timestampMillis, + canReturnLastCommit, + mustBeRecreatable, + canReturnEarliestCommit)) + } + + override def checkVersionExists( + version: Long, + mustBeRecreatable: Boolean, + allowOutOfRange: Boolean): Unit = + withUncachedSnapshotManager(Option(latestCatalogTable.get()))( + _.checkVersionExists(version, mustBeRecreatable, allowOutOfRange)) + + override def checkVersionExists( + version: Long, + mustBeRecreatable: Boolean, + allowOutOfRange: Boolean, + queryContext: DeltaV2QueryContext): Unit = { + Objects.requireNonNull(queryContext, "queryContext is null") + withUncachedSnapshotManager(queryContext.catalogTableOpt)( + _.checkVersionExists(version, mustBeRecreatable, allowOutOfRange)) + } + + override def getTableChanges( + kernelEngine: KernelEngine, + startVersion: Long, + endVersion: Optional[java.lang.Long]): KernelCommitRange = + withUncachedSnapshotManager(Option(latestCatalogTable.get()))( + _.getTableChanges(kernelEngine, startVersion, endVersion)) + + override def getTableChanges( + kernelEngine: KernelEngine, + startVersion: Long, + endVersion: Optional[java.lang.Long], + queryContext: DeltaV2QueryContext): KernelCommitRange = { + Objects.requireNonNull(queryContext, "queryContext is null") + withUncachedSnapshotManager(queryContext.catalogTableOpt)( + _.getTableChanges(kernelEngine, startVersion, endVersion)) + } + + // === Snapshot lifecycle =================================================== + + // Eviction drops persisted StateCache data without invalidating escaped snapshots or managers. + private[tablemanager] def retire(): Unit = withSnapshotLockInterruptibly { + retireSnapshotInternal(currentSnapshot) + } + + // === Acquisition ========================================================== + + private def loadLatestSnapshotInternal(catalogTableOpt: Option[CatalogTable]): Snapshot = { + val requiredFreshAfter = latestSnapshotFreshnessThreshold() + recordFrameProfile("cachedSnapshotManager.loadLatestSnapshotInternal") { + val existing = currentSnapshot + if (isFresh(existing, requiredFreshAfter)) { + return existing.snapshot + } + withSnapshotLockInterruptibly { + val current = currentSnapshot + if (isFresh(current, requiredFreshAfter)) { + current.snapshot + } else { + rebuildAndInstallInternal(catalogTableOpt) + } + } + } + } + + private def rebuildAndInstallInternal(catalogTableOpt: Option[CatalogTable]): Snapshot = { + recordFrameProfile("cachedSnapshotManager.rebuild") { + val validationStartedAt = System.currentTimeMillis() + val existing = currentSnapshot + val refreshed = CachedSnapshot( + withUncachedSnapshotManager(catalogTableOpt)(_.loadLatestSnapshot()), + validationStartedAt) + val sameTable = + existing != null && existing.snapshot.metadata.id == refreshed.snapshot.metadata.id + // Reusing existing.snapshot at the same table ID and version preserves its materialized + // statistics and other cached state. Always installing refreshed would discard that work + // even when the table has not changed. + // Loading and installation both hold snapshotLock, so another refresh cannot install a + // newer snapshot while this load runs. External deletion of tail commits can still make + // refreshed older than existing; install refreshed in that case, matching DeltaLog. + if (sameTable && existing.snapshot.version == refreshed.snapshot.version) { + val validatedAt = math.max(validationStartedAt, existing.validatedAtMs) + currentSnapshot = CachedSnapshot(existing.snapshot, validatedAt) + retireSnapshotInternal(refreshed) + existing.snapshot + } else { + if (existing != null && !sameTable) { + logWarning( + s"Table identity changed while refreshing snapshot: " + + s"previous=${existing.snapshot.metadata.id}, " + + s"current=${refreshed.snapshot.metadata.id}") + } + currentSnapshot = refreshed + retireSnapshotInternal(existing) + refreshed.snapshot + } + } + } + + private def loadSnapshotAtInternal( + version: Long, + catalogTableOpt: Option[CatalogTable]): Snapshot = { + val existing = currentSnapshot + // Exact-version time travel reuses the cached facade; latest-table freshness is irrelevant. + if (existing != null && version == existing.snapshot.version) { + return existing.snapshot + } + // Linearize upper-bound discovery and any exact fallback under one lock acquisition. + val upperBound = withSnapshotLockInterruptibly { + val current = currentSnapshot + if (current != null && version == current.snapshot.version) { + return current.snapshot + } + val refreshed = + if (isFresh(current, latestSnapshotFreshnessThreshold(), Some(version))) current.snapshot + else rebuildAndInstallInternal(catalogTableOpt) + // If latest still trails the requested version, attempt the exact load before rejecting it. + if (version > refreshed.version) { + val loaded = CachedSnapshot( + withUncachedSnapshotManager(catalogTableOpt)(_.loadSnapshotAt(version)), + validatedAtMs = -1L) + val previous = currentSnapshot + if (previous != null && previous.snapshot.metadata.id != loaded.snapshot.metadata.id) { + logWarning( + s"Table identity changed while loading snapshot at version $version: " + + s"previous=${previous.snapshot.metadata.id}, current=${loaded.snapshot.metadata.id}") + } + // An exact-version load proves this snapshot exists, but not that it is the latest. + currentSnapshot = loaded + retireSnapshotInternal(previous) + loaded.snapshot + } else { + refreshed + } + } + if (version == upperBound.version) { + return upperBound + } + // Historical snapshots are returned to the caller but never replace the cached latest snapshot. + val historicalSnapshot = withUncachedSnapshotManager(catalogTableOpt)( + _.loadSnapshotAt(version)) + historicalSnapshot + } + + // === Uncached loading ===================================================== + + private def withUncachedSnapshotManager[T]( + catalogTableOpt: Option[CatalogTable])( + f: DeltaV2SnapshotManager => T): T = { + f(SnapshotManagerFactory.create( + tablePath.toString, + kernelContext.getDefaultEngine(), + catalogTableOpt.toJava)) + } + + private def latestSnapshotFreshnessThreshold(): Long = { + val now = System.currentTimeMillis() + val stalenessLimit = SparkSession.active.sessionState.conf + .getConf(DeltaSQLConf.DELTA_ASYNC_UPDATE_STALENESS_TIME_LIMIT) + if (stalenessLimit > 0) math.max(0, now - stalenessLimit) else now + } + + private def isFresh( + snapshot: CachedSnapshot, + requiredFreshAfter: Long, + requiredVersion: Option[Long] = None): Boolean = + snapshot != null && + snapshot.validatedAtMs >= requiredFreshAfter && + requiredVersion.forall(snapshot.snapshot.version >= _) + + private def withSnapshotLockInterruptibly[T](body: => T): T = { + snapshotLock.lockInterruptibly() + try { + body + } finally { + snapshotLock.unlock() + } + } + + private def retireSnapshotInternal(cachedSnapshot: CachedSnapshot): Unit = { + if (cachedSnapshot != null) { + cachedSnapshot.snapshot.uncache() + } + } + +} diff --git a/spark/v2/src/main/scala/io/delta/spark/internal/v2/tablemanager/DeltaV2TableManager.scala b/spark/v2/src/main/scala/io/delta/spark/internal/v2/tablemanager/DeltaV2TableManager.scala new file mode 100644 index 00000000000..3cb8f6722f3 --- /dev/null +++ b/spark/v2/src/main/scala/io/delta/spark/internal/v2/tablemanager/DeltaV2TableManager.scala @@ -0,0 +1,40 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.spark.internal.v2.tablemanager + +import org.apache.spark.sql.delta.storage.LogStore +import org.apache.spark.sql.delta.v2.interop.DeltaV2SnapshotManager +import io.delta.spark.internal.v2.kernel.KernelContext + +import org.apache.spark.sql.catalyst.catalog.CatalogTable + +/** + * Contract for a Delta table manager used by the DSv2 connector. + */ +private[v2] trait DeltaV2TableManager { + + /** Returns the table-scoped Kernel context. */ + private[v2] def kernelContext: KernelContext + + /** Returns the table-scoped log store. */ + private[v2] def logStore: LogStore + + /** Returns a snapshot manager using the caller's current catalog metadata. */ + private[v2] def snapshotManager(catalogTableOpt: Option[CatalogTable]): DeltaV2SnapshotManager + + /** Retires this manager and releases any resources it owns. */ + def retire(): Unit = {} +} diff --git a/spark/v2/src/main/scala/io/delta/spark/internal/v2/tablemanager/DeltaV2TableManagerCache.scala b/spark/v2/src/main/scala/io/delta/spark/internal/v2/tablemanager/DeltaV2TableManagerCache.scala new file mode 100644 index 00000000000..e90e1dde0bc --- /dev/null +++ b/spark/v2/src/main/scala/io/delta/spark/internal/v2/tablemanager/DeltaV2TableManagerCache.scala @@ -0,0 +1,288 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.spark.internal.v2.tablemanager + +import java.util.concurrent.{ExecutionException, TimeUnit} + +import scala.jdk.CollectionConverters._ + +import org.apache.spark.sql.delta.DeltaTableUtils +import org.apache.spark.sql.delta.sources.DeltaSQLConf +import org.apache.spark.sql.delta.util.{DeltaFileSystemOptions, PathWithFileSystem} +import io.delta.spark.internal.v2.DeltaV2Logging +import com.google.common.base.Ticker +import com.google.common.cache.{Cache, CacheBuilder, RemovalListener} +import com.google.common.util.concurrent.{ExecutionError, UncheckedExecutionException} +import org.apache.hadoop.fs.Path + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.catalog.CatalogTable +import org.apache.spark.sql.internal.SQLConf + +/** + * Per-instance [[DeltaV2TableManager]] cache configured through constructor parameters. The + * companion object owns the process-wide singleton; tests construct independent instances with + * custom settings and injected factories. + */ +private[tablemanager] class DeltaV2TableManagerCache( + maxSize: Long, + ttlMinutes: Long, + ticker: Ticker = Ticker.systemTicker(), + managerFactory: ( + DeltaV2TableManagerCache.CacheKey, + Option[CatalogTable]) => DeltaV2TableManager = + (key, catalog) => new DeltaV2TableManagerImpl( + key.path.getParent, + key.sessionInvariantFsOptions, + catalog) +) extends DeltaV2Logging { + import DeltaV2TableManagerCache.CacheKey + + private val cache: Cache[CacheKey, DeltaV2TableManager] = { + val listener: RemovalListener[CacheKey, DeltaV2TableManager] = notification => { + val manager = notification.getValue + if (manager != null) manager.retire() + } + CacheBuilder.newBuilder() + .maximumSize(maxSize) + .expireAfterAccess(ttlMinutes, TimeUnit.MINUTES) + .removalListener(listener) + .ticker(ticker) + .build[CacheKey, DeltaV2TableManager]() + } + + // Guava's Cache.get wraps loader failures: checked exceptions in ExecutionException, runtime + // exceptions in UncheckedExecutionException, and Errors in ExecutionError. Unwrapping preserves + // the original Delta error class, SQLSTATE, and cause chain rather than exposing Guava wrappers. + def getOrCreate( + key: CacheKey, + initialCatalogTableOpt: Option[CatalogTable] = None + ): DeltaV2TableManager = { + try { + cache.get(key, () => { + recordFrameProfile("tableManagerCache.createManager") { + managerFactory(key, initialCatalogTableOpt) + } + }) + } catch { + case e @ (_: UncheckedExecutionException | + _: ExecutionError | _: ExecutionException) => + val cause = Option(e.getCause).getOrElse(e) + logWarning(log"Cache loader failed; rethrowing original cause", cause) + throw cause + } + } + + def invalidate(key: CacheKey): Unit = cache.invalidate(key) + + /** + * Invalidates all entries whose log path matches the given path. Plain equality is used; no + * normalization or scheme resolution is applied. + */ + def invalidateByLogPath(logPath: Path): Unit = { + cache.asMap().keySet().removeIf(_.path == logPath) + } + + def invalidateAll(): Unit = cache.invalidateAll() + + def size(): Long = cache.size() + + def getIfPresent(key: CacheKey): Option[DeltaV2TableManager] = Option(cache.getIfPresent(key)) + + def contains(key: CacheKey): Boolean = cache.getIfPresent(key) != null + + /** Triggers pending eviction maintenance (Guava defers cleanup). */ + def cleanUp(): Unit = cache.cleanUp() +} + +/** + * Process-local cache of [[DeltaV2TableManager]] composites, mirroring DeltaLog's cache pattern. + * Enabled when `delta.log.cacheSize > 0`. Size and TTL are read from [[SQLConf]] at first access, + * reusing the V1 DeltaLog cache configs. + * + * Wider callers use the [[forTable]] facade, which constructs the internal cache key, resolves the + * session configuration, and delegates to the per-instance cache. The cache key type is not + * exposed outside the `tablemanager` package. + */ +private[v2] object DeltaV2TableManagerCache extends DeltaV2Logging { + + // === Cache key ================================================== + + /** + * Cache identity for the Delta V2 table-manager cache, aligned with the V1 DeltaLogCacheKey. + * + * @param path the Delta log directory path (`_delta_log`), derived from a fully-qualified data + * path. This is NOT the table data path -- it is `dataPath/_delta_log`. Must be absolute; + * scheme presence depends on the caller's qualification (catalog-resolved paths carry a + * scheme, local test paths may not). + * @param sessionInvariantFsOptions filesystem-prefixed options (`fs.*`, `dfs.*`) extracted from + * reader/writer options and catalog storage properties. These are the credential-bearing + * options that distinguish cache entries for the same path accessed with different + * credentials. Called "session-invariant" because they are fixed at table-resolution time and + * do not change across requests to the same cached composite. Values are redacted in + * [[toString]] to prevent credential leakage in logs. + */ + private[tablemanager] case class CacheKey( + path: Path, + sessionInvariantFsOptions: Map[String, String]) { + + override def toString: String = + s"CacheKey(path=$path,fsOptions=)" + } + + private[tablemanager] object CacheKey { + + /** + * Constructs a cache key from caller-supplied table coordinates. + * + * Cache identity must follow DeltaLog's resolved filesystem identity rather than lexical path + * spelling: two paths that resolve to the same filesystem location must share a cache entry, + * and paths that differ by authority or mount point must remain distinct. + * + * To achieve this, `newHadoopConfWithOptions` snapshots the current session's Hadoop + * configuration and overlays the derived session-invariant filesystem options. This + * Configuration is used transiently to choose the filesystem, default authority, and working + * directory, then call `FileSystem.makeQualified`. Neither the SparkSession nor the + * Configuration is retained in the cache key or manager. + * + * Unqualified paths can therefore resolve differently under different session default + * filesystems, while already-qualified absolute paths (e.g. `s3://bucket/path`) are generally + * stable. Different authorities and mount points remain distinct; the code intentionally does + * not lowercase or hand-normalize bucket or authority names. + * + * @param spark the active SparkSession, used to resolve filesystem options via + * [[DeltaFileSystemOptions.buildFsOptions]] and to qualify the cache key path through the + * filesystem. + * @param dataPath the table's data directory path as a string (e.g. + * `s3://bucket/warehouse/db/table` or `/tmp/local-table`). Must NOT include the `_delta_log` + * suffix -- this method appends it. + * @param options reader/writer options (Java map). Only `fs.*` and `dfs.*` prefixed entries are + * retained; others are filtered out. + * @param catalogTableOpt optional catalog table whose storage properties contribute + * additional filesystem options. + */ + def from( + spark: SparkSession, + dataPath: String, + options: java.util.Map[String, String], + catalogTableOpt: Option[CatalogTable] = None + ): CacheKey = { + val sessionInvariantFsOptions = DeltaFileSystemOptions.buildFsOptions( + spark, options.asScala.toMap, catalogTableOpt) + val rawLogPath = DeltaTableUtils.safeConcatPaths(new Path(dataPath), "_delta_log") + // Snapshot the session's Hadoop config with fs options overlaid. + // Used only for qualification; not retained in the key. + // scalastyle:off deltahadoopconfiguration + val hadoopConf = spark.sessionState.newHadoopConfWithOptions(sessionInvariantFsOptions) + // scalastyle:on deltahadoopconfiguration + val qualifiedLogPath = PathWithFileSystem + .withConf(rawLogPath, hadoopConf).fs.makeQualified(rawLogPath) + CacheKey(qualifiedLogPath, sessionInvariantFsOptions) + } + } + + // === Process-global singleton =================================== + + @volatile private var instance: Option[DeltaV2TableManagerCache] = None + + def isEnabled(sqlConf: SQLConf): Boolean = sqlConf.getConf(DeltaSQLConf.DELTA_LOG_CACHE_SIZE) > 0 + + /** + * Returns a cached or freshly-created [[DeltaV2TableManager]] for the given table coordinates. + * + * This is the sole public entry point for wider callers. It constructs the internal [[CacheKey]], + * derives SQLConf from the [[SparkSession]], and routes through the process-global singleton + * cache (or bypasses it when caching is disabled). + * + * @param spark the active SparkSession, used for filesystem resolution and configuration. + * @param dataPath the table's data directory path as a string. + * @param options reader/writer options (Java map). + * @param initialCatalogTableOpt optional catalog table whose storage properties contribute + * additional filesystem options. + */ + def forTable( + spark: SparkSession, + dataPath: String, + options: java.util.Map[String, String], + initialCatalogTableOpt: Option[CatalogTable] = None + ): DeltaV2TableManager = { + recordFrameProfile("tableManagerCache.forTable") { + val key = CacheKey.from(spark, dataPath, options, initialCatalogTableOpt) + val sqlConf = spark.sessionState.conf + if (!isEnabled(sqlConf)) { + return new DeltaV2TableManagerImpl( + key.path.getParent, + key.sessionInvariantFsOptions, + initialCatalogTableOpt) + } + getOrCreateInstance(sqlConf).getOrCreate(key, initialCatalogTableOpt) + } + } + + /** + * Package-private getOrCreate for per-instance tests that inject custom cache keys directly. + */ + private[tablemanager] def getOrCreate( + sqlConf: SQLConf, + key: CacheKey, + initialCatalogTableOpt: Option[CatalogTable] = None + ): DeltaV2TableManager = { + recordFrameProfile("tableManagerCache.getOrCreate") { + if (!isEnabled(sqlConf)) { + return new DeltaV2TableManagerImpl( + key.path.getParent, + key.sessionInvariantFsOptions, + initialCatalogTableOpt) + } + getOrCreateInstance(sqlConf).getOrCreate(key, initialCatalogTableOpt) + } + } + + private[tablemanager] def invalidate(key: CacheKey): Unit = instance.foreach(_.invalidate(key)) + + def clearCache(): Unit = instance.foreach(_.invalidateAll()) + + /** + * Invalidates all cache entries whose key path equals `logPath`. Matching uses exact [[Path]] + * equality; no normalization, qualification, or scheme resolution is applied by this method -- + * the caller must supply an already-qualified path consistent with the key's construction in + * [[CacheKey.from]]. + */ + def invalidateByLogPath(logPath: Path): Unit = instance.foreach(_.invalidateByLogPath(logPath)) + + /** + * Retires and clears all cached managers, then discards the singleton. Matches the + * DeltaLog.unsetCache precedent for process-global test isolation. + */ + private[tablemanager] def unsetCache(): Unit = synchronized { + instance.foreach(_.invalidateAll()) + instance = None + } + + /** + * Returns the process-global cache instance, initializing on first access. First caller's SQLConf + * determines size and TTL for all subsequent callers. + */ + private def getOrCreateInstance(sqlConf: SQLConf): DeltaV2TableManagerCache = synchronized { + instance.getOrElse { + val maxSize = sqlConf.getConf(DeltaSQLConf.DELTA_LOG_CACHE_SIZE) + val ttlMinutes = sqlConf.getConf(DeltaSQLConf.DELTA_LOG_CACHE_RETENTION_MINUTES) + val newInstance = new DeltaV2TableManagerCache(maxSize = maxSize, ttlMinutes = ttlMinutes) + instance = Some(newInstance) + newInstance + } + } +} diff --git a/spark/v2/src/main/scala/io/delta/spark/internal/v2/tablemanager/DeltaV2TableManagerImpl.scala b/spark/v2/src/main/scala/io/delta/spark/internal/v2/tablemanager/DeltaV2TableManagerImpl.scala new file mode 100644 index 00000000000..c292abd647a --- /dev/null +++ b/spark/v2/src/main/scala/io/delta/spark/internal/v2/tablemanager/DeltaV2TableManagerImpl.scala @@ -0,0 +1,69 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.spark.internal.v2.tablemanager + +import java.util.concurrent.atomic.AtomicReference + +import org.apache.spark.sql.delta.storage.LogStoreProvider +import org.apache.spark.sql.delta.v2.interop.DeltaV2SnapshotManager +import io.delta.spark.internal.v2.kernel.KernelContext +import org.apache.hadoop.fs.Path + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.catalog.CatalogTable + +/** + * Process-cached [[DeltaV2TableManager]] implementation. + * + * Reuses a table-scoped snapshot manager backed by the table's [[KernelContext]]. + * + * @param qualifiedTableDataPath the fully-qualified table data directory (parent of `_delta_log`). + * @param sessionInvariantFsOptions filesystem-prefixed credential options (`fs.*`, `dfs.*`) that + * were used to resolve the table path. Retained for downstream engine construction. + * @param initialCatalogTableOpt the catalog table supplied by the first caller that loaded this + * entry, if any. + */ +private[tablemanager] class DeltaV2TableManagerImpl( + val qualifiedTableDataPath: Path, + val sessionInvariantFsOptions: Map[String, String], + val initialCatalogTableOpt: Option[CatalogTable]) + extends DeltaV2TableManager + with LogStoreProvider +{ + + /** The table's data directory, fully qualified. */ + def tablePath: Path = qualifiedTableDataPath + + /** Used to read and write physical log files and checkpoints. */ + override private[v2] val logStore = createLogStore(SparkSession.active) + + override private[v2] val kernelContext = KernelContext(sessionInvariantFsOptions, logStore) + + private val latestCatalogTable = + new AtomicReference[CatalogTable](initialCatalogTableOpt.orNull) + private val cachedSnapshotManager = + new CachedSnapshotManager(tablePath, kernelContext, latestCatalogTable) + + override private[v2] def snapshotManager( + catalogTableOpt: Option[CatalogTable]): DeltaV2SnapshotManager = { + // Catalog metadata is latest-wins best-effort state. A refresh captures one atomic value and + // uses it consistently while selecting its path-based or catalog-managed uncached delegate. + latestCatalogTable.set(catalogTableOpt.orNull) + cachedSnapshotManager + } + + override def retire(): Unit = cachedSnapshotManager.retire() +} diff --git a/spark/v2/src/main/scala/org/apache/spark/sql/delta/v2/CDCDeletionVectorHelper.scala b/spark/v2/src/main/scala/org/apache/spark/sql/delta/v2/CDCDeletionVectorHelper.scala index 1814a59b8db..180a11fbecb 100644 --- a/spark/v2/src/main/scala/org/apache/spark/sql/delta/v2/CDCDeletionVectorHelper.scala +++ b/spark/v2/src/main/scala/org/apache/spark/sql/delta/v2/CDCDeletionVectorHelper.scala @@ -16,10 +16,9 @@ package org.apache.spark.sql.delta.v2 import java.util.{ArrayList => JArrayList, List => JList} -import javax.annotation.Nullable -import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.fs.Path +import io.delta.kernel.internal.actions.{AddFile, RemoveFile} +import io.delta.spark.internal.v2.read.CDCDataFile import org.apache.spark.sql.delta.actions.{DeletionVectorDescriptor => SparkDvDescriptor} import org.apache.spark.sql.delta.commands.DeletionVectorUtils @@ -27,8 +26,9 @@ import org.apache.spark.sql.delta.commands.cdc.CDCReader import org.apache.spark.sql.delta.deletionvectors.{RoaringBitmapArray, RoaringBitmapArrayFormat} import org.apache.spark.sql.delta.storage.dv.DeletionVectorStore -import io.delta.kernel.internal.actions.{AddFile, RemoveFile} -import io.delta.spark.internal.v2.read.CDCDataFile +import javax.annotation.Nullable +import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.fs.Path /** * Helper for DV-diff operations needed by DSv2 CDC streaming. Lives in the @@ -86,7 +86,8 @@ object CDCDeletionVectorHelper { removeDvOpt: Option[String], hadoopConf: Configuration, tablePath: String): Array[(String, String)] = { - require(addDvOpt.isDefined || removeDvOpt.isDefined, + require( + addDvOpt.isDefined || removeDvOpt.isDefined, s"Same-path Add+Remove pair must have at least one DV, tablePath: $tablePath") val addDv = addDvOpt.map(SparkDvDescriptor.deserializeFromBase64) val removeDv = removeDvOpt.map(SparkDvDescriptor.deserializeFromBase64) @@ -100,14 +101,14 @@ object CDCDeletionVectorHelper { // Remove without DV, Add with DV -> rows in addDv are deleted. // Skip disk read if the DV is already inline. val dvBase64 = if (aDv.isInline) addDvOpt.get - else readAndSerializeInline(dvStore, aDv, dataPath) + else readAndSerializeInline(dvStore, aDv, dataPath) results += ((CDCReader.CDC_TYPE_DELETE_STRING, dvBase64)) case (Some(rDv), None) => // Remove with DV, Add without DV -> rows in removeDv are re-added. // Skip disk read if the DV is already inline. val dvBase64 = if (rDv.isInline) removeDvOpt.get - else readAndSerializeInline(dvStore, rDv, dataPath) + else readAndSerializeInline(dvStore, rDv, dataPath) results += ((CDCReader.CDC_TYPE_INSERT, dvBase64)) case (Some(rDv), Some(aDv)) => @@ -145,7 +146,9 @@ object CDCDeletionVectorHelper { private def serializeInline(bitmap: RoaringBitmapArray, dataPath: Path): String = { val bytes = DeletionVectorUtils.serialize( - bitmap, RoaringBitmapArrayFormat.Portable, Some(dataPath)) + bitmap, + RoaringBitmapArrayFormat.Portable, + Some(dataPath)) SparkDvDescriptor.inlineInLog(bytes, bitmap.cardinality).serializeToBase64 } } diff --git a/spark/v2/src/main/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2QueryContext.scala b/spark/v2/src/main/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2QueryContext.scala new file mode 100644 index 00000000000..74b4a8f1c65 --- /dev/null +++ b/spark/v2/src/main/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2QueryContext.scala @@ -0,0 +1,40 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.v2.interop + +import java.util.{Objects, Optional} + +import scala.jdk.OptionConverters._ + +import org.apache.spark.annotation.Experimental +import org.apache.spark.sql.catalyst.catalog.CatalogTable + +/** Request-scoped inputs for Delta DSv2 snapshot operations. */ +@Experimental +final case class DeltaV2QueryContext(catalogTableOpt: Option[CatalogTable]) { + Objects.requireNonNull(catalogTableOpt, "catalogTableOpt is null") +} + +object DeltaV2QueryContext { + /** Creates a query context without catalog metadata. */ + def empty: DeltaV2QueryContext = DeltaV2QueryContext(None) + + /** Creates a query context from optional catalog metadata. */ + def apply(catalogTableOpt: Optional[CatalogTable]): DeltaV2QueryContext = { + DeltaV2QueryContext(Objects.requireNonNull(catalogTableOpt, "catalogTableOpt is null").toScala) + } +} diff --git a/spark-unified/src/main/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2Snapshot.scala b/spark/v2/src/main/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2Snapshot.scala similarity index 56% rename from spark-unified/src/main/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2Snapshot.scala rename to spark/v2/src/main/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2Snapshot.scala index 33e2c75280b..054a0c8005b 100644 --- a/spark-unified/src/main/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2Snapshot.scala +++ b/spark/v2/src/main/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2Snapshot.scala @@ -16,28 +16,25 @@ package org.apache.spark.sql.delta.v2.interop -import org.apache.spark.sql.delta.{ - CheckpointProvider, - DeltaColumnMappingMode, - DeltaLogFileIndex, - Snapshot, - VersionChecksum -} -import org.apache.spark.sql.delta.actions.{ - AddFile, - Metadata, - Protocol, - RemoveFile, - SingleAction -} -import org.apache.spark.sql.delta.coordinatedcommits.TableCommitCoordinatorClient -import org.apache.spark.sql.delta.stats.{DeltaStatsColumnSpec, StatisticsCollection} -import org.apache.hadoop.fs.Path -import io.delta.kernel.defaults.engine.DefaultEngine +import scala.collection.JavaConverters._ +import scala.jdk.OptionConverters._ + import io.delta.kernel.engine.Engine import io.delta.kernel.internal.{SnapshotImpl => KernelSnapshot} +import io.delta.spark.internal.v2.kernel.KernelEngineFactory -import org.apache.spark.sql.{DataFrame, Dataset, SparkSession} +import org.apache.spark.sql.delta.{CheckpointProvider, DeltaColumnMappingMode, DeltaLogFileIndex, Snapshot, VersionChecksum} +import org.apache.spark.sql.delta.actions.{AddFile, DomainMetadata, Metadata, Protocol, RemoveFile, SingleAction} +import org.apache.spark.sql.delta.coordinatedcommits.TableCommitCoordinatorClient +import org.apache.spark.sql.delta.stats.{DeltaStatsColumnSpec, FileSizeHistogram, StatisticsCollection} +import org.apache.spark.sql.delta.v2.kernel.KernelActionUtils +import org.apache.spark.sql.delta.RowId +import org.apache.spark.sql.delta.util.JsonUtils + +import com.databricks.spark.util.TagDefinition +import org.apache.hadoop.fs.Path +import org.apache.spark.sql.{DataFrame, Dataset} +import org.apache.spark.sql.catalyst.catalog.CatalogTable /** * A [[Snapshot]] backed by a Delta Kernel snapshot, for the v2 Connector. It extends the V1 @@ -48,41 +45,36 @@ import org.apache.spark.sql.{DataFrame, Dataset, SparkSession} * member the construction or scan path would dereference them through is overridden below. V1 state * reconstruction (`stateDF` etc.) is not wired to Kernel yet and throws. */ -private[v2] class DeltaV2Snapshot( +class DeltaV2Snapshot( // Typed as the internal SnapshotImpl (aliased KernelSnapshot), not the public Kernel // `Snapshot` interface: the decoded data path is only exposed on SnapshotImpl in the // OSS-published Kernel. The public io.delta.kernel.Snapshot has no `getDataPath`, and its // `getPath` is URL-encoded, which breaks Hadoop `Path` for table roots containing spaces. - kernelSnapshot: KernelSnapshot, - sparkSession: SparkSession, - // Kernel engine for reading scan files and the commit timestamp -- owned and passed by the - // connector. The secondary constructor below builds a default one for callers without a Kernel - // engine of their own. - // The separating comma lives inside the edge block: it is only needed when the edge-only - // catalogTableOpt parameter follows. Without it, the OSS export would strip catalogTableOpt and - // leave a trailing comma that scalac accepts but scalastyle's parser rejects. - kernelEngine: Engine - ) - extends Snapshot( - // toString keeps this compiling across Kernel versions: getDataPath returns String in the - // currently pinned Kernel and io.delta.kernel.internal.fs.Path in newer Kernels. - path = new Path(kernelSnapshot.getDataPath.toString), - version = kernelSnapshot.getVersion, - // A DeltaV2Snapshot must not depend on the V1 LogSegment or DeltaLog. Both are null; the - // construction-path members that would otherwise dereference them are overridden below, so - // the nulls are never dereferenced on the scan path. - logSegment = null, - deltaLog = null, - checksumOpt = None) { - - override protected def spark: SparkSession = sparkSession - - /** Convenience for callers without their own Kernel engine: builds a fresh DefaultEngine. */ + private val kernelSnapshot: KernelSnapshot) + extends Snapshot( + // toString keeps this compiling across Kernel versions: getDataPath returns String in the + // currently pinned Kernel and io.delta.kernel.internal.fs.Path in newer Kernels. + path = new Path(kernelSnapshot.getDataPath.toString), + version = kernelSnapshot.getVersion, + // A DeltaV2Snapshot must not depend on the V1 LogSegment or DeltaLog. Both are null; the + // construction-path members that would otherwise dereference them are overridden below, so + // the nulls are never dereferenced on the scan path. + logSegment = null, + deltaLog = null, + checksumOpt = None) { + + private[this] var resolvedCatalogTableOpt: Option[CatalogTable] = None + + def this( + kernelSnapshot: KernelSnapshot, + catalogTableOpt: Option[CatalogTable]) = { + this(kernelSnapshot) + resolvedCatalogTableOpt = catalogTableOpt + } + // scalastyle:off deltahadoopconfiguration - // No DeltaLog to source a Hadoop conf from, so use the session Hadoop conf for the engine. - def this(kernelSnapshot: KernelSnapshot, sparkSession: SparkSession) = - this(kernelSnapshot, sparkSession, - DefaultEngine.create(sparkSession.sessionState.newHadoopConf())) + private def kernelEngine: Engine = + KernelEngineFactory.createDefaultEngine(spark.sessionState.newHadoopConf()) // scalastyle:on deltahadoopconfiguration // --- logSegment/deltaLog = null guardrail: construction-path overrides ---------------------- @@ -129,14 +121,24 @@ private[v2] class DeltaV2Snapshot( // SnapshotDescriptor.logPath override def logPath: Path = new Path(kernelSnapshot.getLogPath.toString) + // This snapshot has no DeltaLog from which to derive usage-log tags, so emit none. + override def getCommonTags: Map[TagDefinition, String] = Map.empty + // The catalog table this snapshot was resolved from, if the connector supplied one. + def catalogTable: Option[CatalogTable] = Option(resolvedCatalogTableOpt).flatten // --- SnapshotDescriptor / state surface ---------------------------------------------------- override lazy val metadata: Metadata = - DeltaV2SnapshotConversionsUtils.metadataFromKernel(kernelSnapshot.getMetadata) + KernelActionUtils.metadataFromKernel(kernelSnapshot.getMetadata) override lazy val protocol: Protocol = - DeltaV2SnapshotConversionsUtils.protocolFromKernel(kernelSnapshot.getProtocol) + KernelActionUtils.protocolFromKernel(kernelSnapshot.getProtocol) + + override def getRowTrackingHighWaterMark(): Option[Long] = { + kernelSnapshot.getDomainMetadata("delta.rowTracking").toScala.map { configuration => + JsonUtils.fromJson[RowId.RowTrackingMetadataDomain](configuration).rowIdHighWaterMark + } + } override def columnMappingMode: DeltaColumnMappingMode = metadata.columnMappingMode @@ -146,9 +148,12 @@ private[v2] class DeltaV2Snapshot( None } - // Kernel does not surface a cheap total table size, so report None; the data-skipping path - // derives the scanned size from per-file sizes instead. - override protected[delta] def sizeInBytesIfKnown: Option[Long] = None + /** + * Table size in bytes from CRC when present; None otherwise. + */ + override protected[delta] def sizeInBytesIfKnown: Option[Long] = { + kernelSnapshot.getCurrentCrcInfo.toScala.map(_.getTableSizeBytes) + } // No V1 commit-file index; not used by the Kernel scan path. override protected[delta] lazy val deltaFileIndexOpt: Option[DeltaLogFileIndex] = unimplemented @@ -156,6 +161,13 @@ private[v2] class DeltaV2Snapshot( // No V1 checkpoint provider; not used by the Kernel scan path. override lazy val checkpointProvider: CheckpointProvider = unimplemented + /** + * No V1 commit-file index or checkpoint provider, so these V1 commit-file/checkpoint sizes are + * not applicable. -1 is the "did not collect this stat" sentinel. + */ + override def deltaFileSizeInBytes(): Long = -1L + override def checkpointSizeInBytes(): Long = -1L + // V1 reads these from logSegment; report zero. override def numDeltaFiles: Long = 0L override def totalDeltaFilesByteSize: Long = 0L @@ -166,16 +178,61 @@ private[v2] class DeltaV2Snapshot( // which a DeltaV2Snapshot cannot use directly because it has no V1 LogSegment. override def timestamp: Long = kernelSnapshot.getTimestamp(kernelEngine) + def getLatestTransactionVersion( + appId: String): java.util.OptionalLong = { + val result = kernelSnapshot + .getLatestTransactionVersion(kernelEngine, appId) + if (result.isPresent) { + java.util.OptionalLong.of(result.get) + } else { + java.util.OptionalLong.empty() + } + } + // --- Files surfaced via Kernel scan --------------------------------------------------------- override lazy val allFiles: Dataset[AddFile] = - KernelSnapshotUtils.buildAllFiles(kernelSnapshot, sparkSession, kernelEngine) + KernelSnapshotUtils.buildAllFiles(kernelSnapshot, spark, kernelEngine) // --- StatisticsCollection ------------------------------------------------------------------ override lazy val statsColumnSpec: DeltaStatsColumnSpec = StatisticsCollection.configuredDeltaStatsColumnSpec(metadata) + /** + * Table size in bytes, served from the Kernel CRC when known. Returns the -1 "did not collect + * this stat" sentinel. + */ + override lazy val sizeInBytes: Long = sizeInBytesIfKnown.getOrElse(-1L) + + /** + * File-size histogram, served from the Kernel CRC when known and gated by + * [[fileSizeHistogramEnabled]]. Returns None when the config is disabled or the CRC has no + * histogram. + */ + override lazy val fileSizeHistogram: Option[FileSizeHistogram] = + if (fileSizeHistogramEnabled) { + kernelSnapshot.getCurrentCrcInfo.toScala + .flatMap(_.getFileSizeHistogram.toScala) + .map { bins => + val result = bins.captureFileSizeHistogramResult() + FileSizeHistogram( + sortedBinBoundaries = result.getSortedBinBoundaries.toIndexedSeq, + fileCounts = result.getFileCounts, + totalBytes = result.getTotalBytes) + } + } else { + None + } + + override lazy val domainMetadata: Seq[DomainMetadata] = + kernelSnapshot.getActiveDomainMetadataMap.asScala.map { case (domain, kernelDomainMetadata) => + DomainMetadata(domain, kernelDomainMetadata.getConfiguration, kernelDomainMetadata.isRemoved) + }.toSeq + + // Kernel serves domain metadata without V1 state reconstruction, so it is always known. + override protected[delta] def domainMetadatasIfKnown: Option[Seq[DomainMetadata]] = + Some(domainMetadata) // --- ValidateChecksum: Kernel has no V1 checksum to validate; no-op ------------------------ override def validateChecksum(contextInfo: Map[String, String]): Boolean = true @@ -192,5 +249,28 @@ private[v2] class DeltaV2Snapshot( private def unimplemented: Nothing = throw new UnsupportedOperationException( - "This member is not yet supported on a Kernel-backed DeltaV2Snapshot") + "DeltaV2 unsupported operation: this member is not yet supported on a Kernel-backed " + + "DeltaV2Snapshot") +} + +object DeltaV2Snapshot { + + /** + * Returns the Kernel snapshot wrapped by a Kernel-backed V1 snapshot facade. + * + * Temporary: this accessor exists for code paths that still consume Kernel's SnapshotImpl + * directly and will be removed once those consumers are migrated to DSv1-level APIs. + */ + def getKernelSnapshot(snapshot: Snapshot): KernelSnapshot = { + if (snapshot == null) { + throw new NullPointerException("snapshot is null") + } + snapshot match { + case deltaV2Snapshot: DeltaV2Snapshot => deltaV2Snapshot.kernelSnapshot + case other => + throw new IllegalStateException( + s"Expected a DeltaV2Snapshot, but found ${other.getClass.getName}") + } + } + } diff --git a/spark/v2/src/main/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2SnapshotManager.scala b/spark/v2/src/main/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2SnapshotManager.scala new file mode 100644 index 00000000000..6f277205716 --- /dev/null +++ b/spark/v2/src/main/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2SnapshotManager.scala @@ -0,0 +1,166 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.v2.interop + +import java.util.{Objects, Optional} + +import io.delta.kernel.CommitRange +import io.delta.kernel.engine.Engine +import io.delta.kernel.internal.DeltaHistoryManager +import io.delta.kernel.internal.SnapshotImpl +import io.delta.spark.internal.v2.exception.VersionNotFoundException + +import org.apache.spark.sql.delta.Snapshot + +import org.apache.spark.annotation.Experimental + +/** + * Contract for managing Delta table snapshots in the DSv2 connector. + * + * This connector exposes loaded state through the V1 snapshot facade so callers use the same + * metadata, protocol, schema, timestamp, column-mapping, and file-access surface as V1. Kernel + * execution details remain confined to the connector's execution seams. + */ +@Experimental +trait DeltaV2SnapshotManager { + + /** Loads and returns the latest snapshot of the Delta table. */ + @deprecated("Use loadLatestSnapshot with a query context instead") + def loadLatestSnapshot(): Snapshot + + /** Loads the latest snapshot using inputs scoped to the current query. */ + def loadLatestSnapshot(queryContext: DeltaV2QueryContext): Snapshot = { + Objects.requireNonNull(queryContext, "queryContext is null") + loadLatestSnapshot() + } + + /** + * Loads and returns a snapshot at a specific version. + * + * @param version the version number to load (must be >= 0) + */ + @deprecated("Use loadSnapshotAt with a query context instead") + def loadSnapshotAt(version: Long): Snapshot + + /** Loads a versioned snapshot using inputs scoped to the current query. */ + def loadSnapshotAt( + version: Long, + queryContext: DeltaV2QueryContext): Snapshot = { + Objects.requireNonNull(queryContext, "queryContext is null") + loadSnapshotAt(version) + } + + /** + * Finds the commit that was active at a specific timestamp. + * + * @param timestampMillis timestamp in milliseconds since epoch (UTC) + * @param canReturnLastCommit if true, returns the last commit when + * the timestamp is after all commits + * @param mustBeRecreatable if true, only considers commits that can + * be fully recreated from available log files + * @param canReturnEarliestCommit if true, returns the earliest commit + * when the timestamp is before all commits + */ + @deprecated("Use getActiveCommitAtTime with a query context instead") + def getActiveCommitAtTime( + timestampMillis: Long, + canReturnLastCommit: Boolean, + mustBeRecreatable: Boolean, + canReturnEarliestCommit: Boolean): DeltaHistoryManager.Commit + + /** Finds the active commit using inputs scoped to the current query. */ + def getActiveCommitAtTime( + timestampMillis: Long, + canReturnLastCommit: Boolean, + mustBeRecreatable: Boolean, + canReturnEarliestCommit: Boolean, + queryContext: DeltaV2QueryContext): DeltaHistoryManager.Commit = { + Objects.requireNonNull(queryContext, "queryContext is null") + getActiveCommitAtTime( + timestampMillis, + canReturnLastCommit, + mustBeRecreatable, + canReturnEarliestCommit) + } + + /** + * Checks if a specific version exists and is accessible. + * + * @param version the version to check + * @param mustBeRecreatable if true, requires that the version can be + * fully recreated from available log files + * @param allowOutOfRange if true, allows versions greater than the + * latest version without throwing + * @throws VersionNotFoundException if the version is not available + */ + @throws[VersionNotFoundException] + @deprecated("Use checkVersionExists with a query context instead") + def checkVersionExists( + version: Long, + mustBeRecreatable: Boolean, + allowOutOfRange: Boolean): Unit + + /** Checks version availability using inputs scoped to the current query. */ + @throws[VersionNotFoundException] + def checkVersionExists( + version: Long, + mustBeRecreatable: Boolean, + allowOutOfRange: Boolean, + queryContext: DeltaV2QueryContext): Unit = { + Objects.requireNonNull(queryContext, "queryContext is null") + checkVersionExists(version, mustBeRecreatable, allowOutOfRange) + } + + /** + * Gets a range of table changes between start and end versions. + * + * @param engine the engine for executing operations + * @param startVersion starting version (inclusive) + * @param endVersion optional ending version (inclusive) + */ + @deprecated("Use getTableChanges with a query context instead") + def getTableChanges( + engine: Engine, + startVersion: Long, + endVersion: Optional[java.lang.Long]): CommitRange + + /** Gets table changes using inputs scoped to the current query. */ + def getTableChanges( + kernelEngine: Engine, + startVersion: Long, + endVersion: Optional[java.lang.Long], + queryContext: DeltaV2QueryContext): CommitRange = { + Objects.requireNonNull(queryContext, "queryContext is null") + getTableChanges(kernelEngine, startVersion, endVersion) + } +} + +object DeltaV2SnapshotManager { + + /** + * Wraps a Kernel snapshot in the V1 [[Snapshot]] facade returned by manager load APIs. + * + * @param kernelSnapshot the Kernel snapshot to wrap + * @param tablePath table path used in construction-error messages + * @return the V1 snapshot facade + */ + def wrapKernelSnapshot(kernelSnapshot: SnapshotImpl, tablePath: String): Snapshot = { + Objects.requireNonNull(kernelSnapshot, "kernelSnapshot is null") + Objects.requireNonNull(tablePath, "tablePath is null") + new DeltaV2Snapshot(kernelSnapshot) + } +} diff --git a/spark-unified/src/main/scala/org/apache/spark/sql/delta/v2/interop/KernelSnapshotUtils.scala b/spark/v2/src/main/scala/org/apache/spark/sql/delta/v2/interop/KernelSnapshotUtils.scala similarity index 75% rename from spark-unified/src/main/scala/org/apache/spark/sql/delta/v2/interop/KernelSnapshotUtils.scala rename to spark/v2/src/main/scala/org/apache/spark/sql/delta/v2/interop/KernelSnapshotUtils.scala index 1db14041c34..15dca61f711 100644 --- a/spark-unified/src/main/scala/org/apache/spark/sql/delta/v2/interop/KernelSnapshotUtils.scala +++ b/spark/v2/src/main/scala/org/apache/spark/sql/delta/v2/interop/KernelSnapshotUtils.scala @@ -19,15 +19,13 @@ package org.apache.spark.sql.delta.v2.interop import scala.collection.mutable.ArrayBuffer import scala.jdk.OptionConverters._ -import org.apache.spark.sql.delta.actions.{ - AddFile, - DeletionVectorDescriptor => V1DeletionVectorDescriptor -} -import org.apache.spark.sql.delta.implicits._ import io.delta.kernel.data.MapValue import io.delta.kernel.engine.Engine import io.delta.kernel.internal.{ScanImpl, SnapshotImpl} -import io.delta.kernel.internal.actions.{AddFile => KernelAddFile} +import io.delta.kernel.internal.actions.{AddFile => KernelAddFile, DeletionVectorDescriptor => KernelDeletionVectorDescriptor} + +import org.apache.spark.sql.delta.actions.{AddFile, DeletionVectorDescriptor => V1DeletionVectorDescriptor} +import org.apache.spark.sql.delta.implicits._ import org.apache.spark.sql.{Dataset, SparkSession} @@ -64,7 +62,7 @@ private[delta] object KernelSnapshotUtils { kernelSnapshot: SnapshotImpl, engine: Engine): Seq[AddFile] = { val scan = kernelSnapshot.getScanBuilder.build().asInstanceOf[ScanImpl] - val scanFileBatches = scan.getScanFiles(engine, true /* includeStats */) + val scanFileBatches = scan.getScanFiles(engine, true /* includeStats */ ) try { val files = ArrayBuffer.empty[AddFile] while (scanFileBatches.hasNext) { @@ -72,7 +70,7 @@ private[delta] object KernelSnapshotUtils { try { while (rows.hasNext) { val kernelAddFile = new KernelAddFile( - rows.next().getStruct(0 /* addFileColumnOrdinal */)) + rows.next().getStruct(0 /* addFileColumnOrdinal */ )) files += toV1AddFile(kernelAddFile) } } finally { @@ -89,8 +87,8 @@ private[delta] object KernelSnapshotUtils { * Builds one V1 [[AddFile]] from one Kernel AddFile. * * The returned AddFile keeps the path, partition values, size, modification time, stats, - * deletion vector, and row tracking fields from Kernel. The row tracking fields are baseRowId - * and defaultRowCommitVersion. + * tags, deletion vector, and row tracking fields from Kernel. The row tracking fields are + * baseRowId and defaultRowCommitVersion. */ private def toV1AddFile(kernelAddFile: KernelAddFile): AddFile = { AddFile( @@ -101,6 +99,7 @@ private[delta] object KernelSnapshotUtils { // V1 Snapshot.allFiles normalizes active files to dataChange = false. dataChange = false, stats = kernelAddFile.getStatsJson.toScala.orNull, + tags = toV1Tags(kernelAddFile), deletionVector = toV1DeletionVectorDescriptor(kernelAddFile), baseRowId = kernelAddFile.getBaseRowId.toScala.map(_.longValue()), defaultRowCommitVersion = @@ -115,19 +114,20 @@ private[delta] object KernelSnapshotUtils { * V1 AddFile uses null when a file has no deletion vector, so this method does the same. */ private def toV1DeletionVectorDescriptor( - kernelAddFile: KernelAddFile): V1DeletionVectorDescriptor = { - val deletionVector = kernelAddFile.getDeletionVector - if (deletionVector.isPresent) { - val kernelDeletionVector = deletionVector.get - V1DeletionVectorDescriptor( - storageType = kernelDeletionVector.getStorageType, - pathOrInlineDv = kernelDeletionVector.getPathOrInlineDv, - offset = kernelDeletionVector.getOffset.toScala.map(_.intValue()), - sizeInBytes = kernelDeletionVector.getSizeInBytes, - cardinality = kernelDeletionVector.getCardinality) - } else { - null - } + kernelAddFile: KernelAddFile): V1DeletionVectorDescriptor = + kernelAddFile.getDeletionVector.toScala + .map(toV1DeletionVectorDescriptor) + .orNull + + /** Converts one Kernel deletion-vector descriptor to Delta's V1 value type. */ + private[v2] def toV1DeletionVectorDescriptor( + kernelDeletionVector: KernelDeletionVectorDescriptor): V1DeletionVectorDescriptor = { + V1DeletionVectorDescriptor( + storageType = kernelDeletionVector.getStorageType, + pathOrInlineDv = kernelDeletionVector.getPathOrInlineDv, + offset = kernelDeletionVector.getOffset.toScala.map(_.intValue()), + sizeInBytes = kernelDeletionVector.getSizeInBytes, + cardinality = kernelDeletionVector.getCardinality) } /** @@ -149,4 +149,23 @@ private[delta] object KernelSnapshotUtils { Map.empty } } + + /** + * Returns the tags for one AddFile as a Scala map. + * + * Kernel stores tags as an optional string-to-string map, the same shape as partition values. + * A missing Kernel tags map stays null, matching V1 AddFile.tags. + */ + private def toV1Tags(kernelAddFile: KernelAddFile): Map[String, String] = { + kernelAddFile.getTags.toScala + .map { tags => + val keys = tags.getKeys + val values = tags.getValues + (0 until tags.getSize).map { index => + val value = if (values.isNullAt(index)) null else values.getString(index) + keys.getString(index) -> value + }.toMap + } + .orNull + } } diff --git a/spark/v2/src/main/scala/org/apache/spark/sql/delta/v2/kernel/KernelActionUtils.scala b/spark/v2/src/main/scala/org/apache/spark/sql/delta/v2/kernel/KernelActionUtils.scala new file mode 100644 index 00000000000..f945f757839 --- /dev/null +++ b/spark/v2/src/main/scala/org/apache/spark/sql/delta/v2/kernel/KernelActionUtils.scala @@ -0,0 +1,304 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.v2.kernel + +import java.sql.Timestamp + +import scala.collection.JavaConverters._ +import scala.jdk.OptionConverters._ + +import org.apache.spark.sql.delta.actions.{ + Action, + AddFile, + CommitInfo, + DeletionVectorDescriptor, + DomainMetadata, + Format, + Metadata, + Protocol, + RemoveFile, + SetTransaction +} +import io.delta.kernel.{CommitActions => KernelCommitActions} +import io.delta.kernel.data.{ColumnarBatch => KernelColumnarBatch} +import io.delta.kernel.data.{ColumnVector => KernelColumnVector} +import io.delta.kernel.data.{MapValue => KernelMapValue} +import io.delta.kernel.internal.DeltaLogActionUtils.{DeltaAction => KernelDeltaAction} +import io.delta.kernel.internal.actions.{AddFile => KernelAddFile} +import io.delta.kernel.internal.actions.{CommitInfo => KernelCommitInfo} +import io.delta.kernel.internal.actions.{ + DeletionVectorDescriptor => KernelDeletionVectorDescriptor +} +import io.delta.kernel.internal.actions.{DomainMetadata => KernelDomainMetadata} +import io.delta.kernel.internal.actions.{Metadata => KernelMetadata} +import io.delta.kernel.internal.actions.{Protocol => KernelProtocol} +import io.delta.kernel.internal.actions.{RemoveFile => KernelRemoveFile} +import io.delta.kernel.internal.actions.{SetTransaction => KernelSetTransaction} +import io.delta.kernel.internal.data.{StructRow => KernelStructRow} +import io.delta.kernel.internal.util.{VectorUtils => KernelVectorUtils} + +/** + * Bridges Kernel and V1 Delta actions. + */ +private[v2] object KernelActionUtils { + + /** + * Reads a Kernel commit actions into V1 [[Action]]s. + */ + private[v2] def readActions(commitActions: KernelCommitActions): Seq[Action] = { + val kernelActionsBatchIter = commitActions.getActions() + try { + val actions = Seq.newBuilder[Action] + while (kernelActionsBatchIter.hasNext) { + actions ++= readActionBatch(kernelActionsBatchIter.next()) + } + actions.result() + } finally { + kernelActionsBatchIter.close() + } + } + + private def readActionBatch( + kernelActionsBatch: KernelColumnarBatch + ): Seq[Action] = { + val schema = kernelActionsBatch.getSchema + // (action type -> its column vector), for the action columns actually present in this batch. + val actionToColumnVector = KernelDeltaAction.values().flatMap { action => + val ordinal = schema.indexOf(action.colName) + if (ordinal >= 0) Some(action -> kernelActionsBatch.getColumnVector(ordinal)) else None + } + // A Delta log row carries exactly one action, so at most one action column is non-null per row; + // `collectFirst` decodes that single column into an Action. + (0 until kernelActionsBatch.getSize).flatMap { rowId => + actionToColumnVector.collectFirst { + case (action, columnVector) if !columnVector.isNullAt(rowId) => + actionFromKernel(action, columnVector, rowId) + } + } + } + + private def actionFromKernel( + action: KernelDeltaAction, + columnVector: KernelColumnVector, + rowId: Int + ): Action = action match { + case KernelDeltaAction.ADD => + addFileFromKernel( + new KernelAddFile(KernelStructRow.fromStructVector(columnVector, rowId)) + ) + case KernelDeltaAction.REMOVE => + // scalastyle:off removeFile // KernelRemoveFile is the Kernel wrapper, not a V1 RemoveFile + removeFileFromKernel( + new KernelRemoveFile(KernelStructRow.fromStructVector(columnVector, rowId))) + // scalastyle:on removeFile + case KernelDeltaAction.METADATA => + metadataFromKernel( + KernelMetadata.fromColumnVector(columnVector, rowId)) + case KernelDeltaAction.PROTOCOL => + protocolFromKernel( + KernelProtocol.fromColumnVector(columnVector, rowId)) + case KernelDeltaAction.COMMITINFO => + commitInfoFromKernel( + KernelCommitInfo.fromColumnVector(columnVector, rowId)) + case KernelDeltaAction.TXN => + setTransactionFromKernel( + KernelSetTransaction.fromColumnVector(columnVector, rowId)) + case KernelDeltaAction.DOMAINMETADATA => + domainMetadataFromKernel( + KernelDomainMetadata.fromColumnVector(columnVector, rowId)) + case other => + throw new UnsupportedOperationException( + s"No V1 action from Kernel decoder for a '${other.colName}' action yet") + } + + /** + * Converts a Kernel [[KernelAddFile]] into a V1 [[AddFile]]. + */ + def addFileFromKernel(addFile: KernelAddFile): AddFile = { + AddFile( + path = addFile.getPath, + partitionValues = partitionValuesFromKernel(addFile.getPartitionValues), + size = addFile.getSize, + modificationTime = addFile.getModificationTime, + dataChange = addFile.getDataChange, + stats = addFile.getStatsJson.toScala.orNull, + tags = tagsFromKernel(addFile.getTags), + deletionVector = deletionVectorFromKernel(addFile.getDeletionVector), + baseRowId = addFile.getBaseRowId.toScala.map(_.longValue()), + defaultRowCommitVersion = addFile.getDefaultRowCommitVersion.toScala.map(_.longValue())) + } + + /** + * Converts the fields represented by Kernel's scan AddFile schema from a V1 [[AddFile]]. + * Fields absent from Kernel's schema are intentionally dropped. + */ + def toKernelScanAddFile(addFile: AddFile): KernelAddFile = { + val deletionVector = Option(addFile.deletionVector).map { dv => + new KernelDeletionVectorDescriptor( + dv.storageType, + dv.pathOrInlineDv, + dv.offset.map(Int.box).toJava, + dv.sizeInBytes, + dv.cardinality) + } + + val row = KernelAddFile.createAddFileRowWithStatsJson( + addFile.path, + KernelVectorUtils.stringStringMapValue(addFile.partitionValues.asJava), + addFile.size, + addFile.modificationTime, + addFile.dataChange, + deletionVector.toJava, + Option(addFile.tags) + .map(tags => KernelVectorUtils.stringStringMapValue(tags.asJava)) + .toJava, + addFile.baseRowId.map(Long.box).toJava, + addFile.defaultRowCommitVersion.map(Long.box).toJava, + Option(addFile.stats).toJava) + + new KernelAddFile(row) + } + + /** + * Converts a Kernel [[KernelRemoveFile]] into a V1 [[RemoveFile]]. + */ + def removeFileFromKernel(removeFile: KernelRemoveFile): RemoveFile = { + // scalastyle:off removeFile + RemoveFile( + path = removeFile.getPath, + deletionTimestamp = removeFile.getDeletionTimestamp.toScala.map(_.longValue()), + dataChange = removeFile.getDataChange, + extendedFileMetadata = removeFile.getExtendedFileMetadata.toScala.map(_.booleanValue()), + partitionValues = removeFile.getPartitionValues.toScala.map(partitionValuesFromKernel).orNull, + size = removeFile.getSize.toScala.map(_.longValue()), + tags = tagsFromKernel(removeFile.getTags), + deletionVector = deletionVectorFromKernel(removeFile.getDeletionVector), + baseRowId = removeFile.getBaseRowId.toScala.map(_.longValue()), + defaultRowCommitVersion = removeFile.getDefaultRowCommitVersion.toScala.map(_.longValue()), + stats = removeFile.getStatsJson.toScala.orNull) + // scalastyle:on removeFile + } + + /** + * Converts a Kernel [[KernelMetadata]] into a V1 [[Metadata]]. + */ + def metadataFromKernel(metadata: KernelMetadata): Metadata = { + Metadata( + id = metadata.getId, + name = metadata.getName.orElse(null), + description = metadata.getDescription.orElse(null), + format = Format( + provider = metadata.getFormat.getProvider, + options = metadata.getFormat.getOptions.asScala.toMap), + schemaString = metadata.getSchemaString, + // getPartitionColumns preserves the declared order and original case of partition + // columns; getPartitionColNames would return an unordered set of lowercased names. + partitionColumns = + KernelVectorUtils.toJavaList[String](metadata.getPartitionColumns).asScala.toSeq, + configuration = metadata.getConfiguration.asScala.toMap, + createdTime = + if (metadata.getCreatedTime.isPresent) Some(metadata.getCreatedTime.get.longValue()) + else None) + } + + /** + * Converts a Kernel [[KernelProtocol]] into a V1 [[Protocol]]. + */ + def protocolFromKernel(protocol: KernelProtocol): Protocol = { + val readerFeatures = + Option(protocol.getReaderFeatures).map(_.asScala.toSet).getOrElse(Set.empty) + val writerFeatures = + Option(protocol.getWriterFeatures).map(_.asScala.toSet).getOrElse(Set.empty) + + Protocol(protocol.getMinReaderVersion, protocol.getMinWriterVersion) + .withWriterFeatures(writerFeatures) + .withReaderFeatures(readerFeatures) + } + + /** + * Converts a Kernel [[KernelCommitInfo]] into a V1 [[CommitInfo]]. + */ + def commitInfoFromKernel(commitInfo: KernelCommitInfo): CommitInfo = { + CommitInfo( + version = None, + inCommitTimestamp = commitInfo.getInCommitTimestamp.toScala.map(_.longValue()), + timestamp = new Timestamp(commitInfo.getTimestamp), + userId = None, + userName = None, + operation = commitInfo.getOperation.orElse(null), + operationParameters = + Option(commitInfo.getOperationParameters).map(_.asScala.toMap).getOrElse(Map.empty), + job = None, + notebook = None, + clusterId = None, + readVersion = None, + isolationLevel = None, + isBlindAppend = commitInfo.getIsBlindAppend.toScala.map(_.booleanValue()), + dataChange = None, + operationMetrics = Option(commitInfo.getOperationMetrics).map(_.asScala.toMap), + userMetadata = None, + tags = None, + engineInfo = commitInfo.getEngineInfo.toScala, + txnId = commitInfo.getTxnId.toScala, + lastManifestCommit = None) + } + + /** + * Converts a Kernel [[KernelSetTransaction]] into a V1 [[SetTransaction]]. + */ + def setTransactionFromKernel(txn: KernelSetTransaction): SetTransaction = { + SetTransaction( + appId = txn.getAppId, + version = txn.getVersion, + lastUpdated = txn.getLastUpdated.toScala.map(_.longValue())) + } + + /** + * Converts a Kernel [[KernelDomainMetadata]] into a V1 [[DomainMetadata]]. + */ + def domainMetadataFromKernel(domainMetadata: KernelDomainMetadata): DomainMetadata = { + DomainMetadata( + domainMetadata.getDomain, + domainMetadata.getConfiguration, + domainMetadata.isRemoved) + } + + private def deletionVectorFromKernel( + deletionVector: java.util.Optional[KernelDeletionVectorDescriptor]) + : DeletionVectorDescriptor = + deletionVector.toScala.map { dv => + DeletionVectorDescriptor( + storageType = dv.getStorageType, + pathOrInlineDv = dv.getPathOrInlineDv, + offset = dv.getOffset.toScala.map(_.intValue()), + sizeInBytes = dv.getSizeInBytes, + cardinality = dv.getCardinality) + }.orNull + + private def partitionValuesFromKernel(partitionValues: KernelMapValue): Map[String, String] = { + if (partitionValues == null) return Map.empty + val keys = partitionValues.getKeys + val values = partitionValues.getValues + (0 until partitionValues.getSize).map { index => + val value = if (values.isNullAt(index)) null else values.getString(index) + keys.getString(index) -> value + }.toMap + } + + private def tagsFromKernel(tags: java.util.Optional[KernelMapValue]): Map[String, String] = + tags.toScala.map(partitionValuesFromKernel).orNull +} diff --git a/spark/v2/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DeltaV2FilterTranslator.scala b/spark/v2/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DeltaV2FilterTranslator.scala new file mode 100644 index 00000000000..4d17cdd026f --- /dev/null +++ b/spark/v2/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DeltaV2FilterTranslator.scala @@ -0,0 +1,53 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.execution.datasources.v2 + +import org.apache.spark.sql.catalyst.expressions.Expression +import org.apache.spark.sql.execution.datasources.DataSourceStrategy +import org.apache.spark.sql.sources.Filter + +/** + * Bridges Catalyst filter [[Expression]]s to data source [[Filter]]s for the Delta v2 connector. + * + * Lives in the `org.apache.spark.sql.execution.datasources.v2` package so it can reach + * `DataSourceStrategy.translateFilter`, which is `protected[sql]` and therefore not callable from + * the connector's `io.delta.spark.internal.v2.read` package (nor from Java). + * + * TODO: Revisit both call sites once the scan implements the file-source Scan interface. Use (a) + * below exists only to serve `DeltaV2Scan.toBatch`, which becomes unreachable at that point (see + * the TODO there), so that use -- and with it this object's reason to live in a Spark-internal + * package -- should go away. Use (b), translating filters for the Kernel predicate in + * `DeltaV2ScanBuilder`, is independent of `Batch` and needs its own assessment. + */ +object DeltaV2FilterTranslator { + + /** + * Translates each Catalyst filter to a data source [[Filter]], dropping any that cannot be + * converted. The v2 connector selects files at scan-build time via V1 data skipping, so these + * translated filters are used (a) as `DeltaV2Batch`'s data filters, which distinguish two batches + * that select different files under otherwise-equal state and prevent `BatchScanExec` from + * reusing an exchange/subquery across scans with different pushed filters, and (b) for Parquet + * predicate pushdown (partition-column filters are harmlessly ignored by the Parquet reader, + * which only pushes filters referencing physical data columns). + * + * @param filters Catalyst filter expressions pushed to the scan. + * @return the subset that translates to a data source [[Filter]]. + */ + def translate(filters: Array[Expression]): Array[Filter] = + filters + .flatMap(f => DataSourceStrategy.translateFilter(f, supportNestedPredicatePushdown = true)) +} diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/DeltaV2TestBase.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/DeltaV2TestBase.java index b00222ee3f2..e25ed343d6c 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/DeltaV2TestBase.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/DeltaV2TestBase.java @@ -17,7 +17,24 @@ import io.delta.kernel.defaults.engine.DefaultEngine; import io.delta.kernel.engine.Engine; +import io.delta.spark.internal.v2.read.DeltaV2ScanBuilder; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.Collections; import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.catalyst.expressions.AttributeReference; +import org.apache.spark.sql.catalyst.expressions.Expression; +import org.apache.spark.sql.catalyst.expressions.Literal; +import org.apache.spark.sql.sources.And; +import org.apache.spark.sql.sources.EqualTo; +import org.apache.spark.sql.sources.Filter; +import org.apache.spark.sql.sources.GreaterThan; +import org.apache.spark.sql.sources.Not; +import org.apache.spark.sql.sources.Or; +import org.apache.spark.sql.sources.StringEndsWith; +import org.apache.spark.sql.sources.StringStartsWith; +import org.apache.spark.sql.types.StructField; +import org.apache.spark.sql.types.StructType; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -50,6 +67,17 @@ public static void tearDownSpark() { } } + /** Returns a fresh, unique table directory path under the JVM temp dir. */ + protected static String newTablePath(String prefix) { + return Paths.get(System.getProperty("java.io.tmpdir"), prefix + "-" + System.nanoTime()) + .toString(); + } + + /** Returns the DSv2-catalog table reference for a path-based Delta table. */ + protected static String dsv2Table(String tablePath) { + return String.format("dsv2.delta.`%s`", tablePath); + } + protected void createTestTableWithData(String path, String tableName) { spark.sql( String.format( @@ -158,4 +186,79 @@ protected static void createPartitionedTable(String tableName, String path) { + "('2', '20181212', 'sz', 'Eve', '50')", tableName)); } + + /** Pushes legacy test filters through the Catalyst filter API used by DeltaV2ScanBuilder. */ + protected static void pushFilters(DeltaV2ScanBuilder builder, Filter... filters) { + Expression[] expressions = + Arrays.stream(filters) + .map(filter -> toCatalystFilter(builder, filter)) + .toArray(Expression[]::new); + scala.collection.immutable.Seq expressionSeq = + scala.jdk.javaapi.CollectionConverters.asScala(Arrays.asList(expressions)).toList(); + builder.pushFilters(expressionSeq); + } + + protected static Expression toCatalystFilter(DeltaV2ScanBuilder builder, Filter filter) { + // Concatenate data + partition fields via add() rather than StructType.merge: of the two merge + // overloads, one is not available in every build and the other is private[sql], so neither is + // portable. add() exists everywhere and suffices here (the schemas share no column names). + StructType tableSchema = builder.getDataSchema(); + for (StructField field : builder.getPartitionSchema().fields()) { + tableSchema = tableSchema.add(field); + } + return toCatalystFilter(filter, tableSchema); + } + + private static Expression toCatalystFilter(Filter filter, StructType tableSchema) { + if (filter instanceof EqualTo) { + EqualTo equalTo = (EqualTo) filter; + AttributeReference attribute = catalystAttribute(equalTo.attribute(), tableSchema); + return new org.apache.spark.sql.catalyst.expressions.EqualTo( + attribute, Literal.create(equalTo.value(), attribute.dataType())); + } + if (filter instanceof GreaterThan) { + GreaterThan greaterThan = (GreaterThan) filter; + AttributeReference attribute = catalystAttribute(greaterThan.attribute(), tableSchema); + return new org.apache.spark.sql.catalyst.expressions.GreaterThan( + attribute, Literal.create(greaterThan.value(), attribute.dataType())); + } + if (filter instanceof And) { + And and = (And) filter; + return new org.apache.spark.sql.catalyst.expressions.And( + toCatalystFilter(and.left(), tableSchema), toCatalystFilter(and.right(), tableSchema)); + } + if (filter instanceof Or) { + Or or = (Or) filter; + return new org.apache.spark.sql.catalyst.expressions.Or( + toCatalystFilter(or.left(), tableSchema), toCatalystFilter(or.right(), tableSchema)); + } + if (filter instanceof Not) { + return new org.apache.spark.sql.catalyst.expressions.Not( + toCatalystFilter(((Not) filter).child(), tableSchema)); + } + if (filter instanceof StringStartsWith) { + StringStartsWith startsWith = (StringStartsWith) filter; + AttributeReference attribute = catalystAttribute(startsWith.attribute(), tableSchema); + return new org.apache.spark.sql.catalyst.expressions.StartsWith( + attribute, Literal.create(startsWith.value(), attribute.dataType())); + } + if (filter instanceof StringEndsWith) { + StringEndsWith endsWith = (StringEndsWith) filter; + AttributeReference attribute = catalystAttribute(endsWith.attribute(), tableSchema); + return new org.apache.spark.sql.catalyst.expressions.EndsWith( + attribute, Literal.create(endsWith.value(), attribute.dataType())); + } + throw new IllegalArgumentException("Unsupported test filter: " + filter); + } + + private static AttributeReference catalystAttribute(String name, StructType tableSchema) { + StructField field = tableSchema.apply(name); + return new AttributeReference( + name, + field.dataType(), + field.nullable(), + field.metadata(), + org.apache.spark.sql.catalyst.expressions.NamedExpression.newExprId(), + scala.jdk.javaapi.CollectionConverters.asScala(Collections.emptyList()).toList()); + } } diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/V2LimitPushdownTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/V2LimitPushdownTest.java index ff78c71cc0c..cd5f78cb2a7 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/V2LimitPushdownTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/V2LimitPushdownTest.java @@ -248,12 +248,20 @@ public void testLimitWithPartitionFilter(@TempDir File tempDir) { "INSERT INTO delta.`%s` VALUES (1, 'a'), (2, 'a'), (3, 'a'), (4, 'b'), (5, 'b')", tablePath)); - // A partition filter is fully pushed (no post-scan residual), so limit pushdown stays active. + // Whether the partition filter and limit are pushed depends on optimizer support. The query + // result must remain correct either way. long count = spark.sql(str("SELECT * FROM dsv2.delta.`%s` WHERE part = 'a' LIMIT 2", tablePath)).count(); assertEquals(2, count, "Partition filter + LIMIT should return 2 rows"); - assertLimitPushedForQuery( - str("SELECT * FROM dsv2.delta.`%s` WHERE part = 'a' LIMIT 2", tablePath), 2); + if (spark.version().startsWith("4.0")) { + // Spark 4.0 does not route SupportsPushDownCatalystFilters through PushDownUtils, so the + // partition filter remains above the scan and prevents limit pushdown. + assertLimitNotPushedForQuery( + str("SELECT * FROM dsv2.delta.`%s` WHERE part = 'a' LIMIT 2", tablePath)); + } else { + assertLimitPushedForQuery( + str("SELECT * FROM dsv2.delta.`%s` WHERE part = 'a' LIMIT 2", tablePath), 2); + } assertOrderedV1V2SanityWithFilter(tablePath, "part = 'a'", "id", 2); } diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/V2MetadataReadTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/V2MetadataReadTest.java index ac7ef93f5ee..353142dc828 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/V2MetadataReadTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/V2MetadataReadTest.java @@ -21,7 +21,6 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.File; -import java.nio.file.Paths; import java.sql.Timestamp; import java.util.List; import org.apache.spark.sql.Dataset; @@ -196,10 +195,6 @@ public void testSelectMetadataAlongsideStar() { // Helpers // --------------------------------------------------------------------------- - private static String newTablePath(String prefix) { - return Paths.get(System.getProperty("java.io.tmpdir"), prefix + System.nanoTime()).toString(); - } - private void createTable(String path, boolean rowTrackingEnabled) { spark.sql( String.format( diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/catalog/DeltaV2TableTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/catalog/DeltaV2TableTest.java index d712339d3e7..053f95eded5 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/catalog/DeltaV2TableTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/catalog/DeltaV2TableTest.java @@ -23,8 +23,15 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.mockConstruction; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import io.delta.kernel.internal.SnapshotImpl; import io.delta.kernel.internal.actions.Metadata; @@ -34,6 +41,9 @@ import io.delta.spark.internal.v2.adapters.KernelProtocolAdapter; import io.delta.spark.internal.v2.read.cdc.CDCSchemaContext; import io.delta.spark.internal.v2.snapshot.PathBasedSnapshotManager; +import io.delta.spark.internal.v2.tablemanager.CachedSnapshotManager; +import io.delta.spark.internal.v2.tablemanager.DeltaV2TableManager; +import io.delta.spark.internal.v2.tablemanager.DeltaV2TableManagerCache$; import java.io.File; import java.lang.reflect.Method; import java.net.URI; @@ -45,9 +55,14 @@ import java.util.Map; import java.util.Optional; import java.util.UUID; +import java.util.concurrent.Callable; import java.util.function.BiFunction; import java.util.stream.Stream; import org.apache.hadoop.fs.Path; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.RowFactory; +import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.catalyst.TableIdentifier; import org.apache.spark.sql.catalyst.catalog.CatalogTable; import org.apache.spark.sql.catalyst.expressions.FileSourceConstantMetadataStructField; @@ -58,10 +73,14 @@ import org.apache.spark.sql.connector.expressions.Transform; import org.apache.spark.sql.connector.write.LogicalWriteInfo; import org.apache.spark.sql.delta.DeltaOptions; +import org.apache.spark.sql.delta.Snapshot; import org.apache.spark.sql.delta.catalog.DeltaTableV2; import org.apache.spark.sql.delta.sources.DeltaSQLConf; import org.apache.spark.sql.delta.sources.DeltaSourceMetadataTrackingLog; import org.apache.spark.sql.delta.sources.PersistedMetadata; +import org.apache.spark.sql.delta.v2.interop.DeltaV2QueryContext; +import org.apache.spark.sql.delta.v2.interop.DeltaV2QueryContext$; +import org.apache.spark.sql.delta.v2.interop.DeltaV2Snapshot$; import org.apache.spark.sql.execution.datasources.FileFormat$; import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.StructType; @@ -70,7 +89,10 @@ import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.ArgumentCaptor; +import org.mockito.MockedConstruction; import scala.Option; public class DeltaV2TableTest extends DeltaV2TestBase { @@ -531,6 +553,190 @@ public void testVersionReturnsSnapshotVersion(@TempDir File tempDir) { "version() should remain pinned to the construction-time snapshot"); } + @ParameterizedTest(name = "{0}") + @EnumSource(ConstructionMethod.class) + public void testUsesTableManagerKernelContextAndLoadsSnapshotsAcrossSessions( + ConstructionMethod method, @TempDir File tempDir) throws Exception { + String path = tempDir.getAbsolutePath(); + String tableName = "test_table_manager_" + method.name().toLowerCase(); + spark.sql(String.format("CREATE TABLE %s (id INT) USING delta LOCATION '%s'", tableName, path)); + Identifier identifier = Identifier.of(new String[] {"default"}, tableName); + SparkSession sessionA = spark.newSession(); + SparkSession sessionB = spark.newSession(); + Map tableOptions = recordingFileSystemOptions(sessionA, sessionB); + + DeltaV2TableManagerCache$.MODULE$.clearCache(); + try { + CatalogTable catalogTableA = + withActiveSession( + sessionA, + () -> + sessionA + .sessionState() + .catalog() + .getTableMetadata(new TableIdentifier(tableName))); + CatalogTable catalogTableB = + withActiveSession( + sessionB, + () -> + sessionB + .sessionState() + .catalog() + .getTableMetadata(new TableIdentifier(tableName))); + Option catalogTableOptA = + method == ConstructionMethod.FROM_PATH ? Option.empty() : Option.apply(catalogTableA); + Option catalogTableOptB = + method == ConstructionMethod.FROM_PATH ? Option.empty() : Option.apply(catalogTableB); + DeltaV2Table tableA = + withActiveSession( + sessionA, + () -> + method == ConstructionMethod.FROM_PATH + ? new DeltaV2Table(identifier, path, tableOptions) + : new DeltaV2Table(identifier, catalogTableA, tableOptions)); + DeltaV2Table tableB = + withActiveSession( + sessionB, + () -> + method == ConstructionMethod.FROM_PATH + ? new DeltaV2Table(identifier, path, tableOptions) + : new DeltaV2Table(identifier, catalogTableB, tableOptions)); + DeltaV2TableManager managerA = + DeltaV2TableManagerCache$.MODULE$.forTable( + sessionA, path, tableOptions, catalogTableOptA); + DeltaV2TableManager managerB = + DeltaV2TableManagerCache$.MODULE$.forTable( + sessionB, path, tableOptions, catalogTableOptB); + + assertTrue(tableA.getSnapshotManager() instanceof CachedSnapshotManager); + assertTrue(tableB.getSnapshotManager() instanceof CachedSnapshotManager); + assertSame(managerA, managerB); + assertSame(managerA.kernelContext(), managerB.kernelContext()); + assertSame(managerA.kernelContext().getDefaultEngine(), tableA.kernelEngine()); + assertSame(tableA.kernelEngine(), tableB.kernelEngine()); + assertEquals("0", tableA.version()); + assertEquals("0", tableB.version()); + + withActiveSession( + sessionA, + () -> { + sessionA.sql(String.format("INSERT INTO %s VALUES (1)", tableName)); + assertLatestSnapshot(tableA, sessionA, 1L, 1L); + return null; + }); + withActiveSession( + sessionB, + () -> { + sessionB.sql(String.format("INSERT INTO %s VALUES (2)", tableName)); + assertLatestSnapshot(tableB, sessionB, 2L, 2L); + return null; + }); + } finally { + DeltaV2TableManagerCache$.MODULE$.clearCache(); + spark.sql(String.format("DROP TABLE IF EXISTS %s", tableName)); + } + } + + @Test + public void testNamedTableQueryLoadsSnapshotWithItsQueryContext(@TempDir File tempDir) + throws Exception { + String path = tempDir.getAbsolutePath(); + String tableName = "test_query_context_snapshot_load"; + spark.sql(String.format("CREATE TABLE %s (id INT) USING delta LOCATION '%s'", tableName, path)); + spark.sql(String.format("INSERT INTO %s VALUES (1), (2)", tableName)); + CatalogTable catalogTable = + spark.sessionState().catalog().getTableMetadata(new TableIdentifier(tableName)); + TableIdentifier expectedIdentifier = catalogTable.identifier(); + + Snapshot realSnapshot = + new PathBasedSnapshotManager(path, spark.sessionState().newHadoopConf()) + .loadLatestSnapshot(); + AssertionError legacyApiError = + new AssertionError("DeltaV2Table query used the legacy snapshot API"); + DeltaV2TableManagerCache$.MODULE$.clearCache(); + try (MockedConstruction mockedManagers = + mockConstruction( + CachedSnapshotManager.class, + (mock, context) -> { + when(mock.loadLatestSnapshot(any(DeltaV2QueryContext.class))) + .thenReturn(realSnapshot); + when(mock.loadLatestSnapshot()).thenThrow(legacyApiError); + })) { + ThrowingRunnable checkQuery = + () -> + withSQLConf( + "spark.sql.catalog.query_context_test", + QueryContextTestCatalog.class.getName(), + () -> { + String query = + String.format( + "SELECT id FROM query_context_test.default.%s ORDER BY id", tableName); + List rows = spark.sql(query).collectAsList(); + assertEquals(Arrays.asList(RowFactory.create(1), RowFactory.create(2)), rows); + }); + checkQuery.run(); + + assertEquals(1, mockedManagers.constructed().size()); + CachedSnapshotManager manager = mockedManagers.constructed().get(0); + ArgumentCaptor queryContextCaptor = + ArgumentCaptor.forClass(DeltaV2QueryContext.class); + verify(manager, atLeastOnce()).loadLatestSnapshot(queryContextCaptor.capture()); + verify(manager, never()).loadLatestSnapshot(); + DeltaV2QueryContext queryContext = queryContextCaptor.getValue(); + assertTrue(queryContext.catalogTableOpt().isDefined()); + TableIdentifier capturedIdentifier = queryContext.catalogTableOpt().get().identifier(); + assertEquals(expectedIdentifier, capturedIdentifier); + } finally { + DeltaV2TableManagerCache$.MODULE$.clearCache(); + spark.sql(String.format("DROP TABLE IF EXISTS %s", tableName)); + } + } + + /** Loads session-catalog metadata through V2 for query-context propagation tests. */ + public static class QueryContextTestCatalog extends TestCatalog { + @Override + public DeltaV2Table loadTable(Identifier identifier) { + TableIdentifier tableIdentifier = + new TableIdentifier(identifier.name(), Option.apply(identifier.namespace()[0])); + try { + CatalogTable catalogTable = + SparkSession.active().sessionState().catalog().getTableMetadata(tableIdentifier); + return new DeltaV2Table(identifier, catalogTable, Collections.emptyMap()); + } catch (Exception e) { + throw new RuntimeException("Failed to load table: " + identifier, e); + } + } + } + + private static Map recordingFileSystemOptions( + SparkSession sessionA, SparkSession sessionB) { + return Collections.emptyMap(); + } + + private static void assertLatestSnapshot( + DeltaV2Table table, SparkSession activeSession, long expectedVersion, long expectedFiles) { + DeltaV2QueryContext queryContext = DeltaV2QueryContext$.MODULE$.apply(table.getCatalogTable()); + Snapshot snapshot = table.getSnapshotManager().loadLatestSnapshot(queryContext); + Dataset allFiles = snapshot.allFiles(); + assertEquals(expectedVersion, snapshot.version()); + assertSame(activeSession, allFiles.sparkSession()); + assertEquals(expectedFiles, allFiles.count()); + } + + private static T withActiveSession(SparkSession session, Callable body) throws Exception { + Option originalSession = SparkSession.getActiveSession(); + SparkSession.setActiveSession(session); + try { + return body.call(); + } finally { + if (originalSession.isDefined()) { + SparkSession.setActiveSession(originalSession.get()); + } else { + SparkSession.clearActiveSession(); + } + } + } + @Test public void testNewWriteBuilderReturnsWriteBuilder(@TempDir File tempDir) throws Exception { String path = tempDir.getAbsolutePath(); @@ -787,7 +993,10 @@ private void verifyPersistedEntryDrivesSparkTableSchema( // Capture v0 metadata BEFORE evolving the table. PathBasedSnapshotManager snapshotManager = new PathBasedSnapshotManager(tablePath, spark.sessionState().newHadoopConf()); - SnapshotImpl snapshotV0 = (SnapshotImpl) snapshotManager.loadSnapshotAt(0L); + SnapshotImpl snapshotV0 = + DeltaV2Snapshot$.MODULE$.getKernelSnapshot( + snapshotManager.loadSnapshotAt( + 0L, DeltaV2QueryContext$.MODULE$.apply(Optional.empty()))); Metadata metadataV0 = snapshotV0.getMetadata(); Protocol protocolV0 = snapshotV0.getProtocol(); String tableId = metadataV0.getId(); @@ -925,14 +1134,16 @@ public void testSchemaTracking_throwsWhenFeatureFlagDisabled(@TempDir File tempD /** withVersion(N) returns a copy pinned to the snapshot at version N. */ @Test - public void testWithVersionPinsToHistoricalSnapshot(@TempDir File tempDir) { + public void testWithVersionPinsToHistoricalSnapshot(@TempDir File tempDir) throws Exception { String path = tempDir.getAbsolutePath(); spark.sql( String.format("CREATE TABLE test_with_version (id INT) USING delta LOCATION '%s'", path)); spark.sql("ALTER TABLE test_with_version ADD COLUMNS (name STRING)"); Identifier identifier = Identifier.of(new String[] {"default"}, "test_with_version"); - DeltaV2Table latest = new DeltaV2Table(identifier, path); + CatalogTable catalogTable = + spark.sessionState().catalog().getTableMetadata(new TableIdentifier("test_with_version")); + DeltaV2Table latest = new DeltaV2Table(identifier, catalogTable, Collections.emptyMap()); assertEquals(2, latest.schema().fields().length); // withVersion(0) pins to the historical snapshot. @@ -987,14 +1198,16 @@ public void testWithVersionRejectsOutOfRangeVersion(@TempDir File tempDir) { /** withTimestamp(t) resolves to the commit active at t and pins to that snapshot. */ @Test - public void testWithTimestampPinsToHistoricalSnapshot(@TempDir File tempDir) { + public void testWithTimestampPinsToHistoricalSnapshot(@TempDir File tempDir) throws Exception { String path = tempDir.getAbsolutePath(); spark.sql( String.format("CREATE TABLE test_with_timestamp (id INT) USING delta LOCATION '%s'", path)); spark.sql("ALTER TABLE test_with_timestamp ADD COLUMNS (name STRING)"); Identifier identifier = Identifier.of(new String[] {"default"}, "test_with_timestamp"); - DeltaV2Table latest = new DeltaV2Table(identifier, path); + CatalogTable catalogTable = + spark.sessionState().catalog().getTableMetadata(new TableIdentifier("test_with_timestamp")); + DeltaV2Table latest = new DeltaV2Table(identifier, catalogTable, Collections.emptyMap()); assertEquals(2, latest.schema().fields().length); // The timestamp of the v0 commit resolves back to v0. diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/DeltaV2BatchTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/DeltaV2BatchTest.java index bb5c8a3f847..4e64154b8ad 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/DeltaV2BatchTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/DeltaV2BatchTest.java @@ -56,9 +56,8 @@ public static void setupPartitionedTable(@TempDir File tempDir) { Identifier.of(new String[] {"spark_catalog", "default"}, tableName), tablePath, options); // Cases where two batches built from the same table must be equal. city/date are partition - // columns (exercise pushedToKernelFiltersSet); name/cnt are data columns and per - // ExpressionUtils.classifyFilter have isDataFilter=true, so they flow into - // DeltaV2Batch.dataFilters and exercise the dataFiltersSet branch of equals/hashCode. + // columns; name/cnt are data columns. Both kinds flow into DeltaV2Batch.filterSet for + // equals/hashCode, but only data-column filters are passed to the Parquet reader. static Stream equalBatchCasesProvider() { Filter cityEq = new EqualTo("city", "hz"); Filter dateEq = new EqualTo("date", "20180520"); @@ -82,13 +81,13 @@ static Stream equalBatchCasesProvider() { public void testEqualsAndHashCode(String description, Filter[] filters1, Filter[] filters2) { DeltaV2ScanBuilder builder1 = (DeltaV2ScanBuilder) table.newScanBuilder(options); if (filters1.length > 0) { - builder1.pushFilters(filters1); + pushFilters(builder1, filters1); } Batch batch1 = ((DeltaV2Scan) builder1.build()).toBatch(); DeltaV2ScanBuilder builder2 = (DeltaV2ScanBuilder) table.newScanBuilder(options); if (filters2.length > 0) { - builder2.pushFilters(filters2); + pushFilters(builder2, filters2); } Batch batch2 = ((DeltaV2Scan) builder2.build()).toBatch(); @@ -102,7 +101,7 @@ public void testEqualsWithDifferentPushedFilters() { Batch batch1 = ((DeltaV2Scan) builder1.build()).toBatch(); DeltaV2ScanBuilder builder2 = (DeltaV2ScanBuilder) table.newScanBuilder(options); - builder2.pushFilters(new Filter[] {new EqualTo("city", "hz")}); + pushFilters(builder2, new EqualTo("city", "hz")); Batch batch2 = ((DeltaV2Scan) builder2.build()).toBatch(); assertNotEquals(batch1, batch2); diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/DeltaV2MicroBatchStreamCDCTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/DeltaV2MicroBatchStreamCDCTest.java index 0e984106617..7c32d61b6d2 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/DeltaV2MicroBatchStreamCDCTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/DeltaV2MicroBatchStreamCDCTest.java @@ -46,6 +46,7 @@ import org.apache.spark.sql.delta.sources.DeltaSourceOffset; import org.apache.spark.sql.delta.sources.DeltaSourceOffset$; import org.apache.spark.sql.delta.storage.ClosableIterator; +import org.apache.spark.sql.delta.v2.interop.DeltaV2Snapshot$; import org.apache.spark.sql.types.StructType; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -111,7 +112,7 @@ public void testValidateCDFEnabled_passesWhenEnabledAtStartVersionEqualToInit( DeltaV2MicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); - long initVersion = snapshotManager.loadLatestSnapshot().getVersion(); + long initVersion = snapshotManager.loadLatestSnapshot().version(); assertDoesNotThrow(() -> stream.validateCDFEnabledOnTable(initVersion)); } @@ -152,7 +153,7 @@ public void testValidateCDFEnabled_swallowsForUnmaterializedStartVersion(@TempDi PathBasedSnapshotManager snapshotManager = new PathBasedSnapshotManager(tablePath, hadoopConf); DeltaV2MicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); - long latestVersion = snapshotManager.loadLatestSnapshot().getVersion(); + long latestVersion = snapshotManager.loadLatestSnapshot().version(); // startingVersion=latest resolves to latest+1, which is not materialized. The KernelException // is swallowed and the validator returns without throwing. @@ -1035,10 +1036,9 @@ private DeltaV2MicroBatchStream createStreamWithSeededTrackingLog( ScalaUtils.toScalaMap(javaOptions); DeltaOptions deltaOptions = new DeltaOptions(scalaOptions, spark.sessionState().conf()); - io.delta.kernel.internal.SnapshotImpl latestSnapshot = - (io.delta.kernel.internal.SnapshotImpl) snapshotManager.loadLatestSnapshot(); + Snapshot latestSnapshot = snapshotManager.loadLatestSnapshot(); io.delta.kernel.internal.SnapshotImpl seededSnapshot = - (io.delta.kernel.internal.SnapshotImpl) snapshotManager.loadSnapshotAt(seededVersion); + DeltaV2Snapshot$.MODULE$.getKernelSnapshot(snapshotManager.loadSnapshotAt(seededVersion)); org.apache.spark.sql.delta.sources.DeltaSourceMetadataTrackingLog trackingLog = MetadataEvolutionHandler.getMetadataTrackingLogForMicroBatchStream( @@ -1182,10 +1182,8 @@ private DeltaSource createDeltaSource(DeltaLog deltaLog, String tablePath, Delta private DeltaV2MicroBatchStream createTestStreamWithDefaults( PathBasedSnapshotManager snapshotManager, Configuration hadoopConf, DeltaOptions options) { - io.delta.kernel.Snapshot snapshot = snapshotManager.loadLatestSnapshot(); - StructType tableSchema = - io.delta.spark.internal.v2.utils.SchemaUtils.convertKernelSchemaToSparkSchema( - snapshot.getSchema()); + Snapshot snapshot = snapshotManager.loadLatestSnapshot(); + StructType tableSchema = snapshot.schema(); return new DeltaV2MicroBatchStream( snapshotManager, snapshot, diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/DeltaV2MicroBatchStreamTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/DeltaV2MicroBatchStreamTest.java index ff1b9aa7792..d48693e1c5b 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/DeltaV2MicroBatchStreamTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/DeltaV2MicroBatchStreamTest.java @@ -54,6 +54,8 @@ import org.apache.spark.sql.connector.read.streaming.Offset; import org.apache.spark.sql.connector.read.streaming.ReadLimit; import org.apache.spark.sql.delta.*; +import org.apache.spark.sql.delta.amt.AMTTriggerMode; +import org.apache.spark.sql.delta.files.DeltaSourceSnapshot; import org.apache.spark.sql.delta.sources.DeltaSQLConf; import org.apache.spark.sql.delta.sources.DeltaSource; import org.apache.spark.sql.delta.sources.DeltaSourceMetadataTrackingLog; @@ -77,10 +79,7 @@ public class DeltaV2MicroBatchStreamTest extends DeltaV2TestBase { - /** - * Helper method to create a minimal DeltaV2MicroBatchStream instance for tests that only check - * for UnsupportedOperationException. - */ + /** Helper method to create a minimal DeltaV2MicroBatchStream instance. */ private DeltaV2MicroBatchStream createTestStream(File tempDir) { String tablePath = tempDir.getAbsolutePath(); String tableName = "test_unsupported_" + System.nanoTime(); @@ -142,6 +141,58 @@ public void testInitialOffset_withInitialSnapshot(@TempDir File tempDir) throws deltaOffset.isInitialSnapshot(), "Initial offset should be marked as initial snapshot"); } + @Test + public void testInitialOffset_doesNotResetFirstBatchState(@TempDir File tempDir) + throws Exception { + String testTablePath = tempDir.getAbsolutePath(); + String testTableName = "test_idempotent_initial_offset_" + System.nanoTime(); + createEmptyTestTable(testTablePath, testTableName); + insertVersions( + testTableName, + /* numVersions= */ 1, + /* rowsPerVersion= */ 1, + /* includeEmptyVersion= */ false); + + Configuration hadoopConf = new Configuration(); + PathBasedSnapshotManager snapshotManager = + new PathBasedSnapshotManager(testTablePath, hadoopConf); + DeltaV2MicroBatchStream stream = + createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); + + Offset initialOffset = stream.initialOffset(); + Offset firstEndOffset = stream.latestOffset(initialOffset, ReadLimit.allAvailable()); + assertNotNull(firstEndOffset); + + // MicroBatchExecution calls initialOffset again while constructing the first batch's scan. + assertEquals(initialOffset, stream.initialOffset()); + + // With no new data, a subsequent batch returns its start offset rather than null. + assertEquals(firstEndOffset, stream.latestOffset(firstEndOffset, ReadLimit.allAvailable())); + } + + @Test + public void testInitialOffset_noDataRemainsFirstBatch(@TempDir File tempDir) throws Exception { + String testTablePath = tempDir.getAbsolutePath(); + String testTableName = "test_initial_offset_no_data_" + System.nanoTime(); + createEmptyTestTable(testTablePath, testTableName); + + scala.collection.immutable.Map scalaOptions = + Map$.MODULE$.empty().updated("startingVersion", "latest"); + DeltaOptions options = new DeltaOptions(scalaOptions, spark.sessionState().conf()); + Configuration hadoopConf = new Configuration(); + PathBasedSnapshotManager snapshotManager = + new PathBasedSnapshotManager(testTablePath, hadoopConf); + DeltaV2MicroBatchStream stream = + createTestStreamWithDefaults(snapshotManager, hadoopConf, options); + + Offset initialOffset = stream.initialOffset(); + assertNull(stream.latestOffset(initialOffset, ReadLimit.allAvailable())); + + Offset cachedInitialOffset = stream.initialOffset(); + assertEquals(initialOffset, cachedInitialOffset); + assertNull(stream.latestOffset(cachedInitialOffset, ReadLimit.allAvailable())); + } + @Test public void testDeserializeOffset_ValidJson(@TempDir File tempDir) throws Exception { String tablePath = tempDir.getAbsolutePath(); @@ -486,7 +537,14 @@ private static Stream getFileChangesParameters() { isInitialSnapshot, noEndVersion, noEndIndex, - "InitialSnapshot_LatestVersion_NoDelta")); + "InitialSnapshot_LatestVersion_NoDelta"), + Arguments.of( + 5L, + 1L, + isInitialSnapshot, + Optional.of(5L), + Optional.of(3L), + "InitialSnapshot_MidSnapshotRange")); } // ================================================================================================ @@ -2060,10 +2118,7 @@ public void testPlanInputPartitions_dataParity( partitionSchema, dataSchema, SchemaUtils.ddlOrderedOutputSchema( - io.delta.spark.internal.v2.utils.SchemaUtils.convertKernelSchemaToSparkSchema( - snapshotManager.loadLatestSnapshot().getSchema()), - dataSchema, - partitionSchema), + snapshotManager.loadLatestSnapshot().schema(), dataSchema, partitionSchema), new org.apache.spark.sql.sources.Filter[0], Map$.MODULE$.empty(), Option.empty(), @@ -2248,10 +2303,7 @@ public void testPlanInputPartitions_excludeRegex( partitionSchema, dataSchema, SchemaUtils.ddlOrderedOutputSchema( - io.delta.spark.internal.v2.utils.SchemaUtils.convertKernelSchemaToSparkSchema( - snapshotManager.loadLatestSnapshot().getSchema()), - dataSchema, - partitionSchema), + snapshotManager.loadLatestSnapshot().schema(), dataSchema, partitionSchema), new org.apache.spark.sql.sources.Filter[0], Map$.MODULE$.empty(), Option.empty(), @@ -2620,7 +2672,7 @@ public void testGetStartingVersion_protocolValidationNonFeatureExceptionFallback Snapshot snapshotV10 = deltaLog.getSnapshotAt( 10, Option.empty(), Option.empty(), false); - deltaLog.checkpoint(snapshotV10, Option.empty()); + deltaLog.checkpoint(snapshotV10, Option.empty(), Option.empty()); // Simulate log cleanup by timestamp: delete logs 0-5 // This makes version 7 non-recreatable while allowing DeltaLog to load the latest snapshot @@ -3362,9 +3414,7 @@ public void testSchemaTrackingLifecycle_forward( assertPostChangeSchema.accept(evolved.dataSchema()); // Restart with the post-change schema (same trackingLog: it now carries the evolved entry). - StructType postChangeSchema = - io.delta.spark.internal.v2.utils.SchemaUtils.convertKernelSchemaToSparkSchema( - snapshotManager.loadLatestSnapshot().getSchema()); + StructType postChangeSchema = snapshotManager.loadLatestSnapshot().schema(); DeltaV2MicroBatchStream streamPostChange = createSchemaTrackingTestStream( snapshotManager, @@ -3458,9 +3508,7 @@ public void testSchemaTrackingLifecycle_backfill( createTrackingLog(snapshotManager, schemaTrackingLocation, checkpointLocation, optionMap); StructType preChangeSchema = loadSparkSchemaAtVersion(snapshotManager, startVersion); - StructType postChangeSchema = - io.delta.spark.internal.v2.utils.SchemaUtils.convertKernelSchemaToSparkSchema( - snapshotManager.loadLatestSnapshot().getSchema()); + StructType postChangeSchema = snapshotManager.loadLatestSnapshot().schema(); // Round 1: post-change schema (analysis would bind to the latest snapshot since the log // is empty). First latestOffset runs eager-init and throws. @@ -3575,8 +3623,7 @@ public void testSchemaTrackingRejectsFailOnDataLossFalse(@TempDir File tempDir) String schemaTrackingLocation = new File(tempDir, "schema_tracking").getAbsolutePath(); String checkpointLocation = new File(tempDir, "checkpoint").getAbsolutePath(); StructType schema = - loadSparkSchemaAtVersion( - snapshotManager, snapshotManager.loadLatestSnapshot().getVersion()); + loadSparkSchemaAtVersion(snapshotManager, snapshotManager.loadLatestSnapshot().version()); DeltaSourceMetadataTrackingLog trackingLog = createTrackingLog( snapshotManager, schemaTrackingLocation, checkpointLocation, Collections.emptyMap()); @@ -4292,11 +4339,9 @@ private DeltaSource createDeltaSource(DeltaLog deltaLog, String tablePath, Delta /** Helper method to create a DeltaV2MicroBatchStream with default values for testing. */ private DeltaV2MicroBatchStream createTestStreamWithDefaults( PathBasedSnapshotManager snapshotManager, Configuration hadoopConf, DeltaOptions options) { - io.delta.kernel.Snapshot snapshot = snapshotManager.loadLatestSnapshot(); - String tablePath = ((io.delta.kernel.internal.SnapshotImpl) snapshot).getPath(); - StructType tableSchema = - io.delta.spark.internal.v2.utils.SchemaUtils.convertKernelSchemaToSparkSchema( - snapshot.getSchema()); + Snapshot snapshot = snapshotManager.loadLatestSnapshot(); + String tablePath = snapshot.dataPath().toString(); + StructType tableSchema = snapshot.schema(); return new DeltaV2MicroBatchStream( snapshotManager, snapshot, @@ -4322,7 +4367,7 @@ private DeltaV2MicroBatchStream createSchemaTrackingTestStream( StructType dataSchema, Option metadataTrackingLog, String metadataPath) { - io.delta.kernel.Snapshot snapshot = snapshotManager.loadLatestSnapshot(); + Snapshot snapshot = snapshotManager.loadLatestSnapshot(); return new DeltaV2MicroBatchStream( snapshotManager, snapshot, @@ -4334,10 +4379,7 @@ private DeltaV2MicroBatchStream createSchemaTrackingTestStream( /* partitionSchema= */ new StructType(), /* readDataSchema= */ dataSchema, SchemaUtils.ddlOrderedOutputSchema( - io.delta.spark.internal.v2.utils.SchemaUtils.convertKernelSchemaToSparkSchema( - snapshotManager.loadLatestSnapshot().getSchema()), - dataSchema, - new StructType()), + snapshotManager.loadLatestSnapshot().schema(), dataSchema, new StructType()), /* dataFilters= */ new org.apache.spark.sql.sources.Filter[0], /* scalaOptions= */ scala.collection.immutable.Map$.MODULE$.empty(), metadataTrackingLog, @@ -4349,13 +4391,12 @@ private DeltaSourceMetadataTrackingLog createTrackingLog( String schemaTrackingLocation, String checkpointLocation, java.util.Map optionMap) { - io.delta.kernel.internal.SnapshotImpl snapshot = - (io.delta.kernel.internal.SnapshotImpl) snapshotManager.loadLatestSnapshot(); + Snapshot snapshot = snapshotManager.loadLatestSnapshot(); return DeltaSourceMetadataTrackingLog.create( spark, schemaTrackingLocation, - snapshot.getMetadata().getId(), - snapshot.getPath(), + snapshot.metadata().getId(), + snapshot.dataPath().toString(), ScalaUtils.toScalaMap(optionMap), Option.apply(checkpointLocation), /* mergeConsecutiveSchemaChanges= */ false, @@ -4365,8 +4406,7 @@ private DeltaSourceMetadataTrackingLog createTrackingLog( private StructType loadSparkSchemaAtVersion( PathBasedSnapshotManager snapshotManager, long version) { - return io.delta.spark.internal.v2.utils.SchemaUtils.convertKernelSchemaToSparkSchema( - snapshotManager.loadSnapshotAt(version).getSchema()); + return snapshotManager.loadSnapshotAt(version).schema(); } private List readIdsBetweenOffsets( @@ -4559,55 +4599,6 @@ private void compareStartingVersionResults( String.format("DSv2 getStartingVersion should match for %s", testDescription)); } - @Test - public void testMemoryProtection_initialSnapshotTooLarge(@TempDir File tempDir) throws Exception { - String testTablePath = tempDir.getAbsolutePath(); - String testTableName = "test_memory_protection_" + System.nanoTime(); - createEmptyTestTable(testTablePath, testTableName); - - // At version 5, there will be at least 25 files. - insertVersions( - testTableName, - /* numVersions= */ 10, - /* rowsPerVersion= */ 5, - /* includeEmptyVersion= */ false); - - String configKey = DeltaSQLConf.DELTA_STREAMING_INITIAL_SNAPSHOT_MAX_FILES().key(); - spark.conf().set(configKey, "5"); - - try { - Configuration hadoopConf = new Configuration(); - PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(testTablePath, hadoopConf); - DeltaV2MicroBatchStream stream = - createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); - - long version = 5L; - long fromIndex = DeltaSourceOffset.BASE_INDEX(); - boolean isInitialSnapshot = true; - - RuntimeException exception = - assertThrows( - RuntimeException.class, - () -> { - try (CloseableIterator iter = - stream.getFileChanges( - version, fromIndex, isInitialSnapshot, Optional.empty())) { - while (iter.hasNext()) { - iter.next(); - } - } - }); - - String errorMessage = exception.getMessage(); - assertTrue(errorMessage.contains("DELTA_STREAMING_INITIAL_SNAPSHOT_TOO_LARGE")); - assertTrue( - errorMessage.contains("initial snapshot") || errorMessage.contains("Initial snapshot")); - } finally { - spark.conf().unset(configKey); - } - } - /** * Simulates the Kernel integration path where {@code DefaultJsonHandler.hasNext()} wraps a {@link * ClosedByInterruptException} inside a {@link io.delta.kernel.exceptions.KernelEngineException}. @@ -4917,32 +4908,22 @@ public void testValidateSchema_columnNameCaseDiffers_succeeds() { dataSchema, partitionSchema, snapshotSchema)); } + /** The shared DeltaSourceSnapshot path returns only END sentinel for an empty table. */ @Test - public void testDistributedInitialSnapshot_handlesLargeSnapshot(@TempDir File tempDir) - throws Exception { + public void testDeltaSourceSnapshot_emptyTable(@TempDir File tempDir) throws Exception { String testTablePath = tempDir.getAbsolutePath(); - String testTableName = "test_df_snapshot_" + System.nanoTime(); + String testTableName = "test_df_empty_table_" + System.nanoTime(); createEmptyTestTable(testTablePath, testTableName); - insertVersions( - testTableName, - /* numVersions= */ 10, - /* rowsPerVersion= */ 5, - /* includeEmptyVersion= */ false); - - String maxFilesKey = DeltaSQLConf.DELTA_STREAMING_INITIAL_SNAPSHOT_MAX_FILES().key(); - String dfFlagKey = DeltaSQLConf.DELTA_STREAMING_USE_DISTRIBUTED_INITIAL_SNAPSHOT().key(); - spark.conf().set(maxFilesKey, "5"); - spark.conf().set(dfFlagKey, "true"); + // No insertVersions — table has CREATE TABLE metadata but zero data files + Configuration hadoopConf = spark.sessionState().newHadoopConf(); + PathBasedSnapshotManager snapshotManager = + new PathBasedSnapshotManager(testTablePath, hadoopConf); + DeltaV2MicroBatchStream stream = + createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); try { - Configuration hadoopConf = spark.sessionState().newHadoopConf(); - PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(testTablePath, hadoopConf); - DeltaV2MicroBatchStream stream = - createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); - - long version = snapshotManager.loadLatestSnapshot().getVersion(); + long version = snapshotManager.loadLatestSnapshot().version(); long fromIndex = DeltaSourceOffset.BASE_INDEX(); boolean isInitialSnapshot = true; @@ -4954,210 +4935,83 @@ public void testDistributedInitialSnapshot_handlesLargeSnapshot(@TempDir File te } } - assertTrue(files.size() >= 2, "Should have at least one data file and END sentinel"); - assertEquals(DeltaSourceOffset.END_INDEX(), files.get(files.size() - 1).getIndex()); - - // Verify data files are sorted by (modificationTime, path) - for (int i = 2; i < files.size() - 1; i++) { - IndexedFile prev = files.get(i - 1); - IndexedFile curr = files.get(i); - if (prev.getAddFile() != null && curr.getAddFile() != null) { - long prevTime = prev.getAddFile().getModificationTime(); - long currTime = curr.getAddFile().getModificationTime(); - assertTrue( - prevTime < currTime - || (prevTime == currTime - && prev.getAddFile().getPath().compareTo(curr.getAddFile().getPath()) <= 0), - "Files should be sorted by (modificationTime, path)"); - } - } + // Only END sentinel — no data files, BEGIN filtered by boundary check. + assertEquals(1, files.size()); + assertEquals(DeltaSourceOffset.END_INDEX(), files.get(0).getIndex()); + assertNull(files.get(0).getAddFile()); + + assertTrue(files.stream().noneMatch(IndexedFile::hasFileAction)); } finally { - spark.conf().unset(maxFilesKey); - spark.conf().unset(dfFlagKey); + stream.stop(); } } - /** Verifies indices are contiguous 0-based (not partition-relative). */ @Test - public void testDistributedInitialSnapshot_hasContiguousIndices(@TempDir File tempDir) + public void testInitialSnapshotIsReleasedByFirstNonInitialCommit(@TempDir File tempDir) throws Exception { - String testTablePath = tempDir.getAbsolutePath(); - String testTableName = "test_df_contiguous_idx_" + System.nanoTime(); - createEmptyTestTable(testTablePath, testTableName); - - // Multiple versions force multiple partitions, exposing non-contiguous ID bugs. - insertVersions( - testTableName, - /* numVersions= */ 10, - /* rowsPerVersion= */ 5, - /* includeEmptyVersion= */ false); - - String maxFilesKey = DeltaSQLConf.DELTA_STREAMING_INITIAL_SNAPSHOT_MAX_FILES().key(); - String dfFlagKey = DeltaSQLConf.DELTA_STREAMING_USE_DISTRIBUTED_INITIAL_SNAPSHOT().key(); - spark.conf().set(maxFilesKey, "10000"); - spark.conf().set(dfFlagKey, "true"); - + DeltaV2MicroBatchStream stream = createTestStream(tempDir); try { - Configuration hadoopConf = spark.sessionState().newHadoopConf(); - PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(testTablePath, hadoopConf); - DeltaV2MicroBatchStream stream = - createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); - - long version = snapshotManager.loadLatestSnapshot().getVersion(); - long fromIndex = DeltaSourceOffset.BASE_INDEX(); - boolean isInitialSnapshot = true; - - List files = new ArrayList<>(); - try (CloseableIterator iter = - stream.getFileChanges(version, fromIndex, isInitialSnapshot, Optional.empty())) { - while (iter.hasNext()) { - files.add(iter.next()); - } - } - - assertThat(files).hasSizeGreaterThanOrEqualTo(2); - assertThat(files.get(files.size() - 1).getIndex()).isEqualTo(DeltaSourceOffset.END_INDEX()); - - List dataFileIndices = - files.stream() - .filter(f -> f.getAddFile() != null) - .map(IndexedFile::getIndex) - .collect(Collectors.toList()); - - assertThat(dataFileIndices).isNotEmpty(); - - // Verify contiguous 0-based: 0, 1, 2, ..., N-1 - for (int i = 0; i < dataFileIndices.size(); i++) { - assertThat(dataFileIndices.get(i)) - .as("Index at position %d should be %d (contiguous 0-based)", i, i) - .isEqualTo((long) i); - } + DeltaSourceOffset initialOffset = (DeltaSourceOffset) stream.initialOffset(); + exhaustInitialSnapshot(stream, initialOffset); + DeltaSourceSnapshot initialSnapshot = getInitialSnapshot(stream); + + assertNotNull(initialSnapshot); + assertEquals(initialOffset.reservoirVersion(), initialSnapshot.snapshot().version()); + + exhaustInitialSnapshot(stream, initialOffset); + assertSame(initialSnapshot, getInitialSnapshot(stream)); + + stream.commit(initialOffset); + assertSame(initialSnapshot, getInitialSnapshot(stream)); + + DeltaSourceOffset incrementalOffset = + DeltaSourceOffset.apply( + initialOffset.reservoirId(), + initialOffset.reservoirVersion() + 1, + DeltaSourceOffset.BASE_INDEX(), + /* isInitialSnapshot= */ false); + stream.commit(incrementalOffset); + assertNull(getInitialSnapshot(stream)); } finally { - spark.conf().unset(maxFilesKey); - spark.conf().unset(dfFlagKey); + stream.stop(); } } - /** Verifies that DataFrameSnapshotCache.close() nulls the DataFrame and is idempotent. */ - @Test - public void testDataFrameSnapshotCache_closeAndIdempotent() { - Dataset df = - spark.range(10).toDF("value").persist(org.apache.spark.storage.StorageLevel.MEMORY_ONLY()); - df.count(); - - DataFrameSnapshotCache cache = new DataFrameSnapshotCache(42L, df); - assertThat(cache.getVersion()).isEqualTo(42L); - assertThat(cache.getSortedAddFiles()).isNotNull(); - - // close() unpersists but does not null the DataFrame reference - cache.close(); - assertThat(cache.getSortedAddFiles()).isNotNull(); - - // Second close: idempotent (unpersist on already-unpersisted is a no-op) - assertDoesNotThrow(cache::close); - } - - /** - * Verifies that stop() nulls the AtomicReference and unpersists, but the DataFrame reference on - * the cache object remains non-null (close() never nullifies the field). - */ @Test - public void testDistributedInitialSnapshot_stopLeavesDataFrameReferenceIntact( - @TempDir File tempDir) throws Exception { - String testTablePath = tempDir.getAbsolutePath(); - String testTableName = "test_df_stop_" + System.nanoTime(); - createEmptyTestTable(testTablePath, testTableName); - insertVersions( - testTableName, - /* numVersions= */ 3, - /* rowsPerVersion= */ 2, - /* includeEmptyVersion= */ false); - - String dfFlagKey = DeltaSQLConf.DELTA_STREAMING_USE_DISTRIBUTED_INITIAL_SNAPSHOT().key(); - spark.conf().set(dfFlagKey, "true"); - + public void testStopReleasesInitialSnapshot(@TempDir File tempDir) throws Exception { + DeltaV2MicroBatchStream stream = createTestStream(tempDir); try { - Configuration hadoopConf = spark.sessionState().newHadoopConf(); - PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(testTablePath, hadoopConf); - DeltaV2MicroBatchStream stream = - createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); - - long version = snapshotManager.loadLatestSnapshot().getVersion(); - - // Access the internal AtomicReference via reflection - java.lang.reflect.Field cacheField = - DeltaV2MicroBatchStream.class.getDeclaredField("cachedDataFrameSnapshot"); - cacheField.setAccessible(true); - @SuppressWarnings("unchecked") - java.util.concurrent.atomic.AtomicReference cacheRef = - (java.util.concurrent.atomic.AtomicReference) - cacheField.get(stream); - - // Build the cache via a normal getFileChanges call - try (CloseableIterator iter = - stream.getFileChanges(version, DeltaSourceOffset.BASE_INDEX(), true, Optional.empty())) { - assertTrue(iter.hasNext()); - } + DeltaSourceOffset initialOffset = (DeltaSourceOffset) stream.initialOffset(); + exhaustInitialSnapshot(stream, initialOffset); + DeltaSourceSnapshot initialSnapshot = getInitialSnapshot(stream); + assertNotNull(initialSnapshot); - // Grab a direct reference to the cache before stop() - DataFrameSnapshotCache cached = cacheRef.get(); - assertNotNull(cached, "Cache should have been built by getFileChanges"); - - // stop() nulls the AtomicReference and unpersists, but the reference stays non-null stream.stop(); - assertNull(cacheRef.get(), "AtomicReference should be null after stop()"); - assertNotNull( - cached.getSortedAddFiles(), - "DataFrame reference should remain non-null after close() (only unpersisted)"); + assertNull(getInitialSnapshot(stream)); } finally { - spark.conf().unset(dfFlagKey); + stream.stop(); } } - /** DataFrame path returns only END sentinel for an empty table. */ - @Test - public void testDistributedInitialSnapshot_emptyTable(@TempDir File tempDir) throws Exception { - String testTablePath = tempDir.getAbsolutePath(); - String testTableName = "test_df_empty_table_" + System.nanoTime(); - createEmptyTestTable(testTablePath, testTableName); - - // No insertVersions — table has CREATE TABLE metadata but zero data files - - String maxFilesKey = DeltaSQLConf.DELTA_STREAMING_INITIAL_SNAPSHOT_MAX_FILES().key(); - String dfFlagKey = DeltaSQLConf.DELTA_STREAMING_USE_DISTRIBUTED_INITIAL_SNAPSHOT().key(); - spark.conf().set(maxFilesKey, "10000"); - spark.conf().set(dfFlagKey, "true"); - - try { - Configuration hadoopConf = spark.sessionState().newHadoopConf(); - PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(testTablePath, hadoopConf); - DeltaV2MicroBatchStream stream = - createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); - - long version = snapshotManager.loadLatestSnapshot().getVersion(); - long fromIndex = DeltaSourceOffset.BASE_INDEX(); - boolean isInitialSnapshot = true; - - List files = new ArrayList<>(); - try (CloseableIterator iter = - stream.getFileChanges(version, fromIndex, isInitialSnapshot, Optional.empty())) { - while (iter.hasNext()) { - files.add(iter.next()); - } + private static void exhaustInitialSnapshot( + DeltaV2MicroBatchStream stream, DeltaSourceOffset initialOffset) throws Exception { + try (CloseableIterator files = + stream.getFileChanges( + initialOffset.reservoirVersion(), + initialOffset.index(), + /* isInitialSnapshot= */ true, + Optional.empty())) { + while (files.hasNext()) { + files.next(); } - - // Only END sentinel — no data files, BEGIN filtered by boundary check. - assertThat(files).hasSize(1); - assertThat(files.get(0).getIndex()).isEqualTo(DeltaSourceOffset.END_INDEX()); - assertThat(files.get(0).getAddFile()).isNull(); - - assertThat(files.stream().noneMatch(IndexedFile::hasFileAction)).isTrue(); - } finally { - spark.conf().unset(maxFilesKey); - spark.conf().unset(dfFlagKey); } } + + private static DeltaSourceSnapshot getInitialSnapshot(DeltaV2MicroBatchStream stream) + throws Exception { + java.lang.reflect.Field field = + DeltaV2MicroBatchStream.class.getDeclaredField("initialSnapshot"); + field.setAccessible(true); + return (DeltaSourceSnapshot) field.get(stream); + } } diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/DeltaV2ScanBuilderTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/DeltaV2ScanBuilderTest.java index 5db0a57eee6..5573afd1af7 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/DeltaV2ScanBuilderTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/DeltaV2ScanBuilderTest.java @@ -20,23 +20,27 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import io.delta.kernel.Snapshot; -import io.delta.kernel.expressions.Column; -import io.delta.kernel.expressions.Literal; -import io.delta.kernel.expressions.Predicate; +import io.delta.kernel.defaults.engine.DefaultEngine; +import io.delta.kernel.engine.Engine; import io.delta.spark.internal.v2.DeltaV2TestBase; import io.delta.spark.internal.v2.snapshot.PathBasedSnapshotManager; import java.io.File; import java.lang.reflect.Field; -import java.math.BigDecimal; import java.util.Arrays; -import java.util.HashSet; import java.util.Optional; import java.util.OptionalInt; -import java.util.Set; +import org.apache.spark.sql.catalyst.expressions.Expression; import org.apache.spark.sql.connector.read.Scan; import org.apache.spark.sql.connector.read.streaming.MicroBatchStream; -import org.apache.spark.sql.sources.*; +import org.apache.spark.sql.delta.Snapshot; +import org.apache.spark.sql.sources.And; +import org.apache.spark.sql.sources.EqualTo; +import org.apache.spark.sql.sources.Filter; +import org.apache.spark.sql.sources.GreaterThan; +import org.apache.spark.sql.sources.Not; +import org.apache.spark.sql.sources.Or; +import org.apache.spark.sql.sources.StringEndsWith; +import org.apache.spark.sql.sources.StringStartsWith; import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.types.StructType; @@ -44,6 +48,13 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +/** + * Tests for {@link DeltaV2ScanBuilder}'s construction surface: schema pruning, scan/micro-batch + * construction. Filter pushdown itself now flows as Catalyst Expressions through {@code + * SupportsPushDownCatalystFilters} and is exercised end-to-end by the data-skipping differential + * suites (DataSkippingDeltaV2SnapshotSuite), which drive file selection through {@code + * DeltaV2Snapshot.filesForScan}. + */ public class DeltaV2ScanBuilderTest extends DeltaV2TestBase { @Test @@ -54,9 +65,6 @@ public void testBuild_returnsScanWithExpectedSchema(@TempDir File tempDir) { String.format( "CREATE TABLE %s (id INT, name STRING, dep_id INT) USING delta PARTITIONED BY (dep_id) LOCATION '%s'", tableName, path)); - PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(path, spark.sessionState().newHadoopConf()); - Snapshot snapshot = snapshotManager.loadLatestSnapshot(); StructType dataSchema = DataTypes.createStructType( new StructField[] { @@ -74,15 +82,7 @@ public void testBuild_returnsScanWithExpectedSchema(@TempDir File tempDir) { DataTypes.createStructField("dep_id", DataTypes.IntegerType, true) }); DeltaV2ScanBuilder builder = - new DeltaV2ScanBuilder( - tableName, - snapshot, - snapshotManager, - dataSchema, - partitionSchema, - tableSchema, - Optional.empty(), - CaseInsensitiveStringMap.empty()); + newScanBuilder(tableName, path, dataSchema, partitionSchema, tableSchema); StructType expectedSparkSchema = DataTypes.createStructType( @@ -106,9 +106,6 @@ public void testPruneColumns_filtersMixedCaseCDCColumn(@TempDir File tempDir) th String.format( "CREATE TABLE %s (id INT, name STRING, dep_id INT) USING delta PARTITIONED BY (dep_id) LOCATION '%s'", tableName, path)); - PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(path, spark.sessionState().newHadoopConf()); - Snapshot snapshot = snapshotManager.loadLatestSnapshot(); StructType dataSchema = DataTypes.createStructType( new StructField[] { @@ -126,15 +123,7 @@ public void testPruneColumns_filtersMixedCaseCDCColumn(@TempDir File tempDir) th DataTypes.createStructField("dep_id", DataTypes.IntegerType, true) }); DeltaV2ScanBuilder builder = - new DeltaV2ScanBuilder( - tableName, - snapshot, - snapshotManager, - dataSchema, - partitionSchema, - tableSchema, - Optional.empty(), - CaseInsensitiveStringMap.empty()); + newScanBuilder(tableName, path, dataSchema, partitionSchema, tableSchema); StructType requiredSchema = DataTypes.createStructType( @@ -152,12 +141,6 @@ public void testPruneColumns_filtersMixedCaseCDCColumn(@TempDir File tempDir) th } } - private StructType getRequiredDataSchema(DeltaV2ScanBuilder builder) throws Exception { - Field field = DeltaV2ScanBuilder.class.getDeclaredField("requiredDataSchema"); - field.setAccessible(true); - return (StructType) field.get(builder); - } - @Test public void testToMicroBatchStream_returnsDeltaV2MicroBatchStream(@TempDir File tempDir) { String path = tempDir.getAbsolutePath(); @@ -166,9 +149,6 @@ public void testToMicroBatchStream_returnsDeltaV2MicroBatchStream(@TempDir File String.format( "CREATE TABLE %s (id INT, name STRING, dep_id INT) USING delta PARTITIONED BY (dep_id) LOCATION '%s'", tableName, path)); - PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(path, spark.sessionState().newHadoopConf()); - Snapshot snapshot = snapshotManager.loadLatestSnapshot(); StructType dataSchema = DataTypes.createStructType( new StructField[] { @@ -186,15 +166,7 @@ public void testToMicroBatchStream_returnsDeltaV2MicroBatchStream(@TempDir File DataTypes.createStructField("dep_id", DataTypes.IntegerType, true) }); DeltaV2ScanBuilder builder = - new DeltaV2ScanBuilder( - tableName, - snapshot, - snapshotManager, - dataSchema, - partitionSchema, - tableSchema, - Optional.empty(), - CaseInsensitiveStringMap.empty()); + newScanBuilder(tableName, path, dataSchema, partitionSchema, tableSchema); Scan scan = builder.build(); String checkpointLocation = "/tmp/checkpoint"; @@ -206,727 +178,375 @@ public void testToMicroBatchStream_returnsDeltaV2MicroBatchStream(@TempDir File "MicroBatchStream should be an instance of DeltaV2MicroBatchStream"); } + // Filter-classification cases, one per case in the pre-catalyst-pushdown suite, with the same + // names. Under SupportsPushDownCatalystFilters a filter is no longer translated to a data-source + // predicate, so the old "supported"/"unsupported" axis (whether Kernel could represent the filter + // shape) no longer affects anything: only the columns a filter references decide where it lands. + // The cases whose names encode that axis are kept anyway, precisely to pin that they now behave + // identically to their "supported" siblings. + @Test public void testPushFilters_singleSupportedDataFilter(@TempDir File tempDir) throws Exception { - DeltaV2ScanBuilder builder = createTestScanBuilder(tempDir); - checkSupportsPushDownFilters( - builder, - // input filters - new Filter[] {new EqualTo("id", 100)}, - // expected post-scan filters + newFilterScanBuilder(tempDir), + // partition filters + new Filter[0], + // data filters (also the post-scan residuals) new Filter[] {new EqualTo("id", 100)}, - // expected pushed filters - new Filter[] {new EqualTo("id", 100)}, - // expected pushed kernel predicates - new Predicate[] {new Predicate("=", new Column("id"), Literal.ofInt(100))}, - // expected data filters - new Filter[] {new EqualTo("id", 100)}, - // expected kernelScanBuilder.predicate - Optional.of(new Predicate("=", new Column("id"), Literal.ofInt(100)))); + // input + new EqualTo("id", 100)); } @Test public void testPushFilters_singleUnsupportedDataFilter(@TempDir File tempDir) throws Exception { - DeltaV2ScanBuilder builder = createTestScanBuilder(tempDir); - checkSupportsPushDownFilters( - builder, - new Filter[] {new StringEndsWith("name", "test")}, // input filters - new Filter[] { - new StringEndsWith("name", "test") - }, // expected post-scan filters (unsupported, stays for row-level eval) - new Filter[] {}, // expected pushed filters (nothing pushed) - new Predicate[] {}, // expected pushed kernel predicates - new Filter[] {new StringEndsWith("name", "test")}, // expected data filters - Optional.empty() // expected kernelScanBuilder.predicate - ); + newFilterScanBuilder(tempDir), + new Filter[0], + new Filter[] {new StringEndsWith("name", "test")}, + new StringEndsWith("name", "test")); } @Test public void testPushFilters_singleSupportedDataFilter_StringStartsWith(@TempDir File tempDir) throws Exception { - DeltaV2ScanBuilder builder = createTestScanBuilder(tempDir); - checkSupportsPushDownFilters( - builder, - new Filter[] {new StringStartsWith("name", "test")}, // input filters - new Filter[] { - new StringStartsWith("name", "test") - }, // expected post-scan filters (data filter still needs row-level eval) - new Filter[] {new StringStartsWith("name", "test")}, // expected pushed filters - new Predicate[] { - new Predicate("STARTS_WITH", new Column("name"), Literal.ofString("test")) - }, // expected pushed kernel predicates - new Filter[] {new StringStartsWith("name", "test")}, // expected data filters - Optional.of( - new Predicate( - "STARTS_WITH", - new Column("name"), - Literal.ofString("test"))) // expected kernelScanBuilder.predicate - ); + newFilterScanBuilder(tempDir), + new Filter[0], + new Filter[] {new StringStartsWith("name", "test")}, + new StringStartsWith("name", "test")); } @Test public void testPushFilters_multiSupportedDataFilters(@TempDir File tempDir) throws Exception { - DeltaV2ScanBuilder builder = createTestScanBuilder(tempDir); - checkSupportsPushDownFilters( - builder, - // input filters - new Filter[] {new EqualTo("id", 100), new GreaterThan("id", 50)}, - // expected post-scan filters - new Filter[] {new EqualTo("id", 100), new GreaterThan("id", 50)}, - // expected pushed filters - new Filter[] {new EqualTo("id", 100), new GreaterThan("id", 50)}, - // expected pushed kernel predicates - new Predicate[] { - new Predicate("=", new Column("id"), Literal.ofInt(100)), - new Predicate(">", new Column("id"), Literal.ofInt(50)) - }, - // expected data filters - new Filter[] {new EqualTo("id", 100), new GreaterThan("id", 50)}, - // expected kernelScanBuilder.predicate - Optional.of( - new Predicate( - "AND", - Arrays.asList( - new Predicate("=", new Column("id"), Literal.ofInt(100)), - new Predicate(">", new Column("id"), Literal.ofInt(50)))))); + newFilterScanBuilder(tempDir), + new Filter[0], + new Filter[] {new EqualTo("id", 100), new GreaterThan("id", 10)}, + new EqualTo("id", 100), + new GreaterThan("id", 10)); } @Test public void testPushFilters_mixedSupportedAndUnsupportedDataFilters(@TempDir File tempDir) throws Exception { - DeltaV2ScanBuilder builder = createTestScanBuilder(tempDir); - checkSupportsPushDownFilters( - builder, - // input filters - new Filter[] { - new EqualTo("id", 100), // supported - new StringEndsWith("name", "test") // unsupported - }, - // expected post-scan filters - new Filter[] {new EqualTo("id", 100), new StringEndsWith("name", "test")}, - // expected pushed filters (only the supported EqualTo is pushed) - new Filter[] {new EqualTo("id", 100)}, - // expected pushed kernel predicates - new Predicate[] {new Predicate("=", new Column("id"), Literal.ofInt(100))}, - // expected data filters + newFilterScanBuilder(tempDir), + new Filter[0], new Filter[] {new EqualTo("id", 100), new StringEndsWith("name", "test")}, - // expected kernelScanBuilder.predicate (only EqualTo was pushed to kernel) - Optional.of(new Predicate("=", new Column("id"), Literal.ofInt(100)))); + new EqualTo("id", 100), + new StringEndsWith("name", "test")); } @Test public void testPushFilters_singleSupportedPartitionFilter(@TempDir File tempDir) throws Exception { - DeltaV2ScanBuilder builder = createTestScanBuilder(tempDir); - checkSupportsPushDownFilters( - builder, - // input filters + newFilterScanBuilder(tempDir), new Filter[] {new EqualTo("dep_id", 1)}, - // expected post-scan filters - new Filter[] {}, - // expected pushed filters - new Filter[] {new EqualTo("dep_id", 1)}, - // expected pushed kernel predicates - new Predicate[] {new Predicate("=", new Column("dep_id"), Literal.ofInt(1))}, - // expected data filters - new Filter[] {}, - // expected kernelScanBuilder.predicate - Optional.of(new Predicate("=", new Column("dep_id"), Literal.ofInt(1)))); + new Filter[0], + new EqualTo("dep_id", 1)); } @Test - public void testPushFilters_singleUnsupportedPartitionFilter(@TempDir File tempDir) + public void testPruneColumnsRetainsFullyPushedFilterColumn(@TempDir File tempDir) throws Exception { - DeltaV2ScanBuilder builder = createTestScanBuilder(tempDir); + DeltaV2ScanBuilder builder = newFilterScanBuilder(tempDir); + pushFilters(builder, new EqualTo("dep_id", 1)); + builder.pruneColumns(new StructType().add("id", DataTypes.IntegerType, true /* nullable */)); + Scan scan = builder.build(); + assertEquals( + Arrays.asList("id", "dep_id", "dep_name"), Arrays.asList(scan.readSchema().fieldNames())); + } + + @Test + public void testPushFilters_singleUnsupportedPartitionFilter(@TempDir File tempDir) + throws Exception { + // A partition filter Kernel could not represent is still exact, so it is a partition filter. checkSupportsPushDownFilters( - builder, - new Filter[] {new StringStartsWith("dep_id", "1")}, // input filters - new Filter[] {}, // expected post-scan filters (partition filter, fully pushed) - new Filter[] {new StringStartsWith("dep_id", "1")}, // expected pushed filters - new Predicate[] { - new Predicate("STARTS_WITH", new Column("dep_id"), Literal.ofString("1")) - }, // expected pushed kernel predicates - new Filter[] {}, // expected data filters - Optional.of( - new Predicate( - "STARTS_WITH", - new Column("dep_id"), - Literal.ofString("1"))) // expected kernelScanBuilder.predicate - ); + newFilterScanBuilder(tempDir), + new Filter[] {new StringStartsWith("dep_name", "d")}, + new Filter[0], + new StringStartsWith("dep_name", "d")); } @Test public void testPushFilters_multiSupportedPartitionFilters(@TempDir File tempDir) throws Exception { - DeltaV2ScanBuilder builder = createTestScanBuilder(tempDir); - checkSupportsPushDownFilters( - builder, - // input filters - new Filter[] {new EqualTo("dep_id", 2), new GreaterThan("dep_id", 1)}, - // expected post-scan filters - new Filter[] {}, - // expected pushed filters - new Filter[] {new EqualTo("dep_id", 2), new GreaterThan("dep_id", 1)}, - // expected pushed kernel predicates - new Predicate[] { - new Predicate("=", new Column("dep_id"), Literal.ofInt(2)), - new Predicate(">", new Column("dep_id"), Literal.ofInt(1)) - }, - // expected data filters - new Filter[] {}, - // expected kernelScanBuilder.predicate - Optional.of( - new Predicate( - "AND", - Arrays.asList( - new Predicate("=", new Column("dep_id"), Literal.ofInt(2)), - new Predicate(">", new Column("dep_id"), Literal.ofInt(1)))))); + newFilterScanBuilder(tempDir), + new Filter[] {new EqualTo("dep_id", 1), new GreaterThan("dep_id", 0)}, + new Filter[0], + new EqualTo("dep_id", 1), + new GreaterThan("dep_id", 0)); } @Test public void testPushFilters_mixedSupportedAndUnsupportedPartitionFilters(@TempDir File tempDir) throws Exception { - DeltaV2ScanBuilder builder = createTestScanBuilder(tempDir); - checkSupportsPushDownFilters( - builder, - // input filters - new Filter[] { - new EqualTo("dep_id", 1), // supported - new StringStartsWith("dep_id", "1") // unsupported - }, - // expected post-scan filters - new Filter[] {}, - // expected pushed filters - new Filter[] {new EqualTo("dep_id", 1), new StringStartsWith("dep_id", "1")}, - // expected pushed kernel predicates - new Predicate[] { - new Predicate("=", new Column("dep_id"), Literal.ofInt(1)), - new Predicate("STARTS_WITH", new Column("dep_id"), Literal.ofString("1")) - }, - // expected data filters - new Filter[] {}, - // expected kernelScanBuilder.predicate - Optional.of( - new Predicate( - "AND", - new Predicate("=", new Column("dep_id"), Literal.ofInt(1)), - new Predicate("STARTS_WITH", new Column("dep_id"), Literal.ofString("1"))))); + newFilterScanBuilder(tempDir), + new Filter[] {new EqualTo("dep_id", 1), new StringStartsWith("dep_name", "d")}, + new Filter[0], + new EqualTo("dep_id", 1), + new StringStartsWith("dep_name", "d")); } @Test public void testPushFilters_mixedFilters(@TempDir File tempDir) throws Exception { - DeltaV2ScanBuilder builder = createTestScanBuilder(tempDir); - checkSupportsPushDownFilters( - builder, - // input filters - new Filter[] { - new EqualTo("id", 100), // data filter, supported - new StringStartsWith("name", "foo"), // data filter, unsupported - new GreaterThan("dep_id", 1), // partition filter, supported - new StringEndsWith("dep_id", "1") // partition filter, unsupported - }, - // expected post-scan filters - new Filter[] { - new EqualTo("id", 100), // data filter, supported - new StringStartsWith("name", "foo"), // data filter, unsupported - new StringEndsWith("dep_id", "1") // partition filter, unsupported - }, - // expected pushed filters - new Filter[] { - new EqualTo("id", 100), // data filter, supported - new StringStartsWith("name", "foo"), // data filter, supported - new GreaterThan("dep_id", 1) // partition filter, supported - }, - // expected pushed kernel predicates - new Predicate[] { - new Predicate("=", new Column("id"), Literal.ofInt(100)), - new Predicate("STARTS_WITH", new Column("name"), Literal.ofString("foo")), - new Predicate(">", new Column("dep_id"), Literal.ofInt(1)) - }, - // expected data filters - new Filter[] { - new EqualTo("id", 100), // data filter, supported - new StringStartsWith("name", "foo") // data filter, supported - }, - // expected kernelScanBuilder.predicate - Optional.of( - new Predicate( - "AND", - new Predicate( - "AND", - new Predicate("=", new Column("id"), Literal.ofInt(100)), - new Predicate("STARTS_WITH", new Column("name"), Literal.ofString("foo"))), - new Predicate(">", new Column("dep_id"), Literal.ofInt(1))))); + newFilterScanBuilder(tempDir), + new Filter[] {new GreaterThan("dep_id", 0), new StringEndsWith("dep_name", "x")}, + new Filter[] {new EqualTo("id", 100), new StringStartsWith("name", "test")}, + new EqualTo("id", 100), + new StringStartsWith("name", "test"), + new GreaterThan("dep_id", 0), + new StringEndsWith("dep_name", "x")); } @Test public void testPushFilters_ORFilters(@TempDir File tempDir) throws Exception { - DeltaV2ScanBuilder builder = createTestScanBuilder(tempDir); - + // An OR cannot be split, and this one references only data columns. checkSupportsPushDownFilters( - builder, - // input filters - new Filter[] {new Or(new EqualTo("id", 100), new GreaterThan("id", 50))}, - // expected post-scan filters - new Filter[] {new Or(new EqualTo("id", 100), new GreaterThan("id", 50))}, - // expected pushed filters - new Filter[] {new Or(new EqualTo("id", 100), new GreaterThan("id", 50))}, - // expected pushed kernel predicates - new Predicate[] { - new Predicate( - "OR", - Arrays.asList( - new Predicate("=", new Column("id"), Literal.ofInt(100)), - new Predicate(">", new Column("id"), Literal.ofInt(50)))) - }, - // expected data filters - new Filter[] {new Or(new EqualTo("id", 100), new GreaterThan("id", 50))}, - // expected kernelScanBuilder.predicate - Optional.of( - new Predicate( - "OR", - Arrays.asList( - new Predicate("=", new Column("id"), Literal.ofInt(100)), - new Predicate(">", new Column("id"), Literal.ofInt(50)))))); + newFilterScanBuilder(tempDir), + new Filter[0], + new Filter[] {new Or(new GreaterThan("id", 10), new EqualTo("id", 100))}, + new Or(new GreaterThan("id", 10), new EqualTo("id", 100))); } @Test public void testPushFilters_ORSupportedAndUnsupportedDataFilters(@TempDir File tempDir) throws Exception { - DeltaV2ScanBuilder builder = createTestScanBuilder(tempDir); - - // OR(supported, unsupported) cannot be partially pushed: if one branch is unsupported, - // the whole OR must remain for post-scan evaluation and nothing is pushed to the kernel. checkSupportsPushDownFilters( - builder, - // input filters - new Filter[] {new Or(new EqualTo("id", 100), new StringEndsWith("name", "foo"))}, - // expected post-scan filters (whole OR stays, since one branch is unsupported) - new Filter[] {new Or(new EqualTo("id", 100), new StringEndsWith("name", "foo"))}, - // expected pushed filters (nothing pushed) - new Filter[] {}, - // expected pushed kernel predicates - new Predicate[] {}, - // expected data filters - new Filter[] {new Or(new EqualTo("id", 100), new StringEndsWith("name", "foo"))}, - // expected kernelScanBuilder.predicate - Optional.empty()); + newFilterScanBuilder(tempDir), + new Filter[0], + new Filter[] {new Or(new StringEndsWith("name", "test"), new EqualTo("id", 100))}, + new Or(new StringEndsWith("name", "test"), new EqualTo("id", 100))); } @Test public void testPushFilters_ORSupportedDataAndPartitionFilters(@TempDir File tempDir) throws Exception { - DeltaV2ScanBuilder builder = createTestScanBuilder(tempDir); - + // Mixed OR: not exact, so it stays a residual and yields no partition filter. checkSupportsPushDownFilters( - builder, - // input filters - new Filter[] { - new Or(new EqualTo("id", 100), new GreaterThan("dep_id", 1)), - }, - // expected post-scan filters - new Filter[] {new Or(new EqualTo("id", 100), new GreaterThan("dep_id", 1))}, - // expected pushed filters - new Filter[] {new Or(new EqualTo("id", 100), new GreaterThan("dep_id", 1))}, - // expected pushed kernel predicates - new Predicate[] { - new Predicate( - "OR", - Arrays.asList( - new Predicate("=", new Column("id"), Literal.ofInt(100)), - new Predicate(">", new Column("dep_id"), Literal.ofInt(1)))) - }, - // expected data filters - new Filter[] {new Or(new EqualTo("id", 100), new GreaterThan("dep_id", 1))}, - // expected kernelScanBuilder.predicate - Optional.of( - new Predicate( - "OR", - Arrays.asList( - new Predicate("=", new Column("id"), Literal.ofInt(100)), - new Predicate(">", new Column("dep_id"), Literal.ofInt(1)))))); + newFilterScanBuilder(tempDir), + new Filter[0], + new Filter[] {new Or(new GreaterThan("dep_id", 0), new EqualTo("id", 100))}, + new Or(new GreaterThan("dep_id", 0), new EqualTo("id", 100))); } - /* - * (partitionFilterA AND partitionFilterB) OR partitionFilterC - * where A = EqualTo("dep_id", 1), B = StringStartsWith("dep_id", "1"), C = GreaterThan("dep_id", 2) - * All three are fully supported partition filters, so the whole expression is pushed down. - * - * Expected post-scan filters: none (all partition filters, fully pushed) - * Expected pushed filters: (A AND B) OR C - * Expected pushed kernel predicates: (predicateA AND predicateB) OR predicateC - * Expected data filters: none - * Expected kernelScanBuilder.predicate: (predicateA AND predicateB) OR predicateC - */ @Test public void testPushFilters_mixedORandAND(@TempDir File tempDir) throws Exception { - DeltaV2ScanBuilder builder = createTestScanBuilder(tempDir); - + Filter input = + new Or( + new And(new StringStartsWith("dep_name", "d"), new EqualTo("id", 100)), + new GreaterThan("dep_id", 0)); + // The whole OR is a residual, but it still yields a partition filter: extraction distributes + // over the disjunction, dropping the data-column conjunct from the left branch to leave a + // weaker partition-only predicate that is still safe for pruning. checkSupportsPushDownFilters( - builder, - // input filters - new Filter[] { - new Or( - new And(new EqualTo("dep_id", 1), new StringStartsWith("dep_id", "1")), - new GreaterThan("dep_id", 2)) - }, - // expected post-scan filters - new Filter[] {}, - // expected pushed filters - new Filter[] { - new Or( - new And(new EqualTo("dep_id", 1), new StringStartsWith("dep_id", "1")), - new GreaterThan("dep_id", 2)) - }, - // expected pushed kernel predicates - new Predicate[] { - new Predicate( - "OR", - Arrays.asList( - new Predicate( - "AND", - Arrays.asList( - new Predicate("=", new Column("dep_id"), Literal.ofInt(1)), - new Predicate( - "STARTS_WITH", new Column("dep_id"), Literal.ofString("1")))), - new Predicate(">", new Column("dep_id"), Literal.ofInt(2)))) - }, - // expected data filters - new Filter[] {}, - // expected kernelScanBuilder.predicate - Optional.of( - new Predicate( - "OR", - Arrays.asList( - new Predicate( - "AND", - Arrays.asList( - new Predicate("=", new Column("dep_id"), Literal.ofInt(1)), - new Predicate( - "STARTS_WITH", new Column("dep_id"), Literal.ofString("1")))), - new Predicate(">", new Column("dep_id"), Literal.ofInt(2)))))); + newFilterScanBuilder(tempDir), + new Filter[] {new Or(new StringStartsWith("dep_name", "d"), new GreaterThan("dep_id", 0))}, + new Filter[] {input}, + input); } @Test public void testPushFilters_NOTFilters(@TempDir File tempDir) throws Exception { - DeltaV2ScanBuilder builder = createTestScanBuilder(tempDir); - checkSupportsPushDownFilters( - builder, - // input filters - new Filter[] {new Not(new EqualTo("id", 100))}, - // expected post-scan filters - new Filter[] {new Not(new EqualTo("id", 100))}, - // expected pushed filters - new Filter[] {new Not(new EqualTo("id", 100))}, - // expected pushed kernel predicates - new Predicate[] { - new Predicate("NOT", new Predicate("=", new Column("id"), Literal.ofInt(100))) - }, - // expected data filters - new Filter[] {new Not(new EqualTo("id", 100))}, - // expected kernelScanBuilder.predicate - Optional.of( - new Predicate("NOT", new Predicate("=", new Column("id"), Literal.ofInt(100))))); + newFilterScanBuilder(tempDir), + new Filter[] {new Not(new EqualTo("dep_id", 1))}, + new Filter[0], + new Not(new EqualTo("dep_id", 1))); } @Test public void testPushFilters_NOTSupportedDataANDSupportedPartitionFilters(@TempDir File tempDir) throws Exception { - DeltaV2ScanBuilder builder = createTestScanBuilder(tempDir); - + Filter input = new Not(new And(new EqualTo("id", 100), new GreaterThan("dep_id", 0))); checkSupportsPushDownFilters( - builder, - // input filters - new Filter[] { - new Not(new And(new EqualTo("id", 100), new GreaterThan("dep_id", 1))), - }, - // expected post-scan filters - new Filter[] { - new Not(new And(new EqualTo("id", 100), new GreaterThan("dep_id", 1))), - }, - // expected pushed filters - new Filter[] { - new Not(new And(new EqualTo("id", 100), new GreaterThan("dep_id", 1))), - }, - // expected pushed kernel predicates - new Predicate[] { - new Predicate( - "NOT", - new Predicate( - "AND", - Arrays.asList( - new Predicate("=", new Column("id"), Literal.ofInt(100)), - new Predicate(">", new Column("dep_id"), Literal.ofInt(1))))) - }, - // expected data filters - new Filter[] { - new Not(new And(new EqualTo("id", 100), new GreaterThan("dep_id", 1))), - }, - // expected kernelScanBuilder.predicate - Optional.of( - new Predicate( - "NOT", - new Predicate( - "AND", - Arrays.asList( - new Predicate("=", new Column("id"), Literal.ofInt(100)), - new Predicate(">", new Column("dep_id"), Literal.ofInt(1))))))); + newFilterScanBuilder(tempDir), new Filter[0], new Filter[] {input}, input); } @Test public void testPushFilters_NOTSupportedDataANDUnsupportedDataFilters(@TempDir File tempDir) throws Exception { - DeltaV2ScanBuilder builder = createTestScanBuilder(tempDir); - + Filter input = new Not(new And(new EqualTo("id", 100), new StringEndsWith("name", "test"))); checkSupportsPushDownFilters( - builder, - // input filters - new Filter[] {new Not(new And(new EqualTo("id", 100), new StringEndsWith("name", "bar")))}, - // expected post-scan filters - new Filter[] {new Not(new And(new EqualTo("id", 100), new StringEndsWith("name", "bar")))}, - // expected pushed filters - new Filter[] {}, - // expected pushed kernel predicates - new Predicate[] {}, - // expected data filters - new Filter[] {new Not(new And(new EqualTo("id", 100), new StringEndsWith("name", "bar")))}, - // expected kernelScanBuilder.predicate - Optional.empty()); + newFilterScanBuilder(tempDir), new Filter[0], new Filter[] {input}, input); } @Test public void testPushFilters_NOTSupportedDataORSupportedPartitionFilters(@TempDir File tempDir) throws Exception { - DeltaV2ScanBuilder builder = createTestScanBuilder(tempDir); - + Filter input = new Not(new Or(new EqualTo("id", 100), new GreaterThan("dep_id", 0))); checkSupportsPushDownFilters( - builder, - // input filters - new Filter[] { - new Not(new Or(new EqualTo("id", 100), new GreaterThan("dep_id", 1))), - }, - // expected post-scan filters - new Filter[] {new Not(new Or(new EqualTo("id", 100), new GreaterThan("dep_id", 1)))}, - // expected pushed filters - new Filter[] {new Not(new Or(new EqualTo("id", 100), new GreaterThan("dep_id", 1)))}, - // expected pushed kernel predicates - new Predicate[] { - new Predicate( - "NOT", - new Predicate( - "OR", - Arrays.asList( - new Predicate("=", new Column("id"), Literal.ofInt(100)), - new Predicate(">", new Column("dep_id"), Literal.ofInt(1))))) - }, - // expected data filters - new Filter[] {new Not(new Or(new EqualTo("id", 100), new GreaterThan("dep_id", 1)))}, - // expected kernelScanBuilder.predicate - Optional.of( - new Predicate( - "NOT", - new Predicate( - "OR", - Arrays.asList( - new Predicate("=", new Column("id"), Literal.ofInt(100)), - new Predicate(">", new Column("dep_id"), Literal.ofInt(1))))))); + newFilterScanBuilder(tempDir), new Filter[0], new Filter[] {input}, input); } @Test public void testPushFilters_NOTSupportedDataORUnsupportedDataFilters(@TempDir File tempDir) throws Exception { - DeltaV2ScanBuilder builder = createTestScanBuilder(tempDir); + Filter input = new Not(new Or(new EqualTo("id", 100), new StringEndsWith("name", "test"))); + checkSupportsPushDownFilters( + newFilterScanBuilder(tempDir), new Filter[0], new Filter[] {input}, input); + } + + // Cases the pre-catalyst-pushdown suite did not cover: a partition-only AND / OR is exact, so it + // is classified as a partition filter rather than left as a residual. - // NOT(OR(supported, unsupported)): the OR branch contains an unsupported filter - // (StringEndsWith), - // so the whole NOT(OR(...)) cannot be pushed to the kernel. + @Test + public void testPushFilters_partitionOnlyANDFilter(@TempDir File tempDir) throws Exception { + Filter input = new And(new EqualTo("dep_id", 1), new GreaterThan("dep_id", 0)); checkSupportsPushDownFilters( - builder, - // input filters - new Filter[] { - new Not(new Or(new EqualTo("id", 100), new StringEndsWith("name", "foo"))), - }, - // expected post-scan filters (whole NOT(OR) stays, unsupported branch blocks pushdown) - new Filter[] {new Not(new Or(new EqualTo("id", 100), new StringEndsWith("name", "foo")))}, - // expected pushed filters (nothing pushed) - new Filter[] {}, - // expected pushed kernel predicates - new Predicate[] {}, - // expected data filters - new Filter[] {new Not(new Or(new EqualTo("id", 100), new StringEndsWith("name", "foo")))}, - // expected kernelScanBuilder.predicate - Optional.empty()); + newFilterScanBuilder(tempDir), new Filter[] {input}, new Filter[0], input); } + @Test + public void testPushFilters_partitionOnlyORFilter(@TempDir File tempDir) throws Exception { + Filter input = new Or(new EqualTo("dep_id", 1), new GreaterThan("dep_id", 0)); + checkSupportsPushDownFilters( + newFilterScanBuilder(tempDir), new Filter[] {input}, new Filter[0], input); + } + + @Test + public void testPushFilters_mixedANDExtractsPartitionConjunct(@TempDir File tempDir) + throws Exception { + // A mixed AND stays a residual in full, but DataSourceUtils also mines its partition-only + // conjunct out for exact pruning -- the conjunct is added to the partition list without being + // removed from the data list. + Filter input = new And(new EqualTo("dep_id", 1), new GreaterThan("id", 10)); + checkSupportsPushDownFilters( + newFilterScanBuilder(tempDir), + new Filter[] {new EqualTo("dep_id", 1)}, + new Filter[] {input}, + input); + } + + /** + * Pushes {@code inputFilters} and asserts the resulting classification. Same table-driven role as + * the helper of this name before the catalyst-pushdown switch; the two assertion points that + * contract removed (the Kernel predicates and {@code kernelScanBuilder.predicate}) are gone, and + * {@code partitionCatalystFilters} is now checked as well. + */ private void checkSupportsPushDownFilters( DeltaV2ScanBuilder builder, - Filter[] inputFilters, - Filter[] expectedPostScanFilters, - Filter[] expectedPushedFilters, - Predicate[] expectedPushedKernelPredicates, + Filter[] expectedPartitionFilters, Filter[] expectedDataFilters, - Optional expectedKernelScanBuilderPredicate) + Filter... inputFilters) throws Exception { - Filter[] postScanFilters = builder.pushFilters(inputFilters); + scala.collection.immutable.Seq filters = + scala.jdk.javaapi.CollectionConverters.asScala( + Arrays.asList(toCatalystFilters(builder, inputFilters))) + .toList(); - assertEquals( - new HashSet<>(Arrays.asList(expectedPostScanFilters)), - new HashSet<>(Arrays.asList(postScanFilters))); + Object postScanFilters = builder.pushFilters(filters); + // Compare rendered expressions with exprIds stripped: each toCatalystFilter call allocates + // fresh exprIds, so the expectations cannot be compared to the builder's copies by equality. + // Data filters are exactly the post-scan residuals -- min/max skipping is not row-exact. assertEquals( - new HashSet<>(Arrays.asList(expectedPushedFilters)), - new HashSet<>(Arrays.asList(builder.pushedFilters()))); - - Predicate[] pushedPredicates = getPushedKernelPredicates(builder); + describe(toCatalystFilters(builder, expectedDataFilters)), + describe( + scala.jdk.javaapi.CollectionConverters.asJava( + (scala.collection.immutable.Seq) postScanFilters) + .toArray(new Expression[0]))); assertEquals( - new HashSet<>(Arrays.asList(expectedPushedKernelPredicates)), - new HashSet<>(Arrays.asList(pushedPredicates))); - - Filter[] dataFilters = getDataFilters(builder); + describeUnordered(toCatalystFilters(builder, expectedPartitionFilters)), + describeUnordered(getCatalystFilters(builder, "partitionCatalystFilters"))); assertEquals( - new HashSet<>(Arrays.asList(expectedDataFilters)), - new HashSet<>(Arrays.asList(dataFilters))); - - Optional predicateOpt = getKernelScanBuilderPredicate(builder); - assertEquals(expectedKernelScanBuilderPredicate, predicateOpt); + describeUnordered(toCatalystFilters(builder, expectedDataFilters)), + describeUnordered(getCatalystFilters(builder, "dataCatalystFilters"))); + // Nothing is reported through the data-source predicate API under the catalyst contract. + assertEquals(0, builder.pushedFilters().length); } - private DeltaV2ScanBuilder createTestScanBuilder(File tempDir) { - StructType dataSchema = - DataTypes.createStructType( - new StructField[] { - DataTypes.createStructField("id", DataTypes.IntegerType, true), - DataTypes.createStructField("name", DataTypes.StringType, true) - }); - StructType partitionSchema = - DataTypes.createStructType( - new StructField[] {DataTypes.createStructField("dep_id", DataTypes.IntegerType, true)}); - StructType tableSchema = - DataTypes.createStructType( - new StructField[] { - DataTypes.createStructField("id", DataTypes.IntegerType, true), - DataTypes.createStructField("name", DataTypes.StringType, true), - DataTypes.createStructField("dep_id", DataTypes.IntegerType, true) - }); - return createScanBuilder(tempDir, dataSchema, partitionSchema, tableSchema); + /** Renders expressions in order, with {@code #exprId} suffixes stripped. */ + private static java.util.List describe(Expression[] expressions) { + java.util.List rendered = new java.util.ArrayList<>(); + for (Expression expression : expressions) { + rendered.add(expression.toString().replaceAll("#\\d+", "")); + } + return rendered; } - /** - * Integration test: decimal widening end-to-end through pushFilters() → classifyFilter() → - * convertComparisonLiteral(). Verifies that a decimal literal Decimal(5,2) is widened to match - * column type Decimal(7,2) when pushed through the full filter pushdown path. - */ - @Test - public void testPushFilters_decimalWideningEndToEnd(@TempDir File tempDir) throws Exception { - DeltaV2ScanBuilder builder = createDecimalTestScanBuilder(tempDir); - - // price = 100.00 where literal is Decimal(5,2) and column is Decimal(7,2) - Filter[] sparkFilter = new Filter[] {new EqualTo("price", new BigDecimal("100.00"))}; - Predicate kernelPredicate = - new Predicate("=", new Column("price"), Literal.ofDecimal(new BigDecimal("100.00"), 7, 2)); - - checkSupportsPushDownFilters( - builder, - sparkFilter, // input filters - sparkFilter, // expected post-scan filters (data filter, stays for row-level eval) - sparkFilter, // expected pushed filters - new Predicate[] {kernelPredicate}, // expected pushed kernel predicates (widened) - sparkFilter, // expected data filters - Optional.of(kernelPredicate)); // expected kernelScanBuilder.predicate + /** Same as {@link #describe}, sorted so ordering differences do not matter. */ + private static java.util.List describeUnordered(Expression[] expressions) { + java.util.List rendered = describe(expressions); + java.util.Collections.sort(rendered); + return rendered; } - /** - * Integration test: decimal literal with scale exceeding column scale is rejected during - * pushFilters(). The filter AND(price > 99.999, price < 200.00) should partially push only the - * right side because 99.999 has scale=3 exceeding column's Decimal(7,2) scale=2. - */ - @Test - public void testPushFilters_decimalRejectionPartialPushDown(@TempDir File tempDir) + private static Expression[] toCatalystFilters(DeltaV2ScanBuilder builder, Filter[] filters) throws Exception { - DeltaV2ScanBuilder builder = createDecimalTestScanBuilder(tempDir); - - // AND(price > 99.999, price < 200.00): left side has scale=3 exceeding column's scale=2 - Filter[] sparkFilter = - new Filter[] { - new And( - new GreaterThan("price", new BigDecimal("99.999")), - new LessThan("price", new BigDecimal("200.00"))) - }; - // Only the right side (price < 200.00) is pushed as a kernel predicate - Predicate kernelPredicate = - new Predicate("<", new Column("price"), Literal.ofDecimal(new BigDecimal("200.00"), 7, 2)); + Expression[] expressions = new Expression[filters.length]; + for (int i = 0; i < filters.length; i++) { + expressions[i] = toCatalystFilter(builder, filters[i]); + } + return expressions; + } - checkSupportsPushDownFilters( - builder, - sparkFilter, // input filters - sparkFilter, // expected post-scan filters (partial conversion, stays for row-level eval) - new Filter[] {}, // expected pushed filters (partial: Spark filter not added) - new Predicate[] {kernelPredicate}, // expected pushed kernel predicates - sparkFilter, // expected data filters - Optional.of(kernelPredicate)); // expected kernelScanBuilder.predicate + private static Expression[] getCatalystFilters(DeltaV2ScanBuilder builder, String fieldName) + throws Exception { + Field field = DeltaV2ScanBuilder.class.getDeclaredField(fieldName); + field.setAccessible(true); + return (Expression[]) field.get(builder); } - private DeltaV2ScanBuilder createDecimalTestScanBuilder(File tempDir) { + private DeltaV2ScanBuilder newFilterScanBuilder(File tempDir) { + String path = tempDir.getAbsolutePath(); + String tableName = "filter_builder_" + System.nanoTime(); + // dep_name is a STRING partition column so string predicates can be applied to a partition + // column as well as to a data column. + spark.sql( + String.format( + "CREATE TABLE %s (id INT, name STRING, dep_id INT, dep_name STRING) USING delta " + + "PARTITIONED BY (dep_id, dep_name) LOCATION '%s'", + tableName, path)); StructType dataSchema = DataTypes.createStructType( new StructField[] { - DataTypes.createStructField("price", DataTypes.createDecimalType(7, 2), true), - DataTypes.createStructField("quantity", DataTypes.IntegerType, true) + DataTypes.createStructField("id", DataTypes.IntegerType, true), + DataTypes.createStructField("name", DataTypes.StringType, true) }); StructType partitionSchema = DataTypes.createStructType( - new StructField[] {DataTypes.createStructField("dep_id", DataTypes.IntegerType, true)}); + new StructField[] { + DataTypes.createStructField("dep_id", DataTypes.IntegerType, true), + DataTypes.createStructField("dep_name", DataTypes.StringType, true) + }); StructType tableSchema = DataTypes.createStructType( new StructField[] { - DataTypes.createStructField("price", DataTypes.createDecimalType(7, 2), true), - DataTypes.createStructField("quantity", DataTypes.IntegerType, true), - DataTypes.createStructField("dep_id", DataTypes.IntegerType, true) + DataTypes.createStructField("id", DataTypes.IntegerType, true), + DataTypes.createStructField("name", DataTypes.StringType, true), + DataTypes.createStructField("dep_id", DataTypes.IntegerType, true), + DataTypes.createStructField("dep_name", DataTypes.StringType, true) }); - return createScanBuilder(tempDir, dataSchema, partitionSchema, tableSchema); + return newScanBuilder(tableName, path, dataSchema, partitionSchema, tableSchema); } - /** - * Shared helper for creating a DeltaV2ScanBuilder with the given schemas. Both - * createTestScanBuilder and createDecimalTestScanBuilder delegate to this method to avoid - * duplicating snapshot loading and builder instantiation logic. - */ - private DeltaV2ScanBuilder createScanBuilder( - File tempDir, StructType dataSchema, StructType partitionSchema, StructType tableSchema) { - String path = tempDir.getAbsolutePath(); - String tableName = String.format("test_%d", System.currentTimeMillis()); - // Build CREATE TABLE SQL from the tableSchema and partitionSchema - StringBuilder columns = new StringBuilder(); - Set partitionCols = new HashSet<>(); - for (StructField f : partitionSchema.fields()) { - partitionCols.add(f.name()); - } - for (StructField f : tableSchema.fields()) { - if (columns.length() > 0) columns.append(", "); - columns.append(f.name()).append(" ").append(f.dataType().sql()); - } - String partitionColNames = String.join(", ", partitionCols); - spark.sql( - String.format( - "CREATE OR REPLACE TABLE %s (%s) USING delta PARTITIONED BY (%s) LOCATION '%s'", - tableName, columns, partitionColNames, path)); + private StructType getRequiredDataSchema(DeltaV2ScanBuilder builder) throws Exception { + Field field = DeltaV2ScanBuilder.class.getDeclaredField("requiredDataSchema"); + field.setAccessible(true); + return (StructType) field.get(builder); + } + + private DeltaV2ScanBuilder newScanBuilder( + String tableName, + String path, + StructType dataSchema, + StructType partitionSchema, + StructType tableSchema) { PathBasedSnapshotManager snapshotManager = new PathBasedSnapshotManager(path, spark.sessionState().newHadoopConf()); Snapshot snapshot = snapshotManager.loadLatestSnapshot(); + Engine engine = DefaultEngine.create(spark.sessionState().newHadoopConf()); return new DeltaV2ScanBuilder( tableName, snapshot, + engine, + Optional.empty(), snapshotManager, dataSchema, partitionSchema, @@ -935,48 +555,23 @@ private DeltaV2ScanBuilder createScanBuilder( CaseInsensitiveStringMap.empty()); } - private Predicate[] getPushedKernelPredicates(DeltaV2ScanBuilder builder) throws Exception { - // TODO: replace reflection with other testing manners, possibly Mockito ArgumentCaptor - Field field = DeltaV2ScanBuilder.class.getDeclaredField("pushedKernelPredicates"); - field.setAccessible(true); - return (Predicate[]) field.get(builder); - } - - private Filter[] getDataFilters(DeltaV2ScanBuilder builder) throws Exception { - // TODO: replace reflection with other testing manners, possibly Mockito ArgumentCaptor - Field field = DeltaV2ScanBuilder.class.getDeclaredField("dataFilters"); - field.setAccessible(true); - return (Filter[]) field.get(builder); - } - - private Optional getKernelScanBuilderPredicate(DeltaV2ScanBuilder builder) - throws Exception { - // TODO: replace reflection with other testing manners, possibly Mockito ArgumentCaptor - Field field = DeltaV2ScanBuilder.class.getDeclaredField("kernelScanBuilder"); - field.setAccessible(true); - Object kernelScanBuilder = field.get(builder); - - Field predicateField = kernelScanBuilder.getClass().getDeclaredField("predicate"); - predicateField.setAccessible(true); - Object raw = predicateField.get(kernelScanBuilder); - if (raw == null) { - return Optional.empty(); - } - Optional opt = (Optional) raw; - return opt.map(Predicate.class::cast); - } - // Limit Pushdown Tests + // + // Filter pushdown now flows as Catalyst Expressions through SupportsPushDownCatalystFilters, and + // partition/data classification is by partition-column membership: a predicate on a partition + // column (dep_id) is an exact partition filter that leaves no post-scan residual, while one on a + // data column (id/name) is a data filter that does. A pushed limit is kept only when no residual + // remains. @Test public void testPushLimit_nonNegativeAccepted(@TempDir File tempDir) { - DeltaV2ScanBuilder builder = createTestScanBuilder(tempDir); + DeltaV2ScanBuilder builder = newFilterScanBuilder(tempDir); assertTrue(builder.pushLimit(10), "A non-negative limit should be accepted as a hint"); } @Test public void testPushLimit_isPartiallyPushedIsTrue(@TempDir File tempDir) { - DeltaV2ScanBuilder builder = createTestScanBuilder(tempDir); + DeltaV2ScanBuilder builder = newFilterScanBuilder(tempDir); builder.pushLimit(10); // The pushdown is partial because pruning stops only at file boundaries, so the selected files // may produce more than the requested number of rows. Keep the default true so Spark retains @@ -986,7 +581,7 @@ public void testPushLimit_isPartiallyPushedIsTrue(@TempDir File tempDir) { @Test public void testPushLimit_propagatedToScan(@TempDir File tempDir) { - DeltaV2ScanBuilder builder = createTestScanBuilder(tempDir); + DeltaV2ScanBuilder builder = newFilterScanBuilder(tempDir); builder.pushLimit(42); DeltaV2Scan scan = (DeltaV2Scan) builder.build(); assertEquals(OptionalInt.of(42), scan.getPushedLimit()); @@ -994,14 +589,14 @@ public void testPushLimit_propagatedToScan(@TempDir File tempDir) { @Test public void testPushLimit_absentByDefault(@TempDir File tempDir) { - DeltaV2ScanBuilder builder = createTestScanBuilder(tempDir); + DeltaV2ScanBuilder builder = newFilterScanBuilder(tempDir); DeltaV2Scan scan = (DeltaV2Scan) builder.build(); assertEquals(OptionalInt.empty(), scan.getPushedLimit()); } @Test public void testPushLimit_lastValueWins(@TempDir File tempDir) { - DeltaV2ScanBuilder builder = createTestScanBuilder(tempDir); + DeltaV2ScanBuilder builder = newFilterScanBuilder(tempDir); builder.pushLimit(10); builder.pushLimit(20); assertEquals(OptionalInt.of(20), builder.getPushedLimit()); @@ -1011,7 +606,7 @@ public void testPushLimit_lastValueWins(@TempDir File tempDir) { @Test public void testPushLimit_negativeRejected(@TempDir File tempDir) { - DeltaV2ScanBuilder builder = createTestScanBuilder(tempDir); + DeltaV2ScanBuilder builder = newFilterScanBuilder(tempDir); assertThrows( IllegalArgumentException.class, () -> builder.pushLimit(-1), @@ -1019,10 +614,10 @@ public void testPushLimit_negativeRejected(@TempDir File tempDir) { } @Test - public void testPushLimit_clearedWhenDataFiltersPresent(@TempDir File tempDir) throws Exception { - DeltaV2ScanBuilder builder = createTestScanBuilder(tempDir); + public void testPushLimit_clearedWhenDataFiltersPresent(@TempDir File tempDir) { + DeltaV2ScanBuilder builder = newFilterScanBuilder(tempDir); // A data filter (on a non-partition column) becomes a post-scan residual. - builder.pushFilters(new Filter[] {new org.apache.spark.sql.sources.EqualTo("name", "x")}); + pushFilters(builder, new GreaterThan("id", 10)); builder.pushLimit(10); DeltaV2Scan scan = (DeltaV2Scan) builder.build(); assertEquals( @@ -1031,54 +626,26 @@ public void testPushLimit_clearedWhenDataFiltersPresent(@TempDir File tempDir) t "Pushed limit must be cleared when data filters would leave a post-scan residual"); } - @Test - public void testPushLimit_clearedWhenUnsupportedPartitionFilterLeavesResidual( - @TempDir File tempDir) throws Exception { - // StringEndsWith is not convertible to a kernel predicate, so even on a partition column - // (dep_id) it stays as a post-scan residual - but it is NOT a data filter (dep_id is a - // partition column). This is the case a data filters only check would miss: the residual - // exists, so the limit must still be cleared to mirror Spark's PhysicalOperation(_, Nil, _) - // gate. - DeltaV2ScanBuilder builder = createTestScanBuilder(tempDir); - Filter[] residual = - builder.pushFilters( - new Filter[] {new org.apache.spark.sql.sources.StringEndsWith("dep_id", "1")}); - assertEquals(1, residual.length, "unsupported partition filter should be a post-scan residual"); - assertEquals(0, getDataFilters(builder).length, "the residual must not be a data filter"); - - builder.pushLimit(10); - DeltaV2Scan scan = (DeltaV2Scan) builder.build(); - assertEquals( - OptionalInt.empty(), - scan.getPushedLimit(), - "Pushed limit must be cleared for any post-scan residual, not only data filters"); - } - @Test public void testPushLimit_remainsClearedAfterLaterEmptyPushFilters(@TempDir File tempDir) { - DeltaV2ScanBuilder builder = createTestScanBuilder(tempDir); - Filter[] residual = - builder.pushFilters( - new Filter[] {new org.apache.spark.sql.sources.StringEndsWith("dep_id", "1")}); - assertEquals(1, residual.length); + DeltaV2ScanBuilder builder = newFilterScanBuilder(tempDir); + pushFilters(builder, new GreaterThan("id", 10)); // ScanBuilder state is cumulative. A later empty call cannot retract the residual Spark must // still evaluate from the earlier call, so it must not make limit pruning safe again. - assertEquals(0, builder.pushFilters(new Filter[0]).length); + builder.pushFilters( + scala.jdk.javaapi.CollectionConverters.asScala(java.util.List.of()).toList()); assertTrue(builder.pushLimit(10)); assertEquals(OptionalInt.empty(), ((DeltaV2Scan) builder.build()).getPushedLimit()); } @Test - public void testPushLimit_keptWhenFullyPushedPartitionFilterLeavesNoResidual( - @TempDir File tempDir) throws Exception { - // A fully-converted partition filter (EqualTo on dep_id) is used for partition pruning and - // leaves NO post-scan residual, so the pushed limit is safe to keep - matching Spark, which - // still offers pushLimit in this shape. - DeltaV2ScanBuilder builder = createTestScanBuilder(tempDir); - Filter[] residual = - builder.pushFilters(new Filter[] {new org.apache.spark.sql.sources.EqualTo("dep_id", 1)}); - assertEquals(0, residual.length, "fully-pushed partition filter should leave no residual"); + public void testPushLimit_keptWhenPartitionFilterLeavesNoResidual(@TempDir File tempDir) { + // A partition filter (EqualTo on dep_id) is used for partition pruning and leaves NO post-scan + // residual, so the pushed limit is safe to keep - matching Spark, which still offers pushLimit + // in this shape. + DeltaV2ScanBuilder builder = newFilterScanBuilder(tempDir); + pushFilters(builder, new EqualTo("dep_id", 1)); builder.pushLimit(10); DeltaV2Scan scan = (DeltaV2Scan) builder.build(); diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/DeltaV2ScanTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/DeltaV2ScanTest.java index 7c8da7f9b39..de8b32619f2 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/DeltaV2ScanTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/DeltaV2ScanTest.java @@ -11,10 +11,11 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.OptionalInt; +import java.util.OptionalLong; +import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; import org.apache.spark.sql.catalyst.TableIdentifier; import org.apache.spark.sql.catalyst.catalog.CatalogColumnStat; @@ -30,6 +31,9 @@ import org.apache.spark.sql.connector.read.ScanBuilder; import org.apache.spark.sql.connector.read.Statistics; import org.apache.spark.sql.connector.read.colstats.ColumnStatistics; +import org.apache.spark.sql.delta.Snapshot; +import org.apache.spark.sql.delta.actions.AddFile; +import org.apache.spark.sql.execution.datasources.FilePartition; import org.apache.spark.sql.execution.datasources.PartitionedFile; import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.StructType; @@ -503,14 +507,17 @@ protected static void checkSupportsRuntimeFilters( long beforeDppTotalBytes = getTotalBytes(deltaV2Scan); long beforeDppEstimatedSize = getEstimatedSizeInBytes(deltaV2Scan); assert (beforeDppFiles.size() == 5); - // Without column pruning, estimatedSizeInBytes should equal totalBytes + // Snappy tables use the selected-file byte sum directly. assertEquals(beforeDppTotalBytes, beforeDppEstimatedSize); deltaV2Scan.filter(runtimeFilters); List afterDppFiles = getPartitionedFiles(deltaV2Scan); long afterDppTotalBytes = getTotalBytes(deltaV2Scan); long afterDppEstimatedSize = getEstimatedSizeInBytes(deltaV2Scan); - assert (beforeDppFiles.containsAll(afterDppFiles)); + // Compare by file path: each getPartitionedFiles call rebuilds fresh PartitionedFile instances + // (physical input partitions are materialized on demand), so the surviving files are a subset + // by path rather than by object identity. + assert (partitionedFilePaths(beforeDppFiles).containsAll(partitionedFilePaths(afterDppFiles))); assert (beforeDppTotalBytes >= afterDppTotalBytes); List expectedPartitionFilesAfterDpp = new ArrayList<>(); @@ -526,17 +533,21 @@ protected static void checkSupportsRuntimeFilters( } assertEquals(expectedPartitionFilesAfterDpp.size(), afterDppFiles.size()); - assertEquals(new HashSet<>(expectedPartitionFilesAfterDpp), new HashSet<>(afterDppFiles)); + assertEquals( + partitionedFilePaths(expectedPartitionFilesAfterDpp), partitionedFilePaths(afterDppFiles)); assertEquals(expectedTotalBytesAfterDpp, afterDppTotalBytes); - // Without column pruning, estimatedSizeInBytes should equal totalBytes after filtering too + // Runtime filtering updates the selected-file byte sum used by the size API. assertEquals(afterDppTotalBytes, afterDppEstimatedSize); } private static List getPartitionedFiles(DeltaV2Scan scan) throws Exception { - scan.estimateStatistics(); // ensurePlanned - Field field = DeltaV2Scan.class.getDeclaredField("partitionedFiles"); - field.setAccessible(true); - return (List) field.get(scan); + return Arrays.stream(scan.toBatch().planInputPartitions()) + .flatMap(partition -> Arrays.stream(((FilePartition) partition).files())) + .collect(Collectors.toList()); + } + + private static List partitionedFilePaths(List files) { + return files.stream().map(pf -> pf.filePath().toString()).sorted().collect(Collectors.toList()); } @Test @@ -545,60 +556,86 @@ public void testSelectedFilesTracksPlannedAndRuntimeFilteredFiles() throws Excep DeltaV2Scan scan = (DeltaV2Scan) builder.build(); List plannedFiles = getPartitionedFiles(scan); - List selectedFiles = scan.getSelectedFiles(); + List selectedFiles = + scala.jdk.javaapi.CollectionConverters.asJava(scan.preparedScan().files()); assertEquals(plannedFiles.size(), selectedFiles.size()); assertEquals(5, selectedFiles.size(), "test table should start with all selected files"); assertTrue( - selectedFiles.stream().allMatch(file -> file.getPath().contains("city=")), + selectedFiles.stream().allMatch(file -> file.path().contains("city=")), "selected file descriptors should expose Delta-relative AddFile paths"); assertTrue( - selectedFiles.stream().allMatch(file -> file.getSize() > 0), + selectedFiles.stream().allMatch(file -> file.size() > 0), "selected file descriptors should expose file sizes"); scan.filter(new Predicate[] {cityPredicate}); List filteredFiles = getPartitionedFiles(scan); - List filteredSelectedFiles = scan.getSelectedFiles(); + List filteredSelectedFiles = + scala.jdk.javaapi.CollectionConverters.asJava(scan.preparedScan().files()); assertEquals(filteredFiles.size(), filteredSelectedFiles.size()); assertEquals(2, filteredSelectedFiles.size(), "city=hz runtime filter should keep two files"); assertTrue( - filteredSelectedFiles.stream().allMatch(file -> file.getPath().contains("city=hz")), + filteredSelectedFiles.stream().allMatch(file -> file.path().contains("city=hz")), "selected files should be pruned with runtime partition filters"); } + @Test + public void testPlannedSnapshotTriggersPlanningAndReturnsSameSnapshot() throws Exception { + DeltaV2ScanBuilder builder = (DeltaV2ScanBuilder) table.newScanBuilder(options); + DeltaV2Scan scan = (DeltaV2Scan) builder.build(); + assertFalse(getPlanned(scan), "build() should leave file planning lazy"); + + Snapshot plannedSnapshot = scan.preparedScan().scannedSnapshot(); + + assertTrue(getPlanned(scan), "preparedScan() should trigger file planning"); + assertSame( + plannedSnapshot, + scan.preparedScan().scannedSnapshot(), + "repeated preparedScan() calls should return the same planned snapshot"); + } + private static long getTotalBytes(DeltaV2Scan scan) throws Exception { - scan.estimateStatistics(); // ensurePlanned - Field field = DeltaV2Scan.class.getDeclaredField("totalBytes"); - field.setAccessible(true); - return (long) field.get(scan); + scan.estimateStatistics(); // May select files unless catalog statistics are available. + if (!getPlanned(scan)) { + return 0L; + } + scala.Option bytesCompressed = scan.preparedScan().scanned().bytesCompressed(); + if (bytesCompressed.isDefined()) { + return ((Number) bytesCompressed.get()).longValue(); + } + // No scanned-bytes aggregate (e.g. a table without file statistics): fall back to the + // selected-file byte sum, matching estimateSizeInBytes(). + return scala.jdk.javaapi.CollectionConverters.asJava(scan.preparedScan().files()).stream() + .mapToLong(AddFile::size) + .sum(); } private static long getEstimatedSizeInBytes(DeltaV2Scan scan) throws Exception { - scan.estimateStatistics(); // ensurePlanned - Field field = DeltaV2Scan.class.getDeclaredField("estimatedSizeInBytes"); - field.setAccessible(true); - return (long) field.get(scan); + return scan.estimateSizeInBytes().getAsLong(); } - private static org.apache.spark.sql.sources.Filter[] getDataFilters(DeltaV2Scan scan) - throws Exception { + private static org.apache.spark.sql.catalyst.expressions.Expression[] getDataFilters( + DeltaV2Scan scan) throws Exception { Field field = DeltaV2Scan.class.getDeclaredField("dataFilters"); field.setAccessible(true); - return (org.apache.spark.sql.sources.Filter[]) field.get(scan); + return (org.apache.spark.sql.catalyst.expressions.Expression[]) field.get(scan); } - private static long getTotalRows(DeltaV2Scan scan) throws Exception { - scan.estimateStatistics(); // ensurePlanned - Field field = DeltaV2Scan.class.getDeclaredField("totalRows"); + private static boolean getPlanned(DeltaV2Scan scan) throws Exception { + Field field = DeltaV2Scan.class.getDeclaredField("planned"); field.setAccessible(true); - return (long) field.get(scan); + return (boolean) field.get(scan); + } + + private static long getTotalRows(DeltaV2Scan scan) throws Exception { + return isRowCountKnown(scan) + ? ((Number) scan.preparedScan().scanned().rows().get()).longValue() + : 0L; } private static boolean isRowCountKnown(DeltaV2Scan scan) throws Exception { - scan.estimateStatistics(); // ensurePlanned - Field field = DeltaV2Scan.class.getDeclaredField("rowCountKnown"); - field.setAccessible(true); - return (boolean) field.get(scan); + scan.estimateStatistics(); + return getPlanned(scan) && scan.preparedScan().scanned().rows().isDefined(); } // ================================================================================================ @@ -607,20 +644,30 @@ private static boolean isRowCountKnown(DeltaV2Scan scan) throws Exception { @Test public void testNumRowsEmptyWhenStatsDisabled() throws Exception { - // With CBO and planStats disabled (the default), numRows() should return empty even when all - // files have stats, matching V1 behavior (LogicalRelation.computeStats()). - DeltaV2ScanBuilder builder = (DeltaV2ScanBuilder) table.newScanBuilder(options); - DeltaV2Scan scan = (DeltaV2Scan) builder.build(); + // Explicitly disable both settings so the test does not depend on environment defaults. + withSQLConf( + "spark.sql.cbo.enabled", + "false", + () -> { + withSQLConf( + "spark.sql.cbo.planStats.enabled", + "false", + () -> { + DeltaV2ScanBuilder builder = (DeltaV2ScanBuilder) table.newScanBuilder(options); + DeltaV2Scan scan = (DeltaV2Scan) builder.build(); - assertFalse( - isRowCountKnown(scan), "rowCountKnown should be false when CBO and planStats are disabled"); - assertFalse( - scan.estimateStatistics().numRows().isPresent(), - "numRows() should be empty when CBO and planStats are disabled"); + assertFalse( + isRowCountKnown(scan), + "rowCountKnown should be false when CBO and planStats are disabled"); + assertFalse( + scan.estimateStatistics().numRows().isPresent(), + "numRows() should be empty when CBO and planStats are disabled"); + }); + }); } @Test - public void testNumRowsInStatistics() throws Exception { + public void testPathStatisticsDoNotReportPerFileNumRows() throws Exception { // Table has 5 rows inserted as 5 separate partitions (1 row each), all with stats. withSQLConf( "spark.sql.cbo.planStats.enabled", @@ -629,47 +676,47 @@ public void testNumRowsInStatistics() throws Exception { DeltaV2ScanBuilder builder = (DeltaV2ScanBuilder) table.newScanBuilder(options); DeltaV2Scan scan = (DeltaV2Scan) builder.build(); - assertTrue(isRowCountKnown(scan), "Row count should be known when all files have stats"); - assertEquals(5L, getTotalRows(scan), "Total rows should match the 5 inserted rows"); - assertTrue(scan.estimateStatistics().numRows().isPresent(), "numRows should be present"); - assertEquals(5L, scan.estimateStatistics().numRows().getAsLong()); + assertFalse( + isRowCountKnown(scan), + "retaining per-file stats must not synthesize a missing DeltaScan row count"); + assertFalse( + scan.estimateStatistics().numRows().isPresent(), + "path-based optimizer statistics should not expose per-file row counts"); }); } @Test public void testNumRowsAfterRuntimeFiltering() throws Exception { - // Runtime partition filtering recomputes totalRows from the per-file counts of files that - // survive pruning, so numRows() reflects the post-prune row count rather than the - // pre-filter total or empty. + // Start with a known aggregate from file selection. Runtime partition filtering recomputes + // this count, while path-based optimizer statistics intentionally omit numRows(). withSQLConf( - "spark.sql.cbo.planStats.enabled", + "spark.databricks.delta.alwaysCollectStats.enabled", "true", () -> { DeltaV2ScanBuilder builder = (DeltaV2ScanBuilder) table.newScanBuilder(options); DeltaV2Scan scan = (DeltaV2Scan) builder.build(); - assertEquals( - 5L, scan.estimateStatistics().numRows().getAsLong(), "5 rows before filtering"); + scan.estimateStatistics(); + assertEquals(5L, getTotalRows(scan), "5 rows before filtering"); // Two rows in the table have city=hz (Alice and Bob, in different date partitions). scan.filter(new Predicate[] {cityPredicate}); // city=hz - assertTrue( + assertFalse( scan.estimateStatistics().numRows().isPresent(), - "numRows should remain known after runtime filtering"); + "path-based optimizer statistics should not expose per-file row counts"); assertEquals( 2L, - scan.estimateStatistics().numRows().getAsLong(), - "numRows should be recomputed to the post-prune count (2 city=hz rows)"); + getTotalRows(scan), + "the internal row count should be recomputed after runtime filtering"); }); } @Test public void testNumRowsZeroAfterFilteringOutAllFiles() throws Exception { - // When runtime filtering prunes every file, totalRows recomputes to 0 (still a known - // value, not OptionalLong.empty()). + // When runtime filtering prunes every file, the internal totalRows recomputes to 0. withSQLConf( - "spark.sql.cbo.planStats.enabled", + "spark.databricks.delta.alwaysCollectStats.enabled", "true", () -> { DeltaV2ScanBuilder builder = (DeltaV2ScanBuilder) table.newScanBuilder(options); @@ -677,13 +724,15 @@ public void testNumRowsZeroAfterFilteringOutAllFiles() throws Exception { scan.filter(new Predicate[] {negativeCityPredicate}); // city=zz doesn't exist - assertTrue( + assertFalse( scan.estimateStatistics().numRows().isPresent(), - "numRows should remain known after runtime filtering (even when all files filtered)"); + "path-based optimizer statistics should not expose per-file row counts"); assertEquals( 0L, - scan.estimateStatistics().numRows().getAsLong(), - "numRows should be 0 when all files are filtered out"); + getTotalRows(scan), + "the internal row count should be 0 when all files are filtered out"); + assertTrue( + isRowCountKnown(scan), "an empty selection should retain the known zero count"); }); } @@ -917,17 +966,11 @@ public void testEqualsWithDifferentOptions() { public void testEqualsWithSameFilters() { // Both scans with equivalent filters created separately (not same instance) DeltaV2ScanBuilder builder1 = (DeltaV2ScanBuilder) table.newScanBuilder(options); - builder1.pushFilters( - new org.apache.spark.sql.sources.Filter[] { - new org.apache.spark.sql.sources.EqualTo("city", "hz") - }); + pushFilters(builder1, new org.apache.spark.sql.sources.EqualTo("city", "hz")); DeltaV2Scan scan1 = (DeltaV2Scan) builder1.build(); DeltaV2ScanBuilder builder2 = (DeltaV2ScanBuilder) table.newScanBuilder(options); - builder2.pushFilters( - new org.apache.spark.sql.sources.Filter[] { - new org.apache.spark.sql.sources.EqualTo("city", "hz") - }); + pushFilters(builder2, new org.apache.spark.sql.sources.EqualTo("city", "hz")); DeltaV2Scan scan2 = (DeltaV2Scan) builder2.build(); // Same options and equivalent filters should be equal @@ -943,10 +986,7 @@ public void testEqualsWithDifferentFilters() { // Scan with filters pushed DeltaV2ScanBuilder builder2 = (DeltaV2ScanBuilder) table.newScanBuilder(options); - builder2.pushFilters( - new org.apache.spark.sql.sources.Filter[] { - new org.apache.spark.sql.sources.EqualTo("city", "hz") - }); + pushFilters(builder2, new org.apache.spark.sql.sources.EqualTo("city", "hz")); DeltaV2Scan scan2 = (DeltaV2Scan) builder2.build(); // Same options but different filters should not be equal and hashCodes should differ @@ -962,11 +1002,11 @@ public void testEqualsWithPushedFiltersInDifferentOrder() { new org.apache.spark.sql.sources.EqualTo("date", "20180520"); DeltaV2ScanBuilder builder1 = (DeltaV2ScanBuilder) table.newScanBuilder(options); - builder1.pushFilters(new org.apache.spark.sql.sources.Filter[] {cityEq, dateEq}); + pushFilters(builder1, new org.apache.spark.sql.sources.Filter[] {cityEq, dateEq}); DeltaV2Scan scan1 = (DeltaV2Scan) builder1.build(); DeltaV2ScanBuilder builder2 = (DeltaV2ScanBuilder) table.newScanBuilder(options); - builder2.pushFilters(new org.apache.spark.sql.sources.Filter[] {dateEq, cityEq}); + pushFilters(builder2, new org.apache.spark.sql.sources.Filter[] {dateEq, cityEq}); DeltaV2Scan scan2 = (DeltaV2Scan) builder2.build(); assertEquals(scan1, scan2); @@ -985,11 +1025,11 @@ public void testEqualsWithDataFiltersInDifferentOrder() throws Exception { new org.apache.spark.sql.sources.GreaterThan("cnt", 10); DeltaV2ScanBuilder builder1 = (DeltaV2ScanBuilder) table.newScanBuilder(options); - builder1.pushFilters(new org.apache.spark.sql.sources.Filter[] {nameEq, cntGt}); + pushFilters(builder1, new org.apache.spark.sql.sources.Filter[] {nameEq, cntGt}); DeltaV2Scan scan1 = (DeltaV2Scan) builder1.build(); DeltaV2ScanBuilder builder2 = (DeltaV2ScanBuilder) table.newScanBuilder(options); - builder2.pushFilters(new org.apache.spark.sql.sources.Filter[] {cntGt, nameEq}); + pushFilters(builder2, new org.apache.spark.sql.sources.Filter[] {cntGt, nameEq}); DeltaV2Scan scan2 = (DeltaV2Scan) builder2.build(); // Sanity check that dataFilters is actually populated, otherwise this test would trivially @@ -1002,119 +1042,47 @@ public void testEqualsWithDataFiltersInDifferentOrder() throws Exception { } // ================================================================================================ - // Tests for estimated size with column projection + // Tests for selected-file size estimates // ================================================================================================ @Test public void testEstimatedSizeMatchesStatistics() throws Exception { - // Test that estimateStatistics().sizeInBytes() returns the estimatedSizeInBytes field DeltaV2ScanBuilder builder = (DeltaV2ScanBuilder) table.newScanBuilder(options); DeltaV2Scan scan = (DeltaV2Scan) builder.build(); long estimatedSizeFromStats = scan.estimateStatistics().sizeInBytes().getAsLong(); - long estimatedSizeFromField = getEstimatedSizeInBytes(scan); + long estimatedSizeFromSizeOnlyApi = scan.estimateSizeInBytes().getAsLong(); - assertEquals(estimatedSizeFromField, estimatedSizeFromStats); + assertEquals(estimatedSizeFromSizeOnlyApi, estimatedSizeFromStats); } @Test - public void testEstimatedSizeWithColumnPruning() throws Exception { - // Test that with column pruning, estimatedSizeInBytes is computed correctly - // Table schema: (part INT, date STRING, city STRING, name STRING, cnt INT) - // Partition columns: (date STRING, city STRING, part INT) - // Data columns: (name STRING, cnt INT) - // - // Formula: estimatedBytes = (totalBytes * outputRowSize) / fullSchemaRowSize - // Where: - // ROW_OVERHEAD = 8 - // dataSchema.defaultSize() = 20 (STRING) + 4 (INT) = 24 - // partitionSchema.defaultSize() = 20 + 20 + 4 = 44 - // fullSchemaRowSize = 8 + 24 + 44 = 76 - // - // With pruning to only 'name' column: - // readDataSchema.defaultSize() = 20 (STRING only) - // readSchema().defaultSize() = 20 + 44 = 64 - // outputRowSize = 8 + 64 = 72 - // estimatedBytes = (totalBytes * 72) / 76 - DeltaV2ScanBuilder builder = (DeltaV2ScanBuilder) table.newScanBuilder(options); - - // Prune columns to only include 'name' (a data column) and partition columns - // This simulates: SELECT name, date, city, part FROM table - StructType prunedSchema = - new StructType() - .add("name", DataTypes.StringType) // only one data column - .add("date", DataTypes.StringType) // partition columns are always included - .add("city", DataTypes.StringType) - .add("part", DataTypes.IntegerType); - builder.pruneColumns(prunedSchema); - - DeltaV2Scan scan = (DeltaV2Scan) builder.build(); - - long totalBytes = getTotalBytes(scan); - long estimatedSize = getEstimatedSizeInBytes(scan); - - // Calculate expected estimated size using the formula - // outputRowSize = 8 + 64 = 72, fullSchemaRowSize = 8 + 24 + 44 = 76 - // Note: We don't use Math.max(1, ...) here because totalBytes is guaranteed to be large enough - // (parquet files with actual data) that the division result won't be zero. - long expectedEstimatedSize = (totalBytes * 72) / 76; + public void testEstimateSizeInBytesUsesCheapFileSizePath() throws Exception { + withSQLConf( + "spark.sql.cbo.enabled", + "false", + () -> + withSQLConf( + "spark.sql.cbo.planStats.enabled", + "false", + () -> { + DeltaV2ScanBuilder builder = (DeltaV2ScanBuilder) table.newScanBuilder(options); + builder.pruneColumns(new StructType().add("name", DataTypes.StringType)); + DeltaV2Scan scan = (DeltaV2Scan) builder.build(); - assertTrue(totalBytes > 0, "totalBytes should be positive"); - assertEquals( - expectedEstimatedSize, - estimatedSize, - String.format( - "estimatedSize should be (totalBytes * 72) / 76 = (%d * 72) / 76 = %d", - totalBytes, expectedEstimatedSize)); + long sizeInBytes = scan.estimateSizeInBytes().getAsLong(); + assertEquals(getTotalBytes(scan), sizeInBytes); + assertTrue(sizeInBytes > 0, "post-pruning size should be positive"); + assertFalse( + isRowCountKnown(scan), + "the size-only path must not request per-file row statistics"); + })); } @Test - public void testEstimatedSizeWithColumnPruningAndFiltering() throws Exception { - // Test that column pruning and runtime filtering work together correctly - // Using same formula as testEstimatedSizeWithColumnPruning: - // estimatedBytes = (totalBytes * 72) / 76 - DeltaV2ScanBuilder builder = (DeltaV2ScanBuilder) table.newScanBuilder(options); - - // Prune columns to only include 'name' column - StructType prunedSchema = - new StructType() - .add("name", DataTypes.StringType) - .add("date", DataTypes.StringType) - .add("city", DataTypes.StringType) - .add("part", DataTypes.IntegerType); - builder.pruneColumns(prunedSchema); - - DeltaV2Scan scan = (DeltaV2Scan) builder.build(); - - // Get initial stats with column pruning - long initialTotalBytes = getTotalBytes(scan); - long initialEstimatedSize = getEstimatedSizeInBytes(scan); - - // Verify initial estimated size matches formula - // Note: No Math.max(1, ...) needed - totalBytes from parquet files is large enough - long expectedInitialEstimated = (initialTotalBytes * 72) / 76; - assertEquals( - expectedInitialEstimated, - initialEstimatedSize, - "Initial estimatedSize should match formula"); - - // Apply a runtime filter - scan.filter(new Predicate[] {cityPredicate}); // city=hz - - // After filtering, verify both values are updated correctly - long afterFilterTotalBytes = getTotalBytes(scan); - long afterFilterEstimatedSize = getEstimatedSizeInBytes(scan); - - // Verify estimated size matches formula with new totalBytes - long expectedAfterFilterEstimated = (afterFilterTotalBytes * 72) / 76; - assertEquals( - expectedAfterFilterEstimated, - afterFilterEstimatedSize, - "After filter, estimatedSize should match formula with new totalBytes"); - - // Verify both values were reduced - assertTrue(afterFilterTotalBytes < initialTotalBytes, "totalBytes should be reduced"); - assertTrue(afterFilterEstimatedSize < initialEstimatedSize, "estimatedSize should be reduced"); + public void testStatisticsRequireSparkPostPushdownFilterAdjustment() { + DeltaV2Scan scan = (DeltaV2Scan) ((DeltaV2ScanBuilder) table.newScanBuilder(options)).build(); + assertFalse(scan.reflectsFullyPushedDownFilters()); } @Test @@ -1304,10 +1272,9 @@ private CatalogTable injectCatalogStats(String tblName, CatalogStatistics stats) } @Test - public void testEstimateStatisticsWithCatalogStats_cboEnabled(@TempDir File tempDir) - throws Exception { + public void testEstimateStatisticsWithCatalogStats(@TempDir File tempDir) throws Exception { String path = tempDir.getAbsolutePath(); - String tblName = "stats_cbo_enabled"; + String tblName = "catalog_stats"; spark.sql( String.format( "CREATE TABLE %s (id INT, name STRING, value DOUBLE) USING delta LOCATION '%s'", @@ -1334,49 +1301,37 @@ public void testEstimateStatisticsWithCatalogStats_cboEnabled(@TempDir File temp CatalogTable catalogTable = injectCatalogStats(tblName, catalogStats); - withSQLConf( - "spark.sql.cbo.enabled", - "true", - () -> { - Identifier id = Identifier.of(new String[] {"default"}, tblName); - DeltaV2Table sparkTable = new DeltaV2Table(id, catalogTable, Collections.emptyMap()); + Identifier id = Identifier.of(new String[] {"default"}, tblName); + DeltaV2Table sparkTable = new DeltaV2Table(id, catalogTable, Collections.emptyMap()); - DeltaV2ScanBuilder builder = - (DeltaV2ScanBuilder) - sparkTable.newScanBuilder(new CaseInsensitiveStringMap(new HashMap<>())); - DeltaV2Scan scan = (DeltaV2Scan) builder.build(); - Statistics stats = scan.estimateStatistics(); + DeltaV2ScanBuilder builder = + (DeltaV2ScanBuilder) + sparkTable.newScanBuilder(new CaseInsensitiveStringMap(new HashMap<>())); + DeltaV2Scan scan = (DeltaV2Scan) builder.build(); + Statistics stats = scan.estimateStatistics(); - // numRows comes from per-file (post-prune) stats - assertTrue(stats.numRows().isPresent(), "numRows should be present with CBO enabled"); - assertEquals(2L, stats.numRows().getAsLong(), "numRows should be 2"); - - // sizeInBytes should still come from planned files (more accurate) - assertTrue(stats.sizeInBytes().isPresent(), "sizeInBytes should be present"); - assertTrue(stats.sizeInBytes().getAsLong() > 0, "sizeInBytes should be positive"); - - // Should have column stats - Map colStats = stats.columnStats(); - assertNotNull(colStats, "columnStats should not be null"); - assertFalse(colStats.isEmpty(), "columnStats should not be empty"); - - // Check that column stats contain expected columns - ColumnStatistics idStats = colStats.get(FieldReference.apply("id")); - assertNotNull(idStats, "id column stats should be present"); - assertTrue(idStats.nullCount().isPresent(), "id nullCount should be present"); - assertTrue(idStats.distinctCount().isPresent(), "id distinctCount should be present"); - assertTrue(idStats.min().isPresent(), "id min should be present"); - assertTrue(idStats.max().isPresent(), "id max should be present"); - assertEquals(1, idStats.min().get(), "id min should be 1"); - assertEquals(2, idStats.max().get(), "id max should be 2"); - }); + // Catalog rows and columns are returned without an explicit size so Spark can derive a + // projection-aware size from the retained output. + assertTrue(stats.numRows().isPresent(), "numRows should be present"); + assertEquals(2L, stats.numRows().getAsLong(), "numRows should be 2"); + assertFalse(stats.sizeInBytes().isPresent(), "Spark should derive projected size"); + + Map colStats = stats.columnStats(); + assertNotNull(colStats, "columnStats should not be null"); + assertFalse(colStats.isEmpty(), "columnStats should not be empty"); + + ColumnStatistics idStats = colStats.get(FieldReference.apply("id")); + assertNotNull(idStats, "id column stats should be present"); + assertTrue(idStats.nullCount().isPresent(), "id nullCount should be present"); + assertTrue(idStats.distinctCount().isPresent(), "id distinctCount should be present"); + assertTrue(idStats.min().isPresent(), "id min should be present"); + assertTrue(idStats.max().isPresent(), "id max should be present"); + assertEquals(1, idStats.min().get(), "id min should be 1"); + assertEquals(2, idStats.max().get(), "id max should be 2"); } @Test - public void testPerFileNumRowsPreferredOverCatalog(@TempDir File tempDir) throws Exception { - // Verifies that per-file (post-prune) numRows is used in preference to catalog numRows. - // numRows() reflects the row count for this scan after pruning, while catalog stats are - // table-level and unpruned. + public void testCatalogNumRowsPreferredForFullStatistics(@TempDir File tempDir) throws Exception { String path = tempDir.getAbsolutePath(); String tblName = "stats_per_file_wins"; spark.sql( @@ -1392,172 +1347,17 @@ public void testPerFileNumRowsPreferredOverCatalog(@TempDir File tempDir) throws buildColStatsMap(new String[] {}, new CatalogColumnStat[] {})); CatalogTable catalogTable = injectCatalogStats(tblName, catalogStats); - withSQLConf( - "spark.sql.cbo.enabled", - "true", - () -> { - Identifier id = Identifier.of(new String[] {"default"}, tblName); - DeltaV2Table sparkTable = new DeltaV2Table(id, catalogTable, Collections.emptyMap()); - DeltaV2ScanBuilder builder = - (DeltaV2ScanBuilder) - sparkTable.newScanBuilder(new CaseInsensitiveStringMap(new HashMap<>())); - DeltaV2Scan scan = (DeltaV2Scan) builder.build(); - Statistics stats = scan.estimateStatistics(); - - // Per-file numRows wins over catalog stats. - assertTrue(stats.numRows().isPresent(), "numRows should be present"); - assertEquals( - 2L, - stats.numRows().getAsLong(), - "numRows should come from per-file (2), not catalog (999)"); - }); - } - - @Test - public void testCatalogNumRowsFallbackWhenPerFileUnknown(@TempDir File tempDir) throws Exception { - // When per-file numRecords is unavailable for any AddFile (rowCountKnown=false), numRows() - // should fall back to the catalog value rather than return OptionalLong.empty(). - String path = tempDir.getAbsolutePath(); - String tblName = "stats_catalog_fallback"; - try { - spark.sql( - String.format( - "CREATE TABLE %s (id INT, name STRING) USING delta LOCATION '%s'", tblName, path)); - spark.sql(String.format("INSERT INTO %s VALUES (1, 'a')", tblName)); - - // Disable stats collection so the next AddFile has no numRecords. With one file lacking - // numRecords, rowCountKnown is false for the whole scan. - withSQLConf( - "spark.databricks.delta.stats.collect", - "false", - () -> spark.sql(String.format("INSERT INTO %s VALUES (2, 'b')", tblName))); - - // Inject catalog stats with a distinguishable row count (777) so the fallback is observable. - CatalogStatistics catalogStats = - new CatalogStatistics( - scala.math.BigInt.apply(1024L), - scala.Option.apply(scala.math.BigInt.apply(777L)), - buildColStatsMap(new String[] {}, new CatalogColumnStat[] {})); - CatalogTable catalogTable = injectCatalogStats(tblName, catalogStats); - - withSQLConf( - "spark.sql.cbo.enabled", - "true", - () -> { - Identifier id = Identifier.of(new String[] {"default"}, tblName); - DeltaV2Table sparkTable = new DeltaV2Table(id, catalogTable, Collections.emptyMap()); - DeltaV2ScanBuilder builder = - (DeltaV2ScanBuilder) - sparkTable.newScanBuilder(new CaseInsensitiveStringMap(new HashMap<>())); - DeltaV2Scan scan = (DeltaV2Scan) builder.build(); - - assertFalse( - isRowCountKnown(scan), - "rowCountKnown should be false when an AddFile lacks numRecords"); - - Statistics stats = scan.estimateStatistics(); - assertTrue( - stats.numRows().isPresent(), - "numRows should fall back to catalog when per-file unknown"); - assertEquals( - 777L, - stats.numRows().getAsLong(), - "numRows should come from catalog stats (777) when per-file is unknown"); - }); - } finally { - spark.sql("DROP TABLE IF EXISTS " + tblName); - } - } - - @Test - public void testEstimateStatisticsWithCatalogStats_cboDisabled(@TempDir File tempDir) - throws Exception { - String path = tempDir.getAbsolutePath(); - String tblName = "stats_cbo_disabled"; - spark.sql( - String.format( - "CREATE TABLE %s (id INT, name STRING) USING delta LOCATION '%s'", tblName, path)); - spark.sql(String.format("INSERT INTO %s VALUES (1, 'a'), (2, 'b')", tblName)); - - // Inject catalog stats - CatalogStatistics catalogStats = - new CatalogStatistics( - scala.math.BigInt.apply(512L), - scala.Option.apply(scala.math.BigInt.apply(2L)), - scala.collection.immutable.Map$.MODULE$.empty()); - - CatalogTable catalogTable = injectCatalogStats(tblName, catalogStats); - - withSQLConf( - "spark.sql.cbo.enabled", - "false", - () -> { - Identifier id = Identifier.of(new String[] {"default"}, tblName); - DeltaV2Table sparkTable = new DeltaV2Table(id, catalogTable, Collections.emptyMap()); - - DeltaV2ScanBuilder builder = - (DeltaV2ScanBuilder) - sparkTable.newScanBuilder(new CaseInsensitiveStringMap(new HashMap<>())); - DeltaV2Scan scan = (DeltaV2Scan) builder.build(); - Statistics stats = scan.estimateStatistics(); - - // With CBO disabled, numRows should be empty (matching V1 behavior) - assertFalse(stats.numRows().isPresent(), "numRows should be empty with CBO disabled"); - - // sizeInBytes should still come from planned files - assertTrue(stats.sizeInBytes().isPresent(), "sizeInBytes should be present"); - assertTrue(stats.sizeInBytes().getAsLong() > 0, "sizeInBytes should be positive"); - }); - } - - @Test - public void testEstimateStatisticsWithCatalogStats_planStatsEnabled(@TempDir File tempDir) - throws Exception { - String path = tempDir.getAbsolutePath(); - String tblName = "stats_plan_stats_enabled"; - spark.sql( - String.format( - "CREATE TABLE %s (id INT, name STRING) USING delta LOCATION '%s'", tblName, path)); - spark.sql(String.format("INSERT INTO %s VALUES (1, 'a'), (2, 'b')", tblName)); - - // Inject catalog stats with numRows - CatalogStatistics catalogStats = - new CatalogStatistics( - scala.math.BigInt.apply(512L), - scala.Option.apply(scala.math.BigInt.apply(2L)), - scala.collection.immutable.Map$.MODULE$.empty()); - - CatalogTable catalogTable = injectCatalogStats(tblName, catalogStats); - - // CBO disabled but planStatsEnabled=true should still surface catalog stats - withSQLConf( - "spark.sql.cbo.enabled", - "false", - () -> { - withSQLConf( - "spark.sql.cbo.planStats.enabled", - "true", - () -> { - Identifier id = Identifier.of(new String[] {"default"}, tblName); - DeltaV2Table sparkTable = - new DeltaV2Table(id, catalogTable, Collections.emptyMap()); - - DeltaV2ScanBuilder builder = - (DeltaV2ScanBuilder) - sparkTable.newScanBuilder(new CaseInsensitiveStringMap(new HashMap<>())); - DeltaV2Scan scan = (DeltaV2Scan) builder.build(); - Statistics stats = scan.estimateStatistics(); - - // With planStatsEnabled, numRows should be present (from per-file stats) - assertTrue( - stats.numRows().isPresent(), "numRows should be present with planStatsEnabled"); - assertEquals(2L, stats.numRows().getAsLong(), "numRows should be 2"); + Identifier id = Identifier.of(new String[] {"default"}, tblName); + DeltaV2Table sparkTable = new DeltaV2Table(id, catalogTable, Collections.emptyMap()); + DeltaV2ScanBuilder builder = + (DeltaV2ScanBuilder) + sparkTable.newScanBuilder(new CaseInsensitiveStringMap(new HashMap<>())); + DeltaV2Scan scan = (DeltaV2Scan) builder.build(); + Statistics stats = scan.estimateStatistics(); - // sizeInBytes should still come from planned files - assertTrue(stats.sizeInBytes().isPresent(), "sizeInBytes should be present"); - assertTrue(stats.sizeInBytes().getAsLong() > 0, "sizeInBytes should be positive"); - }); - }); + assertTrue(stats.numRows().isPresent(), "numRows should be present"); + assertEquals(999L, stats.numRows().getAsLong(), "numRows should come from catalog statistics"); + assertFalse(getPlanned(scan), "catalog full statistics should not plan Delta files"); } @Test @@ -1570,28 +1370,21 @@ public void testEstimateStatisticsWithoutCatalogStats(@TempDir File tempDir) thr "CREATE TABLE %s (id INT, name STRING) USING delta LOCATION '%s'", tblName, path)); spark.sql(String.format("INSERT INTO %s VALUES (1, 'a')", tblName)); - withSQLConf( - "spark.sql.cbo.enabled", - "true", - () -> { - // Path-based table - no catalog table, no ANALYZE TABLE stats - Identifier id = Identifier.of(new String[] {"default"}, tblName); - DeltaV2Table sparkTable = new DeltaV2Table(id, path); + // Path-based table - no catalog table, no ANALYZE TABLE stats + Identifier id = Identifier.of(new String[] {"default"}, tblName); + DeltaV2Table sparkTable = new DeltaV2Table(id, path); - DeltaV2ScanBuilder builder = - (DeltaV2ScanBuilder) - sparkTable.newScanBuilder(new CaseInsensitiveStringMap(new HashMap<>())); - DeltaV2Scan scan = (DeltaV2Scan) builder.build(); - Statistics stats = scan.estimateStatistics(); + DeltaV2ScanBuilder builder = + (DeltaV2ScanBuilder) + sparkTable.newScanBuilder(new CaseInsensitiveStringMap(new HashMap<>())); + DeltaV2Scan scan = (DeltaV2Scan) builder.build(); + Statistics stats = scan.estimateStatistics(); - // Per-file Delta stats (numRecords in the transaction log) are available even without - // catalog stats from ANALYZE TABLE, so numRows should be present. - assertTrue(stats.numRows().isPresent(), "numRows should be present from per-file stats"); - assertEquals( - 1L, stats.numRows().getAsLong(), "numRows should match the inserted row count"); - assertTrue(stats.sizeInBytes().isPresent(), "sizeInBytes should be present"); - assertTrue(stats.sizeInBytes().getAsLong() > 0, "sizeInBytes should be positive"); - }); + assertFalse( + stats.numRows().isPresent(), + "path-based full statistics should not expose per-file row counts"); + assertTrue(stats.sizeInBytes().isPresent(), "sizeInBytes should be present"); + assertTrue(stats.sizeInBytes().getAsLong() > 0, "sizeInBytes should be positive"); } @Test @@ -1639,40 +1432,35 @@ public void testEstimateStatisticsWithPartitionedTableAndCatalogStats(@TempDir F CatalogTable catalogTable = injectCatalogStats(tblName, catalogStats); - withSQLConf( - "spark.sql.cbo.enabled", - "true", - () -> { - Identifier id = Identifier.of(new String[] {"default"}, tblName); - DeltaV2Table sparkTable = new DeltaV2Table(id, catalogTable, Collections.emptyMap()); + Identifier id = Identifier.of(new String[] {"default"}, tblName); + DeltaV2Table sparkTable = new DeltaV2Table(id, catalogTable, Collections.emptyMap()); - DeltaV2ScanBuilder builder = - (DeltaV2ScanBuilder) - sparkTable.newScanBuilder(new CaseInsensitiveStringMap(new HashMap<>())); - DeltaV2Scan scan = (DeltaV2Scan) builder.build(); - Statistics stats = scan.estimateStatistics(); + DeltaV2ScanBuilder builder = + (DeltaV2ScanBuilder) + sparkTable.newScanBuilder(new CaseInsensitiveStringMap(new HashMap<>())); + DeltaV2Scan scan = (DeltaV2Scan) builder.build(); + Statistics stats = scan.estimateStatistics(); - assertTrue(stats.numRows().isPresent(), "numRows should be present"); - assertEquals(3L, stats.numRows().getAsLong(), "numRows should be 3"); + assertTrue(stats.numRows().isPresent(), "numRows should be present"); + assertEquals(3L, stats.numRows().getAsLong(), "numRows should be 3"); - // Verify column stats include both data and partition columns - Map colStats = stats.columnStats(); - assertNotNull(colStats.get(FieldReference.apply("id")), "id stats should be present"); - assertNotNull(colStats.get(FieldReference.apply("part")), "part stats should be present"); + // Verify column stats include both data and partition columns + Map colStats = stats.columnStats(); + assertNotNull(colStats.get(FieldReference.apply("id")), "id stats should be present"); + assertNotNull(colStats.get(FieldReference.apply("part")), "part stats should be present"); - // Check partition column stats - ColumnStatistics partStats = colStats.get(FieldReference.apply("part")); - assertTrue(partStats.min().isPresent(), "part min should be present"); - assertTrue(partStats.max().isPresent(), "part max should be present"); - assertEquals(1, partStats.min().get(), "part min should be 1"); - assertEquals(2, partStats.max().get(), "part max should be 2"); - }); + ColumnStatistics partStats = colStats.get(FieldReference.apply("part")); + assertTrue(partStats.min().isPresent(), "part min should be present"); + assertTrue(partStats.max().isPresent(), "part max should be present"); + assertEquals(1, partStats.min().get(), "part min should be 1"); + assertEquals(2, partStats.max().get(), "part max should be 2"); } @Test - public void testEstimatedSizeUsesAvgLenFromCatalogStats(@TempDir File tempDir) throws Exception { - // Verify that computeEstimatedSizeWithColumnProjection uses avgLen from catalog stats - // instead of defaultSize(), mirroring EstimationUtils.getSizePerRow() (#5952). + public void testSizeOnlyEstimateDoesNotUseCatalogColumnStats(@TempDir File tempDir) + throws Exception { + // The size-only API should return the catalog size directly without consulting column stats or + // planning Delta files. String path = tempDir.getAbsolutePath(); String tblName = "stats_avglen"; spark.sql( @@ -1700,101 +1488,36 @@ public void testEstimatedSizeUsesAvgLenFromCatalogStats(@TempDir File tempDir) t CatalogTable catalogTable = injectCatalogStats(tblName, catalogStats); - // avgLen is used for sizeInBytes estimation regardless of CBO/planStats settings - withSQLConf( - "spark.sql.cbo.enabled", - "false", - () -> { - withSQLConf( - "spark.sql.cbo.planStats.enabled", - "false", - () -> { - Identifier id = Identifier.of(new String[] {"default"}, tblName); - DeltaV2Table sparkTable = - new DeltaV2Table(id, catalogTable, Collections.emptyMap()); - - DeltaV2ScanBuilder builder = - (DeltaV2ScanBuilder) - sparkTable.newScanBuilder(new CaseInsensitiveStringMap(new HashMap<>())); - - // Prune to only 'name' column to trigger column projection estimation - StructType prunedSchema = new StructType().add("name", DataTypes.StringType); - builder.pruneColumns(prunedSchema); - - DeltaV2Scan scan = (DeltaV2Scan) builder.build(); - - long totalBytes = getTotalBytes(scan); - long estimatedSize = getEstimatedSizeInBytes(scan); - assertTrue(totalBytes > 0, "totalBytes should be positive"); - - // Schema: dataSchema = (id INT, name STRING), partitionSchema = empty - // readSchema = (name STRING) after pruning - // - // With avgLen=5 for name (STRING: avgLen + 12 = 17, vs defaultSize 20): - // fullSchemaRowSize = 8 + (4 [INT default] + 17 [STRING avgLen]) = 29 - // outputRowSize = 8 + 17 = 25 - // estimated = (totalBytes * 25) / 29 - // - // Without avgLen (defaultSize only): - // fullSchemaRowSize = 8 + (4 + 20) = 32 - // outputRowSize = 8 + 20 = 28 - // estimated = (totalBytes * 28) / 32 - long expectedWithAvgLen = (totalBytes * 25) / 29; - long expectedWithoutAvgLen = (totalBytes * 28) / 32; - - assertEquals( - expectedWithAvgLen, - estimatedSize, - "estimatedSize should use avgLen from catalog stats"); - assertNotEquals( - expectedWithoutAvgLen, - estimatedSize, - "estimatedSize should differ from defaultSize-only calculation"); - }); - }); - } + Identifier id = Identifier.of(new String[] {"default"}, tblName); + DeltaV2Table sparkTable = new DeltaV2Table(id, catalogTable, Collections.emptyMap()); + DeltaV2ScanBuilder builder = + (DeltaV2ScanBuilder) + sparkTable.newScanBuilder(new CaseInsensitiveStringMap(new HashMap<>())); - @Test - public void testEstimateStatisticsWithoutAnalyze(@TempDir File tempDir) throws Exception { - // Table exists in catalog but no stats were injected - String path = tempDir.getAbsolutePath(); - String tblName = "stats_no_analyze"; - spark.sql( - String.format( - "CREATE TABLE %s (id INT, name STRING) USING delta LOCATION '%s'", tblName, path)); - spark.sql(String.format("INSERT INTO %s VALUES (1, 'a')", tblName)); - - withSQLConf( - "spark.sql.cbo.enabled", - "true", - () -> { - CatalogTable catalogTable = - spark.sessionState().catalog().getTableMetadata(new TableIdentifier(tblName)); - Identifier id = Identifier.of(new String[] {"default"}, tblName); - DeltaV2Table sparkTable = new DeltaV2Table(id, catalogTable, Collections.emptyMap()); + // Pruning must not affect the catalog size fast path. + StructType prunedSchema = new StructType().add("name", DataTypes.StringType); + builder.pruneColumns(prunedSchema); - DeltaV2ScanBuilder builder = - (DeltaV2ScanBuilder) - sparkTable.newScanBuilder(new CaseInsensitiveStringMap(new HashMap<>())); - DeltaV2Scan scan = (DeltaV2Scan) builder.build(); - Statistics stats = scan.estimateStatistics(); + DeltaV2Scan scan = (DeltaV2Scan) builder.build(); + assertFalse(getPlanned(scan)); + assertEquals(OptionalLong.of(1024L), scan.estimateSizeInBytes()); + assertFalse(getPlanned(scan), "catalog size lookup should not plan Delta files"); - // Per-file Delta stats (numRecords in the transaction log) provide numRows even when - // ANALYZE TABLE has not been run and no catalog stats exist. - assertTrue(stats.numRows().isPresent(), "numRows should be present from per-file stats"); - assertEquals( - 1L, stats.numRows().getAsLong(), "numRows should match the inserted row count"); - assertTrue(stats.sizeInBytes().isPresent(), "sizeInBytes should be present"); - assertTrue(stats.sizeInBytes().getAsLong() > 0, "sizeInBytes should be positive"); - }); + Statistics stats = scan.estimateStatistics(); + assertFalse(stats.sizeInBytes().isPresent()); + assertEquals(OptionalLong.of(3L), stats.numRows()); + assertTrue( + stats.columnStats().containsKey(FieldReference.apply("name")), + "full statistics should preserve catalog column statistics"); + assertFalse(getPlanned(scan), "catalog full statistics should not plan Delta files"); } @Test public void testEstimateStatisticsWithCatalogStats_noNumRows(@TempDir File tempDir) throws Exception { - // Catalog stats with sizeInBytes but no numRows: when per-file stats are available, - // numRows should still be reported from per-file numRecords (not from catalog stats). - // sizeInBytes should come from planned files, not the stale catalog value. + // Catalog stats with sizeInBytes but no numRows: the size-only API should use the catalog + // size without planning files, while full statistics should use the planned file size and + // leave row and column statistics empty. String path = tempDir.getAbsolutePath(); String tblName = "stats_no_numrows"; spark.sql( @@ -1811,37 +1534,35 @@ public void testEstimateStatisticsWithCatalogStats_noNumRows(@TempDir File tempD CatalogTable catalogTable = injectCatalogStats(tblName, catalogStats); - withSQLConf( - "spark.sql.cbo.enabled", - "true", - () -> { - Identifier id = Identifier.of(new String[] {"default"}, tblName); - DeltaV2Table sparkTable = new DeltaV2Table(id, catalogTable, Collections.emptyMap()); - - DeltaV2ScanBuilder builder = - (DeltaV2ScanBuilder) - sparkTable.newScanBuilder(new CaseInsensitiveStringMap(new HashMap<>())); - DeltaV2Scan scan = (DeltaV2Scan) builder.build(); - Statistics stats = scan.estimateStatistics(); - - // Per-file stats provide numRows even when catalog stats lack it - assertTrue(stats.numRows().isPresent(), "numRows should be present from per-file stats"); - assertEquals( - 2L, stats.numRows().getAsLong(), "numRows should reflect per-file row count"); - assertTrue(stats.sizeInBytes().isPresent(), "sizeInBytes should be present"); - assertTrue(stats.sizeInBytes().getAsLong() > 0, "sizeInBytes should be positive"); + Identifier id = Identifier.of(new String[] {"default"}, tblName); + DeltaV2Table sparkTable = new DeltaV2Table(id, catalogTable, Collections.emptyMap()); - // sizeInBytes should NOT be the catalog sizeInBytes (99999) - assertNotEquals( - 99999L, - stats.sizeInBytes().getAsLong(), - "sizeInBytes should come from planned files, not catalog stats"); + DeltaV2ScanBuilder builder = + (DeltaV2ScanBuilder) + sparkTable.newScanBuilder(new CaseInsensitiveStringMap(new HashMap<>())); + DeltaV2Scan scan = (DeltaV2Scan) builder.build(); - // columnStats should be empty (catalog stats had no column stats) - assertTrue( - stats.columnStats().isEmpty(), - "columnStats should be empty when catalog stats have no column stats"); - }); + assertFalse(getPlanned(scan)); + assertEquals( + OptionalLong.of(99999L), + scan.estimateSizeInBytes(), + "size-only statistics should use the catalog size"); + assertFalse(getPlanned(scan), "reading the catalog size should not plan Delta files"); + + Statistics stats = scan.estimateStatistics(); + + // A catalog entry without rowCount stays on the size-only branch. + assertFalse(stats.numRows().isPresent(), "numRows should be absent without catalog rows"); + assertTrue(stats.sizeInBytes().isPresent(), "sizeInBytes should be present"); + assertTrue(stats.sizeInBytes().getAsLong() > 0, "sizeInBytes should be positive"); + + assertNotEquals( + 99999L, + stats.sizeInBytes().getAsLong(), + "sizeInBytes should come from planned files, not catalog stats"); + assertTrue( + stats.columnStats().isEmpty(), + "columnStats should be empty when catalog stats have no column stats"); } @SuppressWarnings({"rawtypes", "unchecked"}) @@ -1942,9 +1663,8 @@ public void testLimitPushdown_getterReflectsBuilder() { @Test public void testLimitPushdown_limit0ReportsZeroNumRowsNotStaleCatalog(@TempDir File tempDir) throws Exception { - // The LIMIT 0 short-circuit reads no files. Under CBO with catalog stats present, numRows() - // must report 0 (the scan logically returns zero rows), NOT the stale table-level catalog - // count. This adds regression guard for the short-circuit setting rowCountKnown/totalRows. + // The LIMIT 0 short-circuit reads no files and reports exact zero statistics instead of stale + // table-level catalog values. String path = tempDir.getAbsolutePath(); String tblName = "stats_limit0_numrows"; spark.sql( @@ -1972,7 +1692,7 @@ public void testLimitPushdown_limit0ReportsZeroNumRowsNotStaleCatalog(@TempDir F builder.pushLimit(0); DeltaV2Scan scan = (DeltaV2Scan) builder.build(); - assertTrue(isRowCountKnown(scan), "LIMIT 0 under CBO should mark the row count known"); + assertFalse(getPlanned(scan), "LIMIT 0 statistics should not plan Delta files"); Statistics stats = scan.estimateStatistics(); assertTrue(stats.numRows().isPresent(), "numRows should be present under CBO"); assertEquals( @@ -1981,6 +1701,80 @@ public void testLimitPushdown_limit0ReportsZeroNumRowsNotStaleCatalog(@TempDir F "LIMIT 0 must report 0 rows, not the stale catalog count (999)"); assertEquals( 0L, getTotalBytes(scan), "LIMIT 0 should read zero bytes even with catalog stats"); + assertFalse(getPlanned(scan), "LIMIT 0 statistics should remain on the fast path"); + }); + } + + @Test + public void testLimitPushdown_numRowsRemainsInternalUnderPlanStats() throws Exception { + // Delta retains aggregate row counts for its internal scan statistics, but the + // generic SupportsReportStatistics handoff intentionally exposes only scan-local size when + // catalog row statistics are absent. + withSQLConf( + "spark.sql.cbo.planStats.enabled", + "true", + () -> { + // Without a limit, retaining numRecords does not populate an aggregate row count. + DeltaV2Scan noLimit = + (DeltaV2Scan) ((DeltaV2ScanBuilder) table.newScanBuilder(options)).build(); + assertFalse(isRowCountKnown(noLimit), "missing aggregate row count should stay unknown"); + assertFalse(noLimit.estimateStatistics().numRows().isPresent()); + + // With a pushed limit, the per-file counts are gone but the aggregate still reports the + // rows the limited scan will read. + DeltaV2Scan withLimit = buildScanWithLimit(3); + assertTrue( + isRowCountKnown(withLimit), + "row count must stay known for a pushed limit (DeltaScan aggregate)"); + assertEquals(3L, getTotalRows(withLimit)); + assertFalse( + withLimit.estimateStatistics().numRows().isPresent(), + "generic relation statistics must not expose the internal limited row count"); + }); + } + + @Test + public void testLimitPushdown_numRowsEmptyWhenPlanStatsDisabled() throws Exception { + // Scan metadata preserves the aggregate independently of CBO/planStats. The generic + // optimizer statistics still do not expose it as numRows for a path-based table. + withSQLConf( + "spark.sql.cbo.enabled", + "false", + () -> { + withSQLConf( + "spark.sql.cbo.planStats.enabled", + "false", + () -> { + DeltaV2Scan scan = buildScanWithLimit(3); + assertTrue( + isRowCountKnown(scan), + "scan metadata should retain the row count collected by limit selection"); + assertEquals(3L, getTotalRows(scan)); + assertFalse( + scan.estimateStatistics().numRows().isPresent(), + "numRows must be empty when CBO and planStats are both disabled"); + }); + }); + } + + @Test + public void testLimitPushdown_numRowsInvalidatedByRuntimeFiltering() throws Exception { + // An aggregate row count describes the pre-prune file set. Runtime partition filtering has no + // per-file breakdown to re-derive from, so the count must become unknown rather than overstate + // what the pruned scan reads. (Without a limit the per-file counts survive pruning -- see + // testNumRowsAfterRuntimeFiltering.) + withSQLConf( + "spark.sql.cbo.planStats.enabled", + "true", + () -> { + DeltaV2Scan scan = buildScanWithLimit(3); + assertTrue(isRowCountKnown(scan), "precondition: count known before runtime filtering"); + + scan.filter(new Predicate[] {cityPredicate}); // city=hz prunes files + + assertFalse( + scan.estimateStatistics().numRows().isPresent(), + "an aggregate-derived count must not survive runtime file pruning"); }); } diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/MetadataEvolutionHandlerTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/MetadataEvolutionHandlerTest.java index 9c5aee508de..29a77348264 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/MetadataEvolutionHandlerTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/MetadataEvolutionHandlerTest.java @@ -18,7 +18,6 @@ import static org.junit.jupiter.api.Assertions.*; import io.delta.kernel.CommitRange; -import io.delta.kernel.Snapshot; import io.delta.kernel.data.ArrayValue; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.engine.Engine; @@ -38,18 +37,20 @@ import io.delta.spark.internal.v2.adapters.KernelMetadataAdapter; import io.delta.spark.internal.v2.adapters.KernelProtocolAdapter; import io.delta.spark.internal.v2.exception.VersionNotFoundException; -import io.delta.spark.internal.v2.snapshot.DeltaSnapshotManager; import io.delta.spark.internal.v2.snapshot.PathBasedSnapshotManager; import java.io.File; import java.util.*; import java.util.stream.Stream; import org.apache.spark.sql.delta.DeltaOptions; import org.apache.spark.sql.delta.DeltaRuntimeException; +import org.apache.spark.sql.delta.Snapshot; import org.apache.spark.sql.delta.sources.DeltaSQLConf; import org.apache.spark.sql.delta.sources.DeltaSourceMetadataTrackingLog; import org.apache.spark.sql.delta.sources.DeltaSourceOffset; import org.apache.spark.sql.delta.sources.DeltaStreamUtils; import org.apache.spark.sql.delta.sources.PersistedMetadata; +import org.apache.spark.sql.delta.v2.interop.DeltaV2Snapshot$; +import org.apache.spark.sql.delta.v2.interop.DeltaV2SnapshotManager; import org.apache.spark.sql.util.CaseInsensitiveStringMap; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -111,8 +112,8 @@ public ColumnVector getElements() { * A no-op snapshot manager that throws on any call. Used for tests that only exercise handler * logic (tracking state, offset arithmetic, iterator manipulation) without hitting the delta log. */ - private static final DeltaSnapshotManager THROWING_SNAPSHOT_MANAGER = - new DeltaSnapshotManager() { + private static final DeltaV2SnapshotManager THROWING_SNAPSHOT_MANAGER = + new DeltaV2SnapshotManager() { @Override public Snapshot loadLatestSnapshot() { throw new UnsupportedOperationException("not expected in this test"); @@ -197,7 +198,8 @@ private HandlerWithLog buildHandlerWithRealTable( String tablePath, long initVersion, boolean seedLogWithInitEntry) { PathBasedSnapshotManager snapshotManager = new PathBasedSnapshotManager(tablePath, spark.sessionState().newHadoopConf()); - SnapshotImpl snapshot = (SnapshotImpl) snapshotManager.loadSnapshotAt(initVersion); + SnapshotImpl snapshot = + DeltaV2Snapshot$.MODULE$.getKernelSnapshot(snapshotManager.loadSnapshotAt(initVersion)); Metadata tableMetadata = snapshot.getMetadata(); Protocol tableProtocol = snapshot.getProtocol(); KernelMetadataAdapter adapter = new KernelMetadataAdapter(tableMetadata); @@ -996,7 +998,7 @@ private Option invokeGetTrackingLog( createEmptyTestTable(tablePath, tableName); PathBasedSnapshotManager snapshotManager = new PathBasedSnapshotManager(tablePath, spark.sessionState().newHadoopConf()); - SnapshotImpl snapshot = (SnapshotImpl) snapshotManager.loadLatestSnapshot(); + Snapshot snapshot = snapshotManager.loadLatestSnapshot(); return MetadataEvolutionHandler.getMetadataTrackingLogForMicroBatchStream( spark, snapshot, @@ -1136,7 +1138,7 @@ private Optional invokeGetPersistedMetadata( createEmptyTestTable(tablePath, tableName); PathBasedSnapshotManager snapshotManager = new PathBasedSnapshotManager(tablePath, spark.sessionState().newHadoopConf()); - SnapshotImpl snapshot = (SnapshotImpl) snapshotManager.loadLatestSnapshot(); + Snapshot snapshot = snapshotManager.loadLatestSnapshot(); return MetadataEvolutionHandler.getPersistedMetadataForMicroBatchStream( spark, snapshot, options, snapshotManager, defaultEngine); } @@ -1158,7 +1160,8 @@ public void testGetPersistedMetadata_returnsSeededEntry(@TempDir File tempDir) t createEmptyTestTable(tablePath, tableName); PathBasedSnapshotManager snapshotManager = new PathBasedSnapshotManager(tablePath, spark.sessionState().newHadoopConf()); - SnapshotImpl snapshot = (SnapshotImpl) snapshotManager.loadLatestSnapshot(); + Snapshot dsv1Snapshot = snapshotManager.loadLatestSnapshot(); + SnapshotImpl snapshot = DeltaV2Snapshot$.MODULE$.getKernelSnapshot(dsv1Snapshot); String schemaLogPath = new File(tempDir, "schema_log").getAbsolutePath(); Map options = new HashMap<>(); @@ -1175,7 +1178,7 @@ public void testGetPersistedMetadata_returnsSeededEntry(@TempDir File tempDir) t DeltaSourceMetadataTrackingLog trackingLog = MetadataEvolutionHandler.getMetadataTrackingLogForMicroBatchStream( spark, - snapshot, + dsv1Snapshot, options, snapshotManager, defaultEngine, @@ -1195,7 +1198,7 @@ public void testGetPersistedMetadata_returnsSeededEntry(@TempDir File tempDir) t Optional result = MetadataEvolutionHandler.getPersistedMetadataForMicroBatchStream( - spark, snapshot, options, snapshotManager, defaultEngine); + spark, dsv1Snapshot, options, snapshotManager, defaultEngine); assertTrue(result.isPresent()); assertEquals(seededVersion, result.get().deltaCommitVersion()); }); @@ -1212,7 +1215,7 @@ private SnapshotImpl loadSourceSnapshot(File tempDir) { createEmptyTestTable(tablePath, tableName); PathBasedSnapshotManager snapshotManager = new PathBasedSnapshotManager(tablePath, spark.sessionState().newHadoopConf()); - return (SnapshotImpl) snapshotManager.loadLatestSnapshot(); + return DeltaV2Snapshot$.MODULE$.getKernelSnapshot(snapshotManager.loadLatestSnapshot()); } /** Persisted metadata with schema, protocol, and configuration distinct from the source. */ @@ -1348,7 +1351,8 @@ public void testBuildReadSnapshot_trapsLogReplayAccess(@TempDir File tempDir) { /** Builds a {@link PersistedMetadata} snapshot of the table state at v0. */ private PersistedMetadata buildCurrentMetadataAtV0( String tablePath, PathBasedSnapshotManager snapshotManager) { - SnapshotImpl v0 = (SnapshotImpl) snapshotManager.loadSnapshotAt(0L); + SnapshotImpl v0 = + DeltaV2Snapshot$.MODULE$.getKernelSnapshot(snapshotManager.loadSnapshotAt(0L)); return PersistedMetadata.apply( "test-table-id", 0L, @@ -1408,6 +1412,23 @@ private static Stream mergerTestCases() { 1L)); } + @Test + public void testSchemaJsonMetadataKeyOrderDoesNotAffectParsedSchema() { + String idBeforePhysicalName = + "{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"integer\"," + + "\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":1," + + "\"delta.columnMapping.physicalName\":\"col-id\"}}]}"; + String physicalNameBeforeId = + "{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"integer\"," + + "\"nullable\":true,\"metadata\":{\"delta.columnMapping.physicalName\":\"col-id\"," + + "\"delta.columnMapping.id\":1}}]}"; + + assertNotEquals(idBeforePhysicalName, physicalNameBeforeId); + assertEquals( + org.apache.spark.sql.types.DataType.fromJson(idBeforePhysicalName), + org.apache.spark.sql.types.DataType.fromJson(physicalNameBeforeId)); + } + @ParameterizedTest(name = "{0}") @MethodSource("mergerTestCases") public void testGetMergedConsecutive( @@ -1437,10 +1458,11 @@ public void testGetMergedConsecutive( // Verify the merger's result against the table's actual schema and protocol at the merged // version. We build the "expected" PersistedMetadata directly from the snapshot at that - // version — if the merger captured the right metadata/protocol actions, the JSONs will match. + // version. Compare parsed schemas because JSON object key ordering is not significant. assertTrue(result.isDefined()); SnapshotImpl mergedSnapshot = - (SnapshotImpl) snapshotManager.loadSnapshotAt(expectedMergedVersion); + DeltaV2Snapshot$.MODULE$.getKernelSnapshot( + snapshotManager.loadSnapshotAt(expectedMergedVersion)); PersistedMetadata expected = PersistedMetadata.apply( "test-table-id", @@ -1449,8 +1471,8 @@ public void testGetMergedConsecutive( new KernelProtocolAdapter(mergedSnapshot.getProtocol()), tablePath + "/_delta_log/_streaming_metadata"); - assertEquals(expected.dataSchemaJson(), result.get().dataSchemaJson()); - assertEquals(expected.partitionSchemaJson(), result.get().partitionSchemaJson()); + assertEquals(expected.dataSchema(), result.get().dataSchema()); + assertEquals(expected.partitionSchema(), result.get().partitionSchema()); assertEquals(expected.protocolJson(), result.get().protocolJson()); } @@ -1481,7 +1503,8 @@ public void testGetMergedConsecutive_stopsAtCdfUpdateCommit(@TempDir File tempDi new PathBasedSnapshotManager(tablePath, spark.sessionState().newHadoopConf()); // current = v2 (the ALTER ADD COLUMN). Merger walks forward from here. - SnapshotImpl v2Snapshot = (SnapshotImpl) snapshotManager.loadSnapshotAt(2L); + SnapshotImpl v2Snapshot = + DeltaV2Snapshot$.MODULE$.getKernelSnapshot(snapshotManager.loadSnapshotAt(2L)); PersistedMetadata current = PersistedMetadata.apply( "test-table-id", @@ -1498,7 +1521,9 @@ public void testGetMergedConsecutive_stopsAtCdfUpdateCommit(@TempDir File tempDi assertTrue(result.isDefined()); assertEquals(3L, result.get().deltaCommitVersion()); - SnapshotImpl v3Snapshot = (SnapshotImpl) snapshotManager.loadSnapshotAt(3L); - assertEquals(v3Snapshot.getMetadata().getSchemaString(), result.get().dataSchemaJson()); + SnapshotImpl v3Snapshot = + DeltaV2Snapshot$.MODULE$.getKernelSnapshot(snapshotManager.loadSnapshotAt(3L)); + assertEquals( + new KernelMetadataAdapter(v3Snapshot.getMetadata()).schema(), result.get().dataSchema()); } } diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/ScanFileRDDTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/ScanFileRDDTest.java index 4a0e718eb2e..cc759a0a423 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/ScanFileRDDTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/ScanFileRDDTest.java @@ -28,6 +28,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; +import org.apache.spark.sql.delta.v2.interop.DeltaV2Snapshot$; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -42,7 +43,7 @@ public void testScanFileRDDProducesAddFileRows() { Configuration hadoopConf = spark.sessionState().newHadoopConf(); PathBasedSnapshotManager mgr = new PathBasedSnapshotManager(tablePath, hadoopConf); - SnapshotImpl snapshot = (SnapshotImpl) mgr.loadLatestSnapshot(); + SnapshotImpl snapshot = DeltaV2Snapshot$.MODULE$.getKernelSnapshot(mgr.loadLatestSnapshot()); SerializableReadOnlySnapshot serializable = SerializableReadOnlySnapshot.fromSnapshot(snapshot, hadoopConf); @@ -67,7 +68,7 @@ public void testScanFileRDDSortable() { Configuration hadoopConf = spark.sessionState().newHadoopConf(); PathBasedSnapshotManager mgr = new PathBasedSnapshotManager(tablePath, hadoopConf); - SnapshotImpl snapshot = (SnapshotImpl) mgr.loadLatestSnapshot(); + SnapshotImpl snapshot = DeltaV2Snapshot$.MODULE$.getKernelSnapshot(mgr.loadLatestSnapshot()); SerializableReadOnlySnapshot serializable = SerializableReadOnlySnapshot.fromSnapshot(snapshot, hadoopConf); diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkGoldenTableTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkGoldenTableTest.java index fa17aa4602b..01c74d08951 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkGoldenTableTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkGoldenTableTest.java @@ -19,9 +19,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import io.delta.golden.GoldenTableUtils$; -import io.delta.kernel.expressions.Column; -import io.delta.kernel.expressions.Literal; -import io.delta.kernel.expressions.Predicate; +import io.delta.spark.internal.v2.DeltaV2TestBase; import io.delta.spark.internal.v2.catalog.DeltaV2Table; import java.io.File; import java.lang.reflect.Field; @@ -187,46 +185,23 @@ public void testDsv2Internal() throws Exception { verifyHadoopConf(sparkScan2.getConfiguration()); // check SupportsPushDownFilters - // case 1: mix of supported and unsupported, data and partition filters + // case 1: mix of data and partition filters checkSupportsPushDownFilters( table, scanOptions, // input filters - new Filter[] { - new GreaterThan("cnt", 10), // supported data filter - new StringStartsWith("name", "foo"), // supported data filter - new EqualTo("date", "2025-09-01"), // supported partition filter - new StringEndsWith("city", "York"), // unsupported partition filter - }, - // expected post-scan filters new Filter[] { new GreaterThan("cnt", 10), new StringStartsWith("name", "foo"), + new EqualTo("date", "2025-09-01"), new StringEndsWith("city", "York"), }, - // expected pushed filters + // expected partition filters new Filter[] { - new GreaterThan("cnt", 10), - new StringStartsWith("name", "foo"), - new EqualTo("date", "2025-09-01") + new EqualTo("date", "2025-09-01"), new StringEndsWith("city", "York"), }, - // expected pushed kernel predicates - new Predicate[] { - new Predicate(">", new Column("cnt"), Literal.ofInt(10)), - new Predicate("STARTS_WITH", new Column("name"), Literal.ofString("foo")), - new Predicate("=", new Column("date"), Literal.ofString("2025-09-01")) - }, - // expected data filters - new Filter[] {new GreaterThan("cnt", 10), new StringStartsWith("name", "foo")}, - // expected kernel scan builder predicate - Optional.of( - new Predicate( - "AND", - new Predicate( - "AND", - new Predicate(">", new Column("cnt"), Literal.ofInt(10)), - new Predicate("STARTS_WITH", new Column("name"), Literal.ofString("foo"))), - new Predicate("=", new Column("date"), Literal.ofString("2025-09-01"))))); + // expected data filters (and post-scan filters) + new Filter[] {new GreaterThan("cnt", 10), new StringStartsWith("name", "foo")}); // case 2: OR and NOT filters checkSupportsPushDownFilters( @@ -239,84 +214,15 @@ public void testDsv2Internal() throws Exception { new Not(new And(new GreaterThan("cnt", 100), new EqualTo("date", "2025-09-01"))), new Not(new Or(new EqualTo("name", "foo"), new StringStartsWith("city", "New"))) }, - // expected post-scan filters - new Filter[] { - new Or(new GreaterThan("cnt", 10), new StringStartsWith("name", "foo")), - new Or(new EqualTo("cnt", 50), new EqualTo("date", "2025-10-01")), - new Not(new And(new GreaterThan("cnt", 100), new EqualTo("date", "2025-09-01"))), - new Not(new Or(new EqualTo("name", "foo"), new StringStartsWith("city", "New"))) - }, - // expected pushed filters + // expected partition filters + new Filter[0], + // expected data filters (and post-scan filters) new Filter[] { new Or(new GreaterThan("cnt", 10), new StringStartsWith("name", "foo")), new Or(new EqualTo("cnt", 50), new EqualTo("date", "2025-10-01")), new Not(new And(new GreaterThan("cnt", 100), new EqualTo("date", "2025-09-01"))), new Not(new Or(new EqualTo("name", "foo"), new StringStartsWith("city", "New"))) - }, - // expected pushed kernel predicates - new Predicate[] { - new Predicate( - "OR", - new Predicate(">", new Column("cnt"), Literal.ofInt(10)), - new Predicate("STARTS_WITH", new Column("name"), Literal.ofString("foo"))), - new Predicate( - "OR", - new Predicate("=", new Column("cnt"), Literal.ofInt(50)), - new Predicate("=", new Column("date"), Literal.ofString("2025-10-01"))), - new Predicate( - "NOT", - new Predicate( - "AND", - new Predicate(">", new Column("cnt"), Literal.ofInt(100)), - new Predicate("=", new Column("date"), Literal.ofString("2025-09-01")))), - new Predicate( - "NOT", - new Predicate( - "OR", - new Predicate("=", new Column("name"), Literal.ofString("foo")), - new Predicate("STARTS_WITH", new Column("city"), Literal.ofString("New")))) - }, - // expected data filters - new Filter[] { - new Or(new GreaterThan("cnt", 10), new StringStartsWith("name", "foo")), - new Or(new EqualTo("cnt", 50), new EqualTo("date", "2025-10-01")), - new Not(new And(new GreaterThan("cnt", 100), new EqualTo("date", "2025-09-01"))), - new Not(new Or(new EqualTo("name", "foo"), new StringStartsWith("city", "New"))) - }, - // expected kernel scan builder predicate - // reduce(And::new) over 4 predicates gives left-associative nesting: - // AND(AND(AND(pred1, pred2), pred3), pred4) - Optional.of( - new Predicate( - "AND", - new Predicate( - "AND", - new Predicate( - "AND", - new Predicate( - "OR", - new Predicate(">", new Column("cnt"), Literal.ofInt(10)), - new Predicate( - "STARTS_WITH", new Column("name"), Literal.ofString("foo"))), - new Predicate( - "OR", - new Predicate("=", new Column("cnt"), Literal.ofInt(50)), - new Predicate( - "=", new Column("date"), Literal.ofString("2025-10-01")))), - new Predicate( - "NOT", - new Predicate( - "AND", - new Predicate(">", new Column("cnt"), Literal.ofInt(100)), - new Predicate( - "=", new Column("date"), Literal.ofString("2025-09-01"))))), - new Predicate( - "NOT", - new Predicate( - "OR", - new Predicate("=", new Column("name"), Literal.ofString("foo")), - new Predicate( - "STARTS_WITH", new Column("city"), Literal.ofString("New"))))))); + }); // check SupportsRuntimeV2Filtering // city = 'hz' AND date = '20180520' @@ -381,64 +287,74 @@ private void checkSupportsPushDownFilters( DeltaV2Table table, CaseInsensitiveStringMap scanOptions, Filter[] inputFilters, - Filter[] expectedPostScanFilters, - Filter[] expectedPushedFilters, - Predicate[] expectedPushedKernelPredicates, - Filter[] expectedDataFilters, - Optional expectedKernelScanBuilderPredicate) + Filter[] expectedPartitionFilters, + Filter[] expectedDataFilters) throws Exception { ScanBuilder newBuilder = table.newScanBuilder(scanOptions); DeltaV2ScanBuilder builder = (DeltaV2ScanBuilder) newBuilder; - Filter[] postScanFilters = builder.pushFilters(inputFilters); - - assertEquals( - new HashSet<>(Arrays.asList(expectedPostScanFilters)), - new HashSet<>(Arrays.asList(postScanFilters))); + scala.collection.immutable.Seq filters = + scala.jdk.javaapi.CollectionConverters.asScala( + Arrays.asList(toCatalystFilters(builder, inputFilters))) + .toList(); + Object postScanFilters = builder.pushFilters(filters); assertEquals( - new HashSet<>(Arrays.asList(expectedPushedFilters)), - new HashSet<>(Arrays.asList(builder.pushedFilters()))); - - Predicate[] pushedPredicates = getPushedKernelPredicates(builder); + describe(toCatalystFilters(builder, expectedDataFilters)), + describe( + scala.jdk.javaapi.CollectionConverters + .asJava( + (scala.collection.immutable.Seq< + org.apache.spark.sql.catalyst.expressions.Expression>) + postScanFilters) + .toArray(new org.apache.spark.sql.catalyst.expressions.Expression[0]))); assertEquals( - new HashSet<>(Arrays.asList(expectedPushedKernelPredicates)), - new HashSet<>(Arrays.asList(pushedPredicates))); - - Filter[] dataFilters = getDataFilters(builder); + describeUnordered(toCatalystFilters(builder, expectedPartitionFilters)), + describeUnordered(getCatalystFilters(builder, "partitionCatalystFilters"))); assertEquals( - new HashSet<>(Arrays.asList(expectedDataFilters)), - new HashSet<>(Arrays.asList(dataFilters))); + describeUnordered(toCatalystFilters(builder, expectedDataFilters)), + describeUnordered(getCatalystFilters(builder, "dataCatalystFilters"))); + assertEquals(0, builder.pushedFilters().length); + } - Optional predicateOpt = getKernelScanBuilderPredicate(builder); - assertEquals(expectedKernelScanBuilderPredicate, predicateOpt); + private static org.apache.spark.sql.catalyst.expressions.Expression[] toCatalystFilters( + DeltaV2ScanBuilder builder, Filter[] filters) { + org.apache.spark.sql.catalyst.expressions.Expression[] expressions = + new org.apache.spark.sql.catalyst.expressions.Expression[filters.length]; + for (int i = 0; i < filters.length; i++) { + expressions[i] = DeltaV2TestBaseAccess.convert(builder, filters[i]); + } + return expressions; } - private Predicate[] getPushedKernelPredicates(DeltaV2ScanBuilder builder) throws Exception { - Field field = DeltaV2ScanBuilder.class.getDeclaredField("pushedKernelPredicates"); - field.setAccessible(true); - return (Predicate[]) field.get(builder); + private static final class DeltaV2TestBaseAccess extends DeltaV2TestBase { + private static org.apache.spark.sql.catalyst.expressions.Expression convert( + DeltaV2ScanBuilder builder, Filter filter) { + return toCatalystFilter(builder, filter); + } } - private Filter[] getDataFilters(DeltaV2ScanBuilder builder) throws Exception { - Field field = DeltaV2ScanBuilder.class.getDeclaredField("dataFilters"); + private static org.apache.spark.sql.catalyst.expressions.Expression[] getCatalystFilters( + DeltaV2ScanBuilder builder, String fieldName) throws Exception { + Field field = DeltaV2ScanBuilder.class.getDeclaredField(fieldName); field.setAccessible(true); - return (Filter[]) field.get(builder); + return (org.apache.spark.sql.catalyst.expressions.Expression[]) field.get(builder); } - private Optional getKernelScanBuilderPredicate(DeltaV2ScanBuilder builder) - throws Exception { - Field field = DeltaV2ScanBuilder.class.getDeclaredField("kernelScanBuilder"); - field.setAccessible(true); - Object kernelScanBuilder = field.get(builder); - Field predicateField = kernelScanBuilder.getClass().getDeclaredField("predicate"); - predicateField.setAccessible(true); - Object raw = predicateField.get(kernelScanBuilder); - if (raw == null) { - return Optional.empty(); + private static List describe( + org.apache.spark.sql.catalyst.expressions.Expression[] expressions) { + List rendered = new ArrayList<>(); + for (org.apache.spark.sql.catalyst.expressions.Expression expression : expressions) { + rendered.add(expression.toString().replaceAll("#\\d+", "")); } - Optional opt = (Optional) raw; - return opt.map(Predicate.class::cast); + return rendered; + } + + private static List describeUnordered( + org.apache.spark.sql.catalyst.expressions.Expression[] expressions) { + List rendered = describe(expressions); + Collections.sort(rendered); + return rendered; } @Test diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/snapshot/PathBasedSnapshotManagerTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/snapshot/PathBasedSnapshotManagerTest.java index c2bc7fedb85..5c7ab67f4f9 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/snapshot/PathBasedSnapshotManagerTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/snapshot/PathBasedSnapshotManagerTest.java @@ -17,10 +17,10 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import io.delta.kernel.Snapshot; import io.delta.kernel.internal.DeltaHistoryManager; import io.delta.spark.internal.v2.DeltaV2TestBase; import io.delta.spark.internal.v2.exception.VersionNotFoundException; @@ -29,6 +29,7 @@ import java.util.stream.Stream; import org.apache.hadoop.fs.Path; import org.apache.spark.sql.delta.DeltaLog; +import org.apache.spark.sql.delta.Snapshot; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; @@ -54,7 +55,7 @@ public void testUnsafeVolatileSnapshot(@TempDir File tempDir) { spark.sql(String.format("INSERT INTO %s VALUES (4, 'David')", testTableName)); assertEquals(0L, deltaSnapshot.version()); - assertEquals(deltaSnapshot.version(), kernelSnapshot.getVersion()); + assertEquals(deltaSnapshot.version(), kernelSnapshot.version()); } @Test @@ -67,7 +68,7 @@ public void testLoadLatestSnapshot(@TempDir File tempDir) { DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); Snapshot initialSnapshot = snapshotManager.loadLatestSnapshot(); - assertEquals(0L, initialSnapshot.getVersion()); + assertEquals(0L, initialSnapshot.version()); spark.sql(String.format("INSERT INTO %s VALUES (4, 'David')", testTableName)); @@ -77,10 +78,11 @@ public void testLoadLatestSnapshot(@TempDir File tempDir) { org.apache.spark.sql.delta.Snapshot cachedSnapshot = deltaLog.unsafeVolatileSnapshot(); Snapshot kernelcachedSnapshot = snapshotManager.loadLatestSnapshot(); - assertEquals(1L, updatedSnapshot.getVersion()); - assertEquals(deltaSnapshot.version(), updatedSnapshot.getVersion()); - assertEquals(1L, kernelcachedSnapshot.getVersion()); - assertEquals(cachedSnapshot.version(), kernelcachedSnapshot.getVersion()); + assertEquals(1L, updatedSnapshot.version()); + assertEquals(deltaSnapshot.version(), updatedSnapshot.version()); + assertEquals(1L, kernelcachedSnapshot.version()); + assertEquals(cachedSnapshot.version(), kernelcachedSnapshot.version()); + assertNotSame(updatedSnapshot, kernelcachedSnapshot); } @Test @@ -93,7 +95,7 @@ public void testMultipleLoadLatestSnapshot(@TempDir File tempDir) { DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); - assertEquals(0L, snapshotManager.loadLatestSnapshot().getVersion()); + assertEquals(0L, snapshotManager.loadLatestSnapshot().version()); for (int i = 0; i < 3; i++) { spark.sql( @@ -105,7 +107,7 @@ public void testMultipleLoadLatestSnapshot(@TempDir File tempDir) { long expectedVersion = i + 1; assertEquals(expectedVersion, deltaSnapshot.version()); - assertEquals(expectedVersion, kernelSnapshot.getVersion()); + assertEquals(expectedVersion, kernelSnapshot.version()); } } @@ -125,16 +127,16 @@ public void testLoadSnapshotAt(@TempDir File tempDir) { // Load specific versions Snapshot snapshot0 = snapshotManager.loadSnapshotAt(0L); - assertEquals(0L, snapshot0.getVersion()); + assertEquals(0L, snapshot0.version()); Snapshot snapshot1 = snapshotManager.loadSnapshotAt(1L); - assertEquals(1L, snapshot1.getVersion()); + assertEquals(1L, snapshot1.version()); Snapshot snapshot2 = snapshotManager.loadSnapshotAt(2L); - assertEquals(2L, snapshot2.getVersion()); + assertEquals(2L, snapshot2.version()); Snapshot snapshot3 = snapshotManager.loadSnapshotAt(3L); - assertEquals(3L, snapshot3.getVersion()); + assertEquals(3L, snapshot3.version()); // Note: loadSnapshotAt does not update the cached snapshot } diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/PartitionUtilsTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/PartitionUtilsTest.java index 7090c59da7c..56b5ac92960 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/PartitionUtilsTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/PartitionUtilsTest.java @@ -268,8 +268,7 @@ public void testCreateDeltaParquetReaderFactory_isCDCRead() { * Read-time CDF calls into PartitionUtils with {@code isWriteTimeCDCRead=false}. The factory is * then a plain Parquet reader factory: PartitionUtils does not augment {@code readDataSchema} * with CDC tail columns and does not wrap the reader with {@code CDCReadFunction}. The tail - * columns are added by {@code DeltaChangelogBatch.CDCPartitionReaderFactory} as per-partition - * constants instead. + * columns are added by the read-time CDF partition reader as per-partition constants instead. */ @Test public void testCreateDeltaParquetReaderFactory_NotWriteTimeCDCRead() { diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/ScalaUtilsTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/ScalaUtilsTest.java index 056fbfbce8b..8385ba65e2b 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/ScalaUtilsTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/ScalaUtilsTest.java @@ -21,7 +21,9 @@ import java.util.Collections; import java.util.Map; +import java.util.OptionalLong; import org.junit.jupiter.api.Test; +import scala.Option; class ScalaUtilsTest { @@ -49,4 +51,13 @@ void testToJavaMap_PopulatedInput_PreservesEntries() { assertEquals(Map.of("foo", "bar"), javaMap, "Scala map entries should be preserved"); } + + @Test + void testToJavaOptionalLong() { + long value = 1234567890123L; + + assertEquals( + OptionalLong.of(value), ScalaUtils.toJavaOptionalLong(Option.apply((Object) value))); + assertEquals(OptionalLong.empty(), ScalaUtils.toJavaOptionalLong(Option.empty())); + } } diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/SerializableReadOnlySnapshotTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/SerializableReadOnlySnapshotTest.java index 95fc5a13ca5..aff51b51dc2 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/SerializableReadOnlySnapshotTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/SerializableReadOnlySnapshotTest.java @@ -34,6 +34,7 @@ import java.util.ArrayList; import java.util.List; import org.apache.hadoop.conf.Configuration; +import org.apache.spark.sql.delta.v2.interop.DeltaV2Snapshot$; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -48,7 +49,7 @@ public void testSerializationRoundTrip() throws Exception { Configuration hadoopConf = spark.sessionState().newHadoopConf(); PathBasedSnapshotManager mgr = new PathBasedSnapshotManager(tablePath, hadoopConf); - SnapshotImpl snapshot = (SnapshotImpl) mgr.loadLatestSnapshot(); + SnapshotImpl snapshot = DeltaV2Snapshot$.MODULE$.getKernelSnapshot(mgr.loadLatestSnapshot()); SerializableReadOnlySnapshot original = SerializableReadOnlySnapshot.fromSnapshot(snapshot, hadoopConf); @@ -68,10 +69,10 @@ public void testSerializationRoundTrip() throws Exception { assertEquals(original.getVersion(), deserialized.getVersion()); - Scan scan = deserialized.toScan(); + Engine engine = DefaultEngine.create(deserialized.getHadoopConf()); + Scan scan = deserialized.toScan(engine); assertNotNull(scan); - Engine engine = DefaultEngine.create(deserialized.getHadoopConf()); List batches = new ArrayList<>(); try (CloseableIterator iter = scan.getScanFiles(engine)) { while (iter.hasNext()) { @@ -88,12 +89,12 @@ public void testToScanReturnsReadOnlyScan() { Configuration hadoopConf = spark.sessionState().newHadoopConf(); PathBasedSnapshotManager mgr = new PathBasedSnapshotManager(tablePath, hadoopConf); - SnapshotImpl snapshot = (SnapshotImpl) mgr.loadLatestSnapshot(); + SnapshotImpl snapshot = DeltaV2Snapshot$.MODULE$.getKernelSnapshot(mgr.loadLatestSnapshot()); SerializableReadOnlySnapshot serializable = SerializableReadOnlySnapshot.fromSnapshot(snapshot, hadoopConf); - Scan scan = serializable.toScan(); + Scan scan = serializable.toScan(defaultEngine); assertNotNull(scan); assertNotNull(scan.getScanState(defaultEngine)); assertNotNull(scan.getRemainingFilter()); diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/StreamingHelperTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/StreamingHelperTest.java index 19aa5107d30..edf0cf97381 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/StreamingHelperTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/StreamingHelperTest.java @@ -20,7 +20,6 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import io.delta.kernel.Snapshot; import io.delta.kernel.internal.DeltaHistoryManager; import io.delta.kernel.internal.actions.Metadata; import io.delta.kernel.internal.actions.Protocol; @@ -38,6 +37,7 @@ import java.util.stream.Stream; import org.apache.hadoop.fs.Path; import org.apache.spark.sql.delta.DeltaLog; +import org.apache.spark.sql.delta.Snapshot; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; @@ -63,7 +63,7 @@ public void testUnsafeVolatileSnapshot(@TempDir File tempDir) { spark.sql(String.format("INSERT INTO %s VALUES (4, 'David')", testTableName)); assertEquals(0L, deltaSnapshot.version()); - assertEquals(deltaSnapshot.version(), kernelSnapshot.getVersion()); + assertEquals(deltaSnapshot.version(), kernelSnapshot.version()); } @Test @@ -76,7 +76,7 @@ public void testLoadLatestSnapshot(@TempDir File tempDir) { DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); Snapshot initialSnapshot = snapshotManager.loadLatestSnapshot(); - assertEquals(0L, initialSnapshot.getVersion()); + assertEquals(0L, initialSnapshot.version()); spark.sql(String.format("INSERT INTO %s VALUES (4, 'David')", testTableName)); @@ -86,10 +86,10 @@ public void testLoadLatestSnapshot(@TempDir File tempDir) { org.apache.spark.sql.delta.Snapshot cachedSnapshot = deltaLog.unsafeVolatileSnapshot(); Snapshot kernelcachedSnapshot = snapshotManager.loadLatestSnapshot(); - assertEquals(1L, updatedSnapshot.getVersion()); - assertEquals(deltaSnapshot.version(), updatedSnapshot.getVersion()); - assertEquals(1L, kernelcachedSnapshot.getVersion()); - assertEquals(cachedSnapshot.version(), kernelcachedSnapshot.getVersion()); + assertEquals(1L, updatedSnapshot.version()); + assertEquals(deltaSnapshot.version(), updatedSnapshot.version()); + assertEquals(1L, kernelcachedSnapshot.version()); + assertEquals(cachedSnapshot.version(), kernelcachedSnapshot.version()); } @Test @@ -102,7 +102,7 @@ public void testMultipleLoadLatestSnapshot(@TempDir File tempDir) { DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); - assertEquals(0L, snapshotManager.loadLatestSnapshot().getVersion()); + assertEquals(0L, snapshotManager.loadLatestSnapshot().version()); for (int i = 0; i < 3; i++) { spark.sql( @@ -114,7 +114,7 @@ public void testMultipleLoadLatestSnapshot(@TempDir File tempDir) { long expectedVersion = i + 1; assertEquals(expectedVersion, deltaSnapshot.version()); - assertEquals(expectedVersion, kernelSnapshot.getVersion()); + assertEquals(expectedVersion, kernelSnapshot.version()); } } diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/write/DeltaRowLevelOperationBuilderTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/write/DeltaRowLevelOperationBuilderTest.java index 9412a16a6dd..fe37f229b25 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/write/DeltaRowLevelOperationBuilderTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/write/DeltaRowLevelOperationBuilderTest.java @@ -17,7 +17,6 @@ import static org.junit.jupiter.api.Assertions.*; -import io.delta.kernel.Snapshot; import io.delta.kernel.defaults.engine.DefaultEngine; import io.delta.kernel.engine.Engine; import io.delta.spark.internal.v2.DeltaV2TestBase; @@ -28,7 +27,6 @@ import java.util.Optional; import org.apache.hadoop.conf.Configuration; import org.apache.spark.sql.connector.catalog.Identifier; -import org.apache.spark.sql.connector.catalog.SupportsRowLevelOperations; import org.apache.spark.sql.connector.expressions.NamedReference; import org.apache.spark.sql.connector.read.Scan; import org.apache.spark.sql.connector.read.ScanBuilder; @@ -36,6 +34,7 @@ import org.apache.spark.sql.connector.write.RowLevelOperation; import org.apache.spark.sql.connector.write.RowLevelOperationBuilder; import org.apache.spark.sql.connector.write.RowLevelOperationInfo; +import org.apache.spark.sql.delta.Snapshot; import org.apache.spark.sql.execution.datasources.FileFormat$; import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.StructType; @@ -45,21 +44,12 @@ public class DeltaRowLevelOperationBuilderTest extends DeltaV2TestBase { - @Test - public void regularTableDoesNotExposeRowLevelOperations(@TempDir File tempDir) { - DeltaV2Table table = createBaseTable(tempDir); - - assertFalse(table instanceof SupportsRowLevelOperations); - } - @Test public void tableExposesRowLevelOperationBuilder(@TempDir File tempDir) { DeltaV2Table table = createRowLevelTable(tempDir); - assertTrue(table instanceof SupportsRowLevelOperations); RowLevelOperationBuilder builder = - ((SupportsRowLevelOperations) table) - .newRowLevelOperationBuilder(testInfo(RowLevelOperation.Command.DELETE)); + table.newRowLevelOperationBuilder(testInfo(RowLevelOperation.Command.DELETE)); assertTrue(builder instanceof DeltaRowLevelOperationBuilder); } @@ -136,18 +126,11 @@ public void constructorRejectsNull(@TempDir File tempDir) { () -> new DeltaRowLevelOperationBuilder(table, engine, hadoopConf, snapshot, null)); } - private DeltaV2Table createBaseTable(File tableDir) { - String path = tableDir.getAbsolutePath(); - String tableName = "delta_row_level_" + System.nanoTime(); - createEmptyTestTable(path, tableName); - return new DeltaV2Table(Identifier.of(new String[] {"default"}, tableName), path); - } - private DeltaV2Table createRowLevelTable(File tableDir) { String path = tableDir.getAbsolutePath(); String tableName = "delta_row_level_" + System.nanoTime(); createEmptyTestTable(path, tableName); - return new RowLevelDeltaV2Table(Identifier.of(new String[] {"default"}, tableName), path); + return new DeltaV2Table(Identifier.of(new String[] {"default"}, tableName), path); } private DeltaRowLevelOperationBuilder createBuilder( @@ -161,13 +144,6 @@ private DeltaRowLevelOperationBuilder createBuilder( table, engine, hadoopConf, snapshot, testInfo(command)); } - private static class RowLevelDeltaV2Table extends DeltaV2Table - implements SupportsRowLevelOperations { - RowLevelDeltaV2Table(Identifier identifier, String tablePath) { - super(identifier, tablePath); - } - } - private static StructType tableSchema() { return new StructType().add("id", DataTypes.IntegerType).add("name", DataTypes.StringType); } diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/write/DeltaV2BatchWriteContextTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/write/DeltaV2BatchWriteContextTest.java index 862ad9b391a..0f8a82c14ff 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/write/DeltaV2BatchWriteContextTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/write/DeltaV2BatchWriteContextTest.java @@ -22,18 +22,19 @@ import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertTrue; -import io.delta.kernel.Snapshot; import io.delta.kernel.TableManager; import io.delta.kernel.defaults.engine.DefaultEngine; import io.delta.kernel.engine.Engine; import io.delta.kernel.utils.CloseableIterable; import io.delta.spark.internal.v2.DeltaV2TestBase; +import io.delta.spark.internal.v2.snapshot.PathBasedSnapshotManager; import io.delta.spark.internal.v2.utils.SchemaUtils; import java.io.File; import java.time.ZoneId; import java.util.Collections; import org.apache.hadoop.conf.Configuration; import org.apache.spark.sql.connector.write.LogicalWriteInfo; +import org.apache.spark.sql.delta.Snapshot; import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.StructType; import org.apache.spark.sql.util.CaseInsensitiveStringMap; @@ -49,7 +50,7 @@ public void createInitializesDriverWriteState(@TempDir File tempDir) throws Exce Configuration hadoopConf = spark.sessionState().newHadoopConf(); Engine engine = DefaultEngine.create(hadoopConf); createKernelTable(path, tableSchema, engine); - Snapshot snapshot = TableManager.loadSnapshot(path).build(engine); + Snapshot snapshot = new PathBasedSnapshotManager(path, engine).loadLatestSnapshot(); DeltaV2BatchWriteContext context = DeltaV2BatchWriteContext.create( @@ -59,7 +60,8 @@ public void createInitializesDriverWriteState(@TempDir File tempDir) throws Exce snapshot, tableSchema, new StructType(), - new TestLogicalWriteInfo(tableSchema)); + new TestLogicalWriteInfo(tableSchema), + /* variantShreddingEnabled */ false); assertSame(engine, context.getEngine()); assertNotNull(context.getTransaction()); @@ -71,7 +73,6 @@ public void createInitializesDriverWriteState(@TempDir File tempDir) throws Exce assertArrayEquals(tableSchema.fieldNames(), context.getDataSchema().fieldNames()); assertEquals(0, context.getPartitionSchema().fields().length); - assertEquals(snapshot.getSchema().length(), context.getKernelTableSchema().length()); String sessionTimeZone = spark.sessionState().conf().sessionLocalTimeZone(); assertEquals(sessionTimeZone, context.getSessionTimeZoneId()); diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/write/DeltaV2BatchWriteTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/write/DeltaV2BatchWriteTest.java index ccbcc914d32..c2408b9c5d4 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/write/DeltaV2BatchWriteTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/write/DeltaV2BatchWriteTest.java @@ -18,7 +18,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import io.delta.kernel.Snapshot; import io.delta.kernel.data.MapValue; import io.delta.kernel.internal.actions.AddFile; import io.delta.spark.internal.v2.DeltaV2TestBase; @@ -39,6 +38,7 @@ import org.apache.spark.sql.connector.write.DataWriterFactory; import org.apache.spark.sql.connector.write.LogicalWriteInfo; import org.apache.spark.sql.connector.write.WriterCommitMessage; +import org.apache.spark.sql.delta.Snapshot; import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.types.StructType; @@ -412,7 +412,8 @@ public void testCommit_partitioned_multipleColumns(@TempDir File tempDir) throws mgr, data, part, - WriteTestUtils.logicalWriteInfo(full, CaseInsensitiveStringMap.empty())) + WriteTestUtils.logicalWriteInfo(full, CaseInsensitiveStringMap.empty()), + /* variantShreddingEnabled */ false) .toBatch(); DataWriter writer = @@ -505,7 +506,8 @@ private DeltaV2BatchWrite newMultiColWrite(String path) { mgr, data, part, - WriteTestUtils.logicalWriteInfo(full, CaseInsensitiveStringMap.empty())) + WriteTestUtils.logicalWriteInfo(full, CaseInsensitiveStringMap.empty()), + /* variantShreddingEnabled */ false) .toBatch(); } @@ -547,7 +549,8 @@ private DeltaV2BatchWrite newTimestampPartWrite(String path) { mgr, data, part, - WriteTestUtils.logicalWriteInfo(full, CaseInsensitiveStringMap.empty())) + WriteTestUtils.logicalWriteInfo(full, CaseInsensitiveStringMap.empty()), + /* variantShreddingEnabled */ false) .toBatch(); } @@ -576,7 +579,8 @@ private DeltaV2BatchWrite newPartitionedWrite(String path) { snapshotManager, PARTITIONED_DATA_SCHEMA, PARTITIONED_PART_SCHEMA, - info); + info, + /* variantShreddingEnabled */ false); return (DeltaV2BatchWrite) write.toBatch(); } @@ -674,6 +678,7 @@ private DeltaV2BatchWrite newWrite(String path) { snapshot, TABLE_SCHEMA, new StructType(), - info); + info, + /* variantShreddingEnabled */ false); } } diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/write/DeltaV2StreamingWriteTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/write/DeltaV2StreamingWriteTest.java index 0f87a69f469..cdba8be59ca 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/write/DeltaV2StreamingWriteTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/write/DeltaV2StreamingWriteTest.java @@ -19,25 +19,35 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; -import io.delta.kernel.Snapshot; import io.delta.spark.internal.v2.DeltaV2TestBase; import io.delta.spark.internal.v2.InternalRowTestUtils; import io.delta.spark.internal.v2.snapshot.PathBasedSnapshotManager; import java.io.File; import java.util.List; +import org.apache.hadoop.fs.Path; +import org.apache.parquet.format.converter.ParquetMetadataConverter; +import org.apache.parquet.hadoop.ParquetFileReader; import org.apache.spark.sql.Row; import org.apache.spark.sql.catalyst.InternalRow; import org.apache.spark.sql.connector.write.DataWriter; import org.apache.spark.sql.connector.write.LogicalWriteInfo; import org.apache.spark.sql.connector.write.WriterCommitMessage; import org.apache.spark.sql.connector.write.streaming.StreamingDataWriterFactory; +import org.apache.spark.sql.delta.DeltaConfigs; +import org.apache.spark.sql.delta.Snapshot; +import org.apache.spark.sql.delta.shims.VariantShreddingShims; import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.types.StructType; import org.apache.spark.sql.util.CaseInsensitiveStringMap; +import org.apache.spark.types.variant.VariantBuilder; +import org.apache.spark.unsafe.types.VariantVal; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; /** * Unit tests for {@link DeltaV2StreamingWrite}; the streaming counterpart of DeltaV2BatchWriteTest. @@ -51,6 +61,14 @@ public class DeltaV2StreamingWriteTest extends DeltaV2TestBase { DataTypes.createStructField("name", DataTypes.StringType, true) }); + // A variant table, so a shredding-property change can actually change the file layout. + private static final StructType VARIANT_TABLE_SCHEMA = + DataTypes.createStructType( + new StructField[] { + DataTypes.createStructField("id", DataTypes.IntegerType, true), + DataTypes.createStructField("v", DataTypes.VariantType, true) + }); + // Table (value INT) partitioned by (part STRING). The write row layout is the full table schema // [value, part]; the writer projects [value] into the Parquet body and routes on [part]. private static final StructType PARTITIONED_FULL_SCHEMA = @@ -167,6 +185,174 @@ public void testCommit_failsOnConcurrentProtocolChange(@TempDir File tempDir) th assertEquals(0L, spark.read().format("delta").load(path).count()); } + /** + * Turning variant shredding off mid-stream must fail the epoch when the property actually governs + * the layout: the executor writer was built to shred, so committing would land a shredded file on + * an explicit opt-out. Neither other guard sees it -- the schema is untouched and the {@code + * variantShredding} feature stays in the protocol -- so only this branch can fire. + */ + @Test + public void testCommit_failsWhenShreddingPropertyChangeAffectsLayout(@TempDir File tempDir) + throws Exception { + // On a Spark version without shredding support the layout cannot depend on the property, so the + // guard is a deliberate no-op and the commit succeeds; the assertion below would not hold. + assumeTrue(shreddedWritesSupported(), SHREDDING_UNSUPPORTED); + String shreddingKey = DeltaConfigs.ENABLE_VARIANT_SHREDDING().key(); + String path = createVariantTable(tempDir, "streaming_shredding_layout_change", true); + DeltaV2StreamingWrite write = newVariantWrite(path, /* variantShreddingEnabled */ true); + + // Opt out after the write state was built. Adds and drops no feature, so the protocol guard + // cannot fire on it. + spark.sql( + String.format( + "ALTER TABLE delta.`%s` SET TBLPROPERTIES ('%s' = 'false')", path, shreddingKey)); + + IllegalStateException e = + assertThrows( + IllegalStateException.class, + () -> write.commit(0L, new WriterCommitMessage[] {variantEpoch(write, 0L)})); + assertTrue( + e.getMessage().contains(shreddingKey), + "guard should name the changed property: " + e.getMessage()); + // Nothing was appended: the guard fired before the commit. + assertEquals(0L, spark.read().format("delta").load(path).count()); + } + + /** + * The opposite direction: opting in mid-stream must also fail the epoch, because the executor + * writer was built not to shred. The table starts with the {@code variantShredding} feature + * already supported, so enabling the property leaves the protocol unchanged; otherwise enabling + * it would add the feature and the protocol guard would fire first, never reaching this branch. + */ + @Test + public void testCommit_failsWhenShreddingPropertyEnabledMidStream(@TempDir File tempDir) + throws Exception { + assumeTrue(shreddedWritesSupported(), SHREDDING_UNSUPPORTED); + String shreddingKey = DeltaConfigs.ENABLE_VARIANT_SHREDDING().key(); + String path = tempDir.getAbsolutePath(); + spark.sql( + String.format( + "CREATE TABLE streaming_shredding_enabled_mid_stream (id INT, v VARIANT) USING delta " + + "LOCATION '%s' TBLPROPERTIES ('delta.feature.variantShredding' = 'supported', " + + "'%s' = 'false')", + path, shreddingKey)); + DeltaV2StreamingWrite write = newVariantWrite(path, /* variantShreddingEnabled */ false); + + spark.sql( + String.format( + "ALTER TABLE delta.`%s` SET TBLPROPERTIES ('%s' = 'true')", path, shreddingKey)); + + IllegalStateException e = + assertThrows( + IllegalStateException.class, + () -> write.commit(0L, new WriterCommitMessage[] {variantEpoch(write, 0L)})); + assertTrue( + e.getMessage().contains(shreddingKey), + "guard should name the changed property: " + e.getMessage()); + assertEquals(0L, spark.read().format("delta").load(path).count()); + } + + /** + * A shredding-property change that cannot alter the layout must be ignored, not fail the epoch. + * On a table with no variant column the property governs nothing, so the write still commits. + * (Spark 4.0 and a disabled kill switch are the other two cases the same guard condition covers.) + */ + @Test + public void testCommit_ignoresShreddingPropertyChangeThatCannotAffectLayout(@TempDir File tempDir) + throws Exception { + String shreddingKey = DeltaConfigs.ENABLE_VARIANT_SHREDDING().key(); + String path = tempDir.getAbsolutePath(); + spark.sql( + String.format( + "CREATE TABLE streaming_no_variant_toggle (id INT, name STRING) USING delta " + + "LOCATION '%s' TBLPROPERTIES ('%s' = 'true')", + path, shreddingKey)); + DeltaV2StreamingWrite write = newWrite(path, /* variantShreddingEnabled */ true); + WriterCommitMessage[] messages = {writeEpoch(write, 0L, 1, "Alice", 2, "Bob")}; + + spark.sql( + String.format( + "ALTER TABLE delta.`%s` SET TBLPROPERTIES ('%s' = 'false')", path, shreddingKey)); + + // No variant column, so the toggle cannot change the layout and the epoch commits normally. + write.commit(0L, messages); + assertEquals(2L, spark.read().format("delta").load(path).count()); + } + + /** + * Replaying an already-committed epoch must stay an idempotent no-op even when the shredding + * property changed in between. The committed-epoch skip runs before the layout guards, so a + * repeated commit of a committed epoch returns rather than being failed by the property change -- + * the batch already succeeded under the layout in force then. + */ + @Test + public void testCommit_replayOfCommittedEpochIsIdempotentAfterPropertyChange( + @TempDir File tempDir) throws Exception { + String shreddingKey = DeltaConfigs.ENABLE_VARIANT_SHREDDING().key(); + String path = createVariantTable(tempDir, "streaming_variant_replay", true); + DeltaV2StreamingWrite write = newVariantWrite(path, /* variantShreddingEnabled */ true); + + write.commit(0L, new WriterCommitMessage[] {variantEpoch(write, 0L)}); + long rowsAfterFirstCommit = spark.read().format("delta").load(path).count(); + + // Opt out (this itself adds a table version), then capture the baseline just before the replay + // so the assertions isolate the replay's effect from the property change's. + spark.sql( + String.format( + "ALTER TABLE delta.`%s` SET TBLPROPERTIES ('%s' = 'false')", path, shreddingKey)); + long versionsBeforeReplay = spark.sql("DESCRIBE HISTORY delta.`" + path + "`").count(); + + // Replay the committed epoch with freshly written files (a real retry's paths). + write.commit(0L, new WriterCommitMessage[] {variantEpoch(write, 0L)}); + + assertEquals( + rowsAfterFirstCommit, + spark.read().format("delta").load(path).count(), + "replaying a committed epoch must not add data, even after the property changed"); + assertEquals( + versionsBeforeReplay, + spark.sql("DESCRIBE HISTORY delta.`" + path + "`").count(), + "replaying a committed epoch must not create a new table version"); + } + + /** + * A streaming write to a shredding-enabled table must actually shred the variant column: the + * property-derived inference option has to reach the executor-side Parquet writer, not just the + * driver guard. Verified by reading the committed file's Parquet footer for a {@code typed_value} + * child on the variant column. + */ + @Test + public void testCommit_streamingWriteShredsWhenPropertyEnabled(@TempDir File tempDir) + throws Exception { + assumeTrue(shreddedWritesSupported(), SHREDDING_UNSUPPORTED); + String path = createVariantTable(tempDir, "streaming_shreds_enabled", true); + DeltaV2StreamingWrite write = newVariantWrite(path, /* variantShreddingEnabled */ true); + + write.commit(0L, new WriterCommitMessage[] {variantEpoch(write, 0L)}); + + assertTrue( + snapshotHasShreddedVariant(path), + "a streaming write to a shredding-enabled table must produce a shredded file"); + } + + /** + * The converse: a streaming write to a table that has not enabled shredding must leave the + * variant unshredded, matching V1. Holds on every Spark version (nothing shreds), so it is not + * gated on shredding support. + */ + @Test + public void testCommit_streamingWriteDoesNotShredWhenPropertyDisabled(@TempDir File tempDir) + throws Exception { + String path = createVariantTable(tempDir, "streaming_no_shred_disabled", false); + DeltaV2StreamingWrite write = newVariantWrite(path, /* variantShreddingEnabled */ false); + + write.commit(0L, new WriterCommitMessage[] {variantEpoch(write, 0L)}); + + assertFalse( + snapshotHasShreddedVariant(path), + "a streaming write to a table without shredding enabled must not shred"); + } + /** * Replaying an already-committed epoch must be an idempotent no-op: the data is committed once, * and re-committing the same epoch does not duplicate it (the {@code withTransactionId} guard). @@ -290,6 +476,76 @@ public void testCommit_partitioned_samePartitionValueAcrossEpochs(@TempDir File assertEquals(11, partA.get(1).getInt(partA.get(1).fieldIndex("value"))); } + /** + * Partitioned streaming write to a column-mapped table across two epochs; asserts each partition + * lands in a physical {@code col-=} directory (not the logical {@code part=} + * one). Streaming counterpart of V2WriteTest.writeToPartitionedColumnMappingTable. + */ + @ParameterizedTest(name = "columnMappingMode={0}") + @ValueSource(strings = {"name", "id"}) + public void testCommit_partitionedColumnMapping_appendsAcrossEpochs( + String mappingMode, @TempDir File tempDir) throws Exception { + // Unique name per mode: the shared session's catalog persists across invocations, so a shared + // name would collide on the second run. + String path = + createPartitionedColumnMappingTable( + tempDir, "streaming_partitioned_cm_" + mappingMode, mappingMode); + DeltaV2StreamingWrite write = newPartitionedWrite(path); + + // Each epoch spans two partitions (rows pre-sorted by part), exercising file rotation at the + // partition boundary. + write.commit( + 0L, new WriterCommitMessage[] {writePartitionedEpoch(write, 0L, 10, "a", 20, "b")}); + write.commit( + 1L, new WriterCommitMessage[] {writePartitionedEpoch(write, 1L, 11, "a", 30, "c")}); + + List rows = spark.read().format("delta").load(path).orderBy("value").collectAsList(); + assertEquals(4, rows.size(), "distinct epochs must accumulate"); + assertEquals(10, rows.get(0).getInt(rows.get(0).fieldIndex("value"))); + assertEquals("a", rows.get(0).getString(rows.get(0).fieldIndex("part"))); + + // Both epochs' partitions use physical col-= dirs, never logical part=. + assertPhysicalColumnMappedPartitionDir(path, "a"); + assertPhysicalColumnMappedPartitionDir(path, "b"); + assertPhysicalColumnMappedPartitionDir(path, "c"); + + // Partition-predicate read returns only that partition across both epochs. + List onlyA = + spark + .read() + .format("delta") + .load(path) + .filter("part = 'a'") + .orderBy("value") + .collectAsList(); + assertEquals(2, onlyA.size()); + assertEquals(10, onlyA.get(0).getInt(onlyA.get(0).fieldIndex("value"))); + assertEquals(11, onlyA.get(1).getInt(onlyA.get(1).fieldIndex("value"))); + } + + /** + * A null partition value lands in {@code __HIVE_DEFAULT_PARTITION__} under the physical {@code + * col-} parent and reads back as null. + */ + @Test + public void testCommit_partitionedColumnMapping_nullPartitionValue(@TempDir File tempDir) + throws Exception { + String path = + createPartitionedColumnMappingTable(tempDir, "streaming_partitioned_cm_null", "name"); + DeltaV2StreamingWrite write = newPartitionedWrite(path); + + write.commit(0L, new WriterCommitMessage[] {writePartitionedEpoch(write, 0L, 10, null)}); + + List rows = spark.read().format("delta").load(path).collectAsList(); + assertEquals(1, rows.size()); + assertEquals(10, rows.get(0).getInt(rows.get(0).fieldIndex("value"))); + assertTrue( + rows.get(0).isNullAt(rows.get(0).fieldIndex("part")), "null partition must read back null"); + + // The null value is Hive-encoded under the physical col- parent, not a logical part=. + assertPhysicalColumnMappedPartitionDir(path, "__HIVE_DEFAULT_PARTITION__"); + } + /** * Runs the executor-side writer for one epoch and returns its commit message. {@code idNamePairs} * is a flat list of (int id, String name) values. @@ -332,6 +588,10 @@ private String createTable(File tempDir, String tableName) { } private DeltaV2StreamingWrite newWrite(String path) { + return newWrite(path, /* variantShreddingEnabled */ false); + } + + private DeltaV2StreamingWrite newWrite(String path, boolean variantShreddingEnabled) { PathBasedSnapshotManager snapshotManager = new PathBasedSnapshotManager(path, spark.sessionState().newHadoopConf()); Snapshot snapshot = snapshotManager.loadLatestSnapshot(); @@ -346,10 +606,87 @@ private DeltaV2StreamingWrite newWrite(String path) { snapshotManager, TABLE_SCHEMA, new StructType(), - info); + info, + variantShreddingEnabled); return (DeltaV2StreamingWrite) write.toStreaming(); } + private static final String SHREDDING_UNSUPPORTED = + "This Spark version cannot infer a variant shredding schema, so nothing shreds on write"; + + /** Whether writes can shred variant columns on this Spark version (empty shim map means no). */ + private static boolean shreddedWritesSupported() { + return !VariantShreddingShims.getVariantInferShreddingSchemaOptions(true).isEmpty(); + } + + /** Whether any data file in the table at {@code path}'s current snapshot stores v shredded. */ + private boolean snapshotHasShreddedVariant(String path) throws Exception { + List files = + spark + .sql("SELECT DISTINCT input_file_name() AS f FROM delta.`" + path + "`") + .collectAsList(); + assertFalse(files.isEmpty(), "Expected at least one data file at " + path); + for (Row file : files) { + org.apache.parquet.schema.Type variant = + ParquetFileReader.readFooter( + spark.sessionState().newHadoopConf(), + new Path(file.getString(0)), + ParquetMetadataConverter.NO_FILTER) + .getFileMetaData() + .getSchema() + .getType("v"); + assertTrue( + variant instanceof org.apache.parquet.schema.GroupType, + "Expected the variant column to be a Parquet group, got: " + variant); + if (((org.apache.parquet.schema.GroupType) variant).containsField("typed_value")) { + return true; + } + } + return false; + } + + private String createVariantTable(File tempDir, String tableName, boolean shreddingEnabled) { + String path = tempDir.getAbsolutePath(); + spark.sql( + String.format( + "CREATE TABLE %s (id INT, v VARIANT) USING delta LOCATION '%s' " + + "TBLPROPERTIES ('%s' = '%s')", + tableName, path, DeltaConfigs.ENABLE_VARIANT_SHREDDING().key(), shreddingEnabled)); + return path; + } + + private DeltaV2StreamingWrite newVariantWrite(String path, boolean variantShreddingEnabled) { + PathBasedSnapshotManager snapshotManager = + new PathBasedSnapshotManager(path, spark.sessionState().newHadoopConf()); + Snapshot snapshot = snapshotManager.loadLatestSnapshot(); + LogicalWriteInfo info = + WriteTestUtils.logicalWriteInfo(VARIANT_TABLE_SCHEMA, CaseInsensitiveStringMap.empty()); + DeltaV2Write write = + new DeltaV2Write( + defaultEngine, + spark.sessionState().newHadoopConf(), + path, + snapshot, + snapshotManager, + VARIANT_TABLE_SCHEMA, + new StructType(), + info, + variantShreddingEnabled); + return (DeltaV2StreamingWrite) write.toStreaming(); + } + + /** Runs the executor-side writer for one epoch over a single [id, variant] row. */ + private WriterCommitMessage variantEpoch(DeltaV2StreamingWrite write, long epochId) + throws Exception { + DataWriter writer = + write + .createStreamingWriterFactory(WriteTestUtils.physicalWriteInfo(1)) + .createWriter(0, 0L, epochId); + org.apache.spark.types.variant.Variant v = VariantBuilder.parseJson("{\"a\":1}", false); + writer.write(InternalRowTestUtils.row(1, new VariantVal(v.getValue(), v.getMetadata()))); + return writer.commit(); + } + private String createPartitionedTable(File tempDir, String tableName) { String path = tempDir.getAbsolutePath(); spark.sql( @@ -375,7 +712,42 @@ private DeltaV2StreamingWrite newPartitionedWrite(String path) { snapshotManager, PARTITIONED_DATA_SCHEMA, PARTITIONED_PART_SCHEMA, - info); + info, + /* variantShreddingEnabled */ false); return (DeltaV2StreamingWrite) write.toStreaming(); } + + /** + * Creates a (value INT, part STRING) table partitioned by {@code part} with column mapping in + * {@code mappingMode} ({@code name} or {@code id}). The logical schema matches {@link + * #PARTITIONED_FULL_SCHEMA}, so {@link #newPartitionedWrite} can drive it. + */ + private String createPartitionedColumnMappingTable( + File tempDir, String tableName, String mappingMode) { + String path = tempDir.getAbsolutePath(); + spark.sql( + String.format( + "CREATE TABLE %s (value INT, part STRING) USING delta PARTITIONED BY (part) " + + "LOCATION '%s' TBLPROPERTIES ('delta.columnMapping.mode' = '%s')", + tableName, path, mappingMode)); + return path; + } + + /** + * Asserts a physical {@code col-=} partition directory exists under {@code + * tablePath} and that no logical {@code part=} directory was written instead (which would + * be a column-mapping regression). Mirrors V2WriteTest.assertPhysicalPartitionDirExists. + */ + private void assertPhysicalColumnMappedPartitionDir(String tablePath, String value) { + File[] physical = + new File(tablePath).listFiles((d, n) -> n.startsWith("col-") && n.endsWith("=" + value)); + assertTrue( + physical != null && physical.length == 1, + "Expected one physical partition directory col-*=" + value); + File[] logical = + new File(tablePath).listFiles((d, n) -> !n.startsWith("col-") && n.endsWith("=" + value)); + assertTrue( + logical == null || logical.length == 0, + "Did not expect a logical partition directory ending in '=" + value + "'"); + } } diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/write/DeltaV2WriteContextTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/write/DeltaV2WriteContextTest.java index bb09663108c..c3f2ca02306 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/write/DeltaV2WriteContextTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/write/DeltaV2WriteContextTest.java @@ -17,24 +17,36 @@ import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeFalse; +import static org.junit.jupiter.api.Assumptions.assumeTrue; import io.delta.kernel.Operation; -import io.delta.kernel.Snapshot; import io.delta.kernel.TableManager; import io.delta.kernel.Transaction; import io.delta.kernel.defaults.engine.DefaultEngine; import io.delta.kernel.engine.Engine; import io.delta.kernel.utils.CloseableIterable; import io.delta.spark.internal.v2.DeltaV2TestBase; +import io.delta.spark.internal.v2.snapshot.PathBasedSnapshotManager; +import io.delta.spark.internal.v2.utils.ScalaUtils; import io.delta.spark.internal.v2.utils.SchemaUtils; import java.io.File; import java.time.ZoneId; import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; import org.apache.spark.sql.connector.write.LogicalWriteInfo; +import org.apache.spark.sql.delta.Snapshot; +import org.apache.spark.sql.delta.shims.VariantShreddingShims; +import org.apache.spark.sql.delta.v2.interop.DeltaV2Snapshot$; import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.StructType; import org.apache.spark.sql.util.CaseInsensitiveStringMap; @@ -55,7 +67,7 @@ public void createInitializesOperationIndependentState(@TempDir File tempDir) th Configuration hadoopConf = spark.sessionState().newHadoopConf(); Engine engine = DefaultEngine.create(hadoopConf); createKernelTable(path, tableSchema, engine); - Snapshot snapshot = TableManager.loadSnapshot(path).build(engine); + Snapshot snapshot = new PathBasedSnapshotManager(path, engine).loadLatestSnapshot(); DeltaV2WriteContext context = DeltaV2WriteContext.create( @@ -65,7 +77,8 @@ public void createInitializesOperationIndependentState(@TempDir File tempDir) th snapshot, tableSchema, new StructType(), - new TestLogicalWriteInfo(tableSchema)); + new TestLogicalWriteInfo(tableSchema), + /* variantShreddingEnabled */ false); assertSame(engine, context.getEngine()); assertNotNull(context.getOutputWriterFactory()); @@ -89,7 +102,7 @@ public void buildDataWriterFactoryProducesExecutorState(@TempDir File tempDir) t Configuration hadoopConf = spark.sessionState().newHadoopConf(); Engine engine = DefaultEngine.create(hadoopConf); createKernelTable(path, tableSchema, engine); - Snapshot snapshot = TableManager.loadSnapshot(path).build(engine); + Snapshot snapshot = new PathBasedSnapshotManager(path, engine).loadLatestSnapshot(); DeltaV2WriteContext context = DeltaV2WriteContext.create( @@ -99,13 +112,15 @@ public void buildDataWriterFactoryProducesExecutorState(@TempDir File tempDir) t snapshot, tableSchema, new StructType(), - new TestLogicalWriteInfo(tableSchema)); + new TestLogicalWriteInfo(tableSchema), + /* variantShreddingEnabled */ false); // The base is operation-independent: any transaction (here a WRITE txn off the snapshot) can be // turned into the executor-side factory. A real factory with a serialized txn state proves the // shared setup produced usable state. Transaction txn = - snapshot + DeltaV2Snapshot$.MODULE$ + .getKernelSnapshot(snapshot) .buildUpdateTableTransaction(DeltaV2WriteContext.getEngineInfo(), Operation.WRITE) .build(engine); DeltaV2DataWriterFactory factory = context.buildDataWriterFactory(txn); @@ -129,7 +144,7 @@ public void wiredPartitionSchemaIsPreserved(@TempDir File tempDir) throws Except Configuration hadoopConf = spark.sessionState().newHadoopConf(); Engine engine = DefaultEngine.create(hadoopConf); - Snapshot snapshot = TableManager.loadSnapshot(path).build(engine); + Snapshot snapshot = new PathBasedSnapshotManager(path, engine).loadLatestSnapshot(); DeltaV2WriteContext context = DeltaV2WriteContext.create( @@ -139,7 +154,8 @@ public void wiredPartitionSchemaIsPreserved(@TempDir File tempDir) throws Except snapshot, dataSchema, partitionSchema, - new TestLogicalWriteInfo(fullSchema)); + new TestLogicalWriteInfo(fullSchema), + /* variantShreddingEnabled */ false); assertArrayEquals(dataSchema.fieldNames(), context.getDataSchema().fieldNames()); assertArrayEquals(partitionSchema.fieldNames(), context.getPartitionSchema().fieldNames()); @@ -164,7 +180,7 @@ public void buildDataWriterFactoryResolvesColumnsCaseInsensitively(@TempDir File Configuration hadoopConf = spark.sessionState().newHadoopConf(); Engine engine = DefaultEngine.create(hadoopConf); - Snapshot snapshot = TableManager.loadSnapshot(path).build(engine); + Snapshot snapshot = new PathBasedSnapshotManager(path, engine).loadLatestSnapshot(); DeltaV2WriteContext context = DeltaV2WriteContext.create( @@ -174,20 +190,121 @@ public void buildDataWriterFactoryResolvesColumnsCaseInsensitively(@TempDir File snapshot, dataSchema, partitionSchema, - new TestLogicalWriteInfo(mixedCaseWriteSchema)); + new TestLogicalWriteInfo(mixedCaseWriteSchema), + /* variantShreddingEnabled */ false); Transaction txn = - snapshot + DeltaV2Snapshot$.MODULE$ + .getKernelSnapshot(snapshot) .buildUpdateTableTransaction(DeltaV2WriteContext.getEngineInfo(), Operation.WRITE) .build(engine); assertNotNull(context.buildDataWriterFactory(txn)); } + /** + * The table-derived shredding option must be the only surviving spelling of its key. + * + *

Asserted on the merge rather than on the resulting file layout, because the layout cannot + * tell which of two colliding spellings won: {@code ParquetOptions} collapses them through a + * case-insensitive map in iteration order, which a test cannot steer. Deleting the normalization + * in {@link DeltaV2WriteContext#mergeVariantShreddingOptions} fails this test. + */ + @Test + public void mergeVariantShreddingOptionsOverridesCallerSpellings() { + Map shreddingOptions = + ScalaUtils.toJavaMap(VariantShreddingShims.getVariantInferShreddingSchemaOptions(false)); + assumeFalse( + shreddingOptions.isEmpty(), + "This Spark version has no shredding inference option to override"); + + Map callerOptions = new HashMap<>(); + callerOptions.put("unrelated.option", "kept"); + for (String key : shreddingOptions.keySet()) { + // Two spellings that differ from the shim's key only in case, as a caller could pass. + callerOptions.put(key.toUpperCase(Locale.ROOT), "true"); + callerOptions.put(key.substring(0, 1).toUpperCase(Locale.ROOT) + key.substring(1), "true"); + } + + Map merged = + DeltaV2WriteContext.mergeVariantShreddingOptions(callerOptions, false); + + assertEquals("kept", merged.get("unrelated.option"), "unrelated options must pass through"); + for (Map.Entry expected : shreddingOptions.entrySet()) { + List spellings = + merged.keySet().stream() + .filter(key -> key.equalsIgnoreCase(expected.getKey())) + .collect(Collectors.toList()); + assertEquals( + Collections.singletonList(expected.getKey()), + spellings, + "only the table-derived spelling of " + expected.getKey() + " may survive the merge"); + assertEquals( + expected.getValue(), + merged.get(expected.getKey()), + "the surviving value must come from the table property, not the caller"); + } + } + @Test public void engineInfoUsesExpectedPrefix() { assertTrue(DeltaV2WriteContext.getEngineInfo().startsWith("Apache-Spark/")); } + /** + * {@code variantLayoutFollowsProperty} must track the writer's actual shredding eligibility: a + * variant at the top level or nested through structs, but not one reached through an array or map + * element ({@code InferVariantShreddingSchema.getPathsToVariant}). For an unshreddable schema a + * shredding-property change cannot alter any file, so the layout must not be marked + * property-sensitive. + */ + @Test + public void variantLayoutFollowsPropertyMatchesShreddingEligibility(@TempDir File tempDir) + throws Exception { + StructType arrayOfVariant = + new StructType() + .add("id", DataTypes.IntegerType) + .add("arr", DataTypes.createArrayType(DataTypes.VariantType, true)); + assertFalse( + buildContext(new File(tempDir, "arr"), arrayOfVariant).variantLayoutFollowsProperty(), + "array-of-variant is not shreddable, so its layout does not follow the property"); + + // The shreddable schemas depend on the property only where this Spark version can shred (the + // kill switch is on under test); on a version without shredding the layout follows nothing. + assumeTrue( + !VariantShreddingShims.getVariantInferShreddingSchemaOptions(true).isEmpty(), + "no variant shredding support on this Spark version"); + StructType topLevelVariant = + new StructType().add("id", DataTypes.IntegerType).add("v", DataTypes.VariantType); + assertTrue( + buildContext(new File(tempDir, "top"), topLevelVariant).variantLayoutFollowsProperty(), + "top-level variant is shreddable, so its layout follows the property"); + StructType structNestedVariant = + new StructType() + .add("id", DataTypes.IntegerType) + .add("s", new StructType().add("v", DataTypes.VariantType)); + assertTrue( + buildContext(new File(tempDir, "struct"), structNestedVariant) + .variantLayoutFollowsProperty(), + "struct-nested variant is shreddable, so its layout follows the property"); + } + + private DeltaV2WriteContext buildContext(File dir, StructType schema) throws Exception { + String path = dir.getAbsolutePath(); + Configuration hadoopConf = spark.sessionState().newHadoopConf(); + Engine engine = DefaultEngine.create(hadoopConf); + createKernelTable(path, schema, engine); + Snapshot snapshot = new PathBasedSnapshotManager(path, engine).loadLatestSnapshot(); + return DeltaV2WriteContext.create( + engine, + hadoopConf, + path, + snapshot, + schema, + new StructType(), + new TestLogicalWriteInfo(schema), + /* variantShreddingEnabled */ false); + } + private static void createKernelTable(String path, StructType schema, Engine engine) { TableManager.buildCreateTableTransaction( path, SchemaUtils.convertSparkSchemaToKernelSchema(schema), "test") diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/write/DeltaV2WriteTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/write/DeltaV2WriteTest.java index 4855ae20af0..aa57dbb631e 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/write/DeltaV2WriteTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/write/DeltaV2WriteTest.java @@ -21,7 +21,6 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import io.delta.kernel.Snapshot; import io.delta.spark.internal.v2.DeltaV2TestBase; import io.delta.spark.internal.v2.snapshot.PathBasedSnapshotManager; import java.io.File; @@ -30,6 +29,7 @@ import org.apache.spark.sql.connector.expressions.NamedReference; import org.apache.spark.sql.connector.expressions.SortOrder; import org.apache.spark.sql.connector.write.LogicalWriteInfo; +import org.apache.spark.sql.delta.Snapshot; import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.types.StructType; @@ -117,7 +117,8 @@ public void testDistributionAndOrdering_partitionedSortsWithoutDistribution( mgr, dataSchema, partitionSchema, - WriteTestUtils.logicalWriteInfo(TABLE_SCHEMA, CaseInsensitiveStringMap.empty())); + WriteTestUtils.logicalWriteInfo(TABLE_SCHEMA, CaseInsensitiveStringMap.empty()), + /* variantShreddingEnabled */ false); assertInstanceOf(UnspecifiedDistribution.class, write.requiredDistribution()); SortOrder[] ordering = write.requiredOrdering(); @@ -163,7 +164,8 @@ public void testDistributionAndOrdering_multipleColumnsSortInPartitionOrder( mgr, dataSchema, partitionSchema, - WriteTestUtils.logicalWriteInfo(fullSchema, CaseInsensitiveStringMap.empty())); + WriteTestUtils.logicalWriteInfo(fullSchema, CaseInsensitiveStringMap.empty()), + /* variantShreddingEnabled */ false); assertInstanceOf(UnspecifiedDistribution.class, write.requiredDistribution()); SortOrder[] ordering = write.requiredOrdering(); @@ -195,6 +197,7 @@ private DeltaV2Write newWrite(String path, CaseInsensitiveStringMap options) { snapshotManager, TABLE_SCHEMA, new StructType(), - info); + info, + /* variantShreddingEnabled */ false); } } diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/write/DeltaV2WriterCommitMessageTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/write/DeltaV2WriterCommitMessageTest.java index e95ad39162f..cdfbe09202d 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/write/DeltaV2WriterCommitMessageTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/write/DeltaV2WriterCommitMessageTest.java @@ -17,7 +17,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; -import io.delta.kernel.Snapshot; import io.delta.kernel.data.Row; import io.delta.kernel.utils.CloseableIterable; import io.delta.kernel.utils.CloseableIterator; @@ -28,6 +27,7 @@ import org.apache.spark.sql.catalyst.InternalRow; import org.apache.spark.sql.connector.write.DataWriter; import org.apache.spark.sql.connector.write.WriterCommitMessage; +import org.apache.spark.sql.delta.Snapshot; import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.types.StructType; @@ -130,7 +130,8 @@ private DeltaV2DataWriterFactory dataWriterFactory(String path) { snapshot, TABLE_SCHEMA, new StructType(), - WriteTestUtils.logicalWriteInfo(TABLE_SCHEMA, CaseInsensitiveStringMap.empty())); + WriteTestUtils.logicalWriteInfo(TABLE_SCHEMA, CaseInsensitiveStringMap.empty()), + /* variantShreddingEnabled */ false); return (DeltaV2DataWriterFactory) write.createBatchWriterFactory(WriteTestUtils.physicalWriteInfo(1)); } diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/write/V2WriteTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/write/V2WriteTest.java index 55bc712a79f..cd294b76f23 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/write/V2WriteTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/write/V2WriteTest.java @@ -15,20 +15,27 @@ */ package io.delta.spark.internal.v2.write; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; import io.delta.spark.internal.v2.V2TestBase; import java.io.File; import java.util.List; +import java.util.Map; import org.apache.hadoop.fs.Path; import org.apache.parquet.format.converter.ParquetMetadataConverter; import org.apache.parquet.hadoop.ParquetFileReader; +import org.apache.spark.sql.delta.DeltaConfigs; +import org.apache.spark.sql.delta.shims.VariantShreddingShims; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; +import scala.Option; /** E2E DSv2 batch-write tests for column-mapped tables. */ public class V2WriteTest extends V2TestBase { @@ -94,6 +101,100 @@ public void multipleAppendsOnColumnMappingTable(@TempDir File deltaTablePath) { List.of(row(1, "Alice", 100.0), row(2, "Bob", 200.0), row(3, "Carol", 300.0))); } + @ParameterizedTest(name = "columnMappingMode={0}") + @ValueSource(strings = {"name", "id"}) + public void writeToPartitionedColumnMappingTable(String mappingMode, @TempDir File deltaTablePath) + throws Exception { + String tablePath = deltaTablePath.getAbsolutePath(); + createPartitionedColumnMappingTable(tablePath, mappingMode); + + spark.sql( + str( + "INSERT INTO dsv2.delta.`%s` VALUES (1, 'Alice', 100.0), (2, 'Bob', 200.0)", + tablePath)); + + // Read back through both the V2 and V1 paths, both resolve partition values by physical name, + // so a logical-directory regression would surface here as null partition columns. + check( + str("SELECT * FROM dsv2.delta.`%s` ORDER BY id", tablePath), + List.of(row(1, "Alice", 100.0), row(2, "Bob", 200.0))); + check( + str("SELECT * FROM delta.`%s` ORDER BY id", tablePath), + List.of(row(1, "Alice", 100.0), row(2, "Bob", 200.0))); + + assertPhysicalPartitionDirExists(tablePath, "Alice"); + assertPhysicalPartitionDirExists(tablePath, "Bob"); + assertAddFilePartitionValuesArePhysical(tablePath, "name"); + // The Parquet body omits the partition column and uses physical col-* names for the rest. + assertPhysicalParquetUsesMappedColumnNames( + physicalPartitionDir(tablePath, "Alice").getAbsolutePath(), "id", "value"); + assertParquetBodyOmitsPartitionColumns( + physicalPartitionDir(tablePath, "Alice"), /* expectedDataColumns */ 2); + } + + @Test + public void writeToMultiColumnPartitionedColumnMappingTable(@TempDir File deltaTablePath) + throws Exception { + String tablePath = deltaTablePath.getAbsolutePath(); + spark.sql( + str( + "CREATE TABLE delta.`%s` (id INT, region STRING, tier INT) USING delta " + + "PARTITIONED BY (region, tier) " + + "TBLPROPERTIES ('delta.columnMapping.mode' = 'name')", + tablePath)); + + spark.sql(str("INSERT INTO dsv2.delta.`%s` VALUES (1, 'us', 1), (2, 'eu', 2)", tablePath)); + + check( + str("SELECT * FROM dsv2.delta.`%s` ORDER BY id", tablePath), + List.of(row(1, "us", 1), row(2, "eu", 2))); + check( + str("SELECT * FROM delta.`%s` ORDER BY id", tablePath), + List.of(row(1, "us", 1), row(2, "eu", 2))); + + // Nested physical partition directories: col-=us/col-=1. + File outer = physicalPartitionDir(tablePath, "us"); + File inner = physicalPartitionChildDir(outer, "1"); + assertTrue( + inner != null, "Expected a nested physical partition directory col-*=1 under " + outer); + // Both partition columns are path-encoded. + assertParquetBodyOmitsPartitionColumns(inner, /* expectedDataColumns */ 1); + } + + @Test + public void writeNullPartitionValueColumnMappingTable(@TempDir File deltaTablePath) + throws Exception { + String tablePath = deltaTablePath.getAbsolutePath(); + createPartitionedColumnMappingTable(tablePath, "name"); + + spark.sql(str("INSERT INTO dsv2.delta.`%s` VALUES (1, NULL, 100.0)", tablePath)); + + check(str("SELECT * FROM dsv2.delta.`%s`", tablePath), List.of(row(1, null, 100.0))); + check(str("SELECT * FROM delta.`%s`", tablePath), List.of(row(1, null, 100.0))); + + // A null partition value is Hive-encoded under the physical parent directory. + assertPhysicalPartitionDirExists(tablePath, "__HIVE_DEFAULT_PARTITION__"); + assertParquetBodyOmitsPartitionColumns( + physicalPartitionDir(tablePath, "__HIVE_DEFAULT_PARTITION__"), /* expectedDataColumns */ 2); + } + + @Test + public void partitionedIcebergCompatWriteIsRejected(@TempDir File deltaTablePath) { + String tablePath = deltaTablePath.getAbsolutePath(); + spark.sql( + str( + "CREATE TABLE delta.`%s` (id INT, name STRING, value DOUBLE) USING delta " + + "PARTITIONED BY (name) TBLPROPERTIES (" + + "'delta.columnMapping.mode' = 'name', 'delta.enableIcebergCompatV2' = 'true')", + tablePath)); + + // IcebergCompat materializes partition columns into the Parquet body, which this write path + // does not do yet. + assertThrows( + UnsupportedOperationException.class, + () -> spark.sql(str("INSERT INTO dsv2.delta.`%s` VALUES (1, 'Alice', 100.0)", tablePath))); + } + private void createColumnMappingTable(String tablePath, String mappingMode) { spark.sql( str( @@ -102,6 +203,74 @@ private void createColumnMappingTable(String tablePath, String mappingMode) { tablePath, mappingMode)); } + private void createPartitionedColumnMappingTable(String tablePath, String mappingMode) { + spark.sql( + str( + "CREATE TABLE delta.`%s` (id INT, name STRING, value DOUBLE) USING delta " + + "PARTITIONED BY (name) TBLPROPERTIES ('delta.columnMapping.mode' = '%s')", + tablePath, mappingMode)); + } + + /** Returns the {@code col-=} partition directory under {@code tablePath}. */ + private File physicalPartitionDir(String tablePath, String value) { + File[] dirs = + new File(tablePath).listFiles((d, n) -> n.startsWith("col-") && n.endsWith("=" + value)); + assertNotNull(dirs, "Expected partition directories under " + tablePath); + assertTrue( + dirs.length == 1, + "Expected one physical partition directory col-*=" + value + ", got " + dirs.length); + return dirs[0]; + } + + /** Returns the nested {@code col-=} directory under {@code parent}, or null. */ + private File physicalPartitionChildDir(File parent, String value) { + File[] dirs = parent.listFiles((d, n) -> n.startsWith("col-") && n.endsWith("=" + value)); + return (dirs != null && dirs.length == 1) ? dirs[0] : null; + } + + /** + * Asserts a physical {@code col-=} partition directory exists and contains data, and + * that no logical {@code =} directory was written instead. + */ + private void assertPhysicalPartitionDirExists(String tablePath, String value) { + File dir = physicalPartitionDir(tablePath, value); + assertTrue(dir.isDirectory(), "Expected a directory at " + dir); + File[] parquet = dir.listFiles((d, n) -> n.endsWith(".parquet")); + assertTrue(parquet != null && parquet.length > 0, "Expected a parquet file under " + dir); + File[] logical = + new File(tablePath).listFiles((d, n) -> !n.startsWith("col-") && n.endsWith("=" + value)); + assertTrue( + logical == null || logical.length == 0, + "Did not expect a logical partition directory ending in '=" + value + "'"); + } + + /** + * Asserts the AddFile {@code partitionValues} in the log are keyed by the physical col-* name. + * The commit JSON is read with schema inference, so {@code partitionValues} surfaces as a struct + * whose field names are the partition keys. + */ + private void assertAddFilePartitionValuesArePhysical(String tablePath, String logicalName) { + org.apache.spark.sql.types.StructType addType = + (org.apache.spark.sql.types.StructType) + spark + .read() + .json(tablePath + "/_delta_log/*.json") + .where("add is not null") + .schema() + .apply("add") + .dataType(); + org.apache.spark.sql.types.StructType partitionValuesType = + (org.apache.spark.sql.types.StructType) addType.apply("partitionValues").dataType(); + List keys = List.of(partitionValuesType.fieldNames()); + assertFalse(keys.isEmpty(), "Expected at least one partition-value key"); + assertTrue( + keys.stream().allMatch(k -> k.startsWith("col-")), + "Expected physical col-* partition-value keys, got: " + keys); + assertFalse( + keys.contains(logicalName), + "AddFile partitionValues must not use the logical key '" + logicalName + "'"); + } + /** * Verifies the on-disk Parquet schema uses physical column mapping names and that every field * carries a Parquet field id. @@ -140,4 +309,223 @@ private void assertPhysicalParquetUsesMappedColumnNames( field.getId(), "Expected a Parquet field id on column '" + field.getName() + "'"); } } + + /** + * Verifies the Parquet body under {@code partitionDir} holds only the data columns. Partition + * columns are path-encoded, not materialized into the file. + */ + private void assertParquetBodyOmitsPartitionColumns(File partitionDir, int expectedDataColumns) + throws Exception { + File[] parquetFiles = partitionDir.listFiles((dir, name) -> name.endsWith(".parquet")); + assertNotNull(parquetFiles, "Expected parquet data files under " + partitionDir); + assertTrue(parquetFiles.length > 0, "Expected at least one parquet data file"); + List parquetFieldNames = + ParquetFileReader.readFooter( + spark.sessionState().newHadoopConf(), + new Path(parquetFiles[0].getAbsolutePath()), + ParquetMetadataConverter.NO_FILTER) + .getFileMetaData().getSchema().getFields().stream() + .map(org.apache.parquet.schema.Type::getName) + .collect(java.util.stream.Collectors.toList()); + assertEquals( + expectedDataColumns, + parquetFieldNames.size(), + "Expected only the data columns in the Parquet body, got: " + parquetFieldNames); + } + + @ParameterizedTest(name = "enableVariantShredding={0}") + @ValueSource(booleans = {false, true}) + public void variantWriteFollowsTableShreddingProperty( + boolean shreddingEnabled, @TempDir File deltaTablePath) throws Exception { + // The opt-in arm asserts that shredding happens, which needs a Spark version that can infer a + // shredding schema. The opt-out arm asserts it does not, which holds on every version. + assumeTrue(!shreddingEnabled || shreddedWritesSupported(), SHREDDING_UNSUPPORTED); + String v2Path = new File(deltaTablePath, "v2").getAbsolutePath(); + String v1Path = new File(deltaTablePath, "v1").getAbsolutePath(); + withShreddedWritesAllowed( + () -> { + for (String path : List.of(v1Path, v2Path)) { + spark.sql( + str( + "CREATE TABLE delta.`%s` (id INT, v VARIANT) USING delta " + + "TBLPROPERTIES ('%s' = '%s')", + path, DeltaConfigs.ENABLE_VARIANT_SHREDDING().key(), shreddingEnabled)); + } + spark.sql(str("INSERT INTO delta.`%s` %s", v1Path, VARIANT_ROW)); + spark.sql(str("INSERT INTO dsv2.delta.`%s` %s", v2Path, VARIANT_ROW)); + + assertEquals( + shreddingEnabled, + snapshotHasShreddedVariant(str("delta.`%s`", v1Path)), + "V1 shredding state must follow the table property"); + assertEquals( + shreddingEnabled, + snapshotHasShreddedVariant(str("delta.`%s`", v2Path)), + "V2 shredding state must follow the table property, like V1"); + + List> expected = List.of(row(1, 1, "xy")); + check(str("%s FROM dsv2.delta.`%s`", VARIANT_PROJECTION, v2Path), expected); + check(str("%s FROM delta.`%s`", VARIANT_PROJECTION, v2Path), expected); + }); + } + + /** + * The distinguishing state: the {@code variantShredding} feature is in the protocol while the + * table property does not enable shredding. Shredding must follow the property, not the protocol + * feature, so neither connector may shred here. Engines whose SQL parser supports {@code REORG + * ... APPLY (UNSHRED VARIANT)} also reach this state through that command, but the syntax is not + * in the OSS SQL parser, so the state is built directly at creation via {@code + * delta.feature.variantShredding = supported} with the property explicitly disabled. + */ + @Test + public void variantWriteFollowsPropertyWhenFeaturePresentButDisabled( + @TempDir File deltaTablePath) { + // Adding the feature and asserting the no-shred outcome both need shredding support. + assumeTrue(shreddedWritesSupported(), SHREDDING_UNSUPPORTED); + String enableKey = DeltaConfigs.ENABLE_VARIANT_SHREDDING().key(); + String tablePath = new File(deltaTablePath, "featureNoProp").getAbsolutePath(); + String tbl = "variant_feature_disabled_tbl"; + withShreddedWritesAllowed( + () -> { + try { + spark.sql(str("DROP TABLE IF EXISTS %s", tbl)); + spark.sql( + str( + "CREATE TABLE %s (id INT, v VARIANT) USING delta LOCATION '%s' " + + "TBLPROPERTIES ('delta.feature.variantShredding' = 'supported', " + + "'%s' = 'false')", + tbl, tablePath, enableKey)); + assertTrue( + tableFeatures(tbl).contains("variantShredding"), + "Premise: the protocol carries the shredding feature"); + assertFalse( + "true".equalsIgnoreCase(tableProperties(tbl).getOrDefault(enableKey, "false")), + "Premise: the property does not enable shredding, so the two cannot be conflated"); + + // Neither connector may shred while the property does not enable it. + spark.sql(str("INSERT INTO dsv2.delta.`%s` %s", tablePath, VARIANT_ROW)); + assertFalse( + snapshotHasShreddedVariant(tbl), + "A DSv2 write must not shred a table whose property does not enable shredding"); + spark.sql(str("INSERT INTO %s %s", tbl, VARIANT_ROW)); + assertFalse(snapshotHasShreddedVariant(tbl), "A V1 write must not shred it either"); + } finally { + spark.sql(str("DROP TABLE IF EXISTS %s", tbl)); + } + }); + } + + /** + * The table-derived option must win over a caller-supplied spelling of the same option that + * differs only in case: Parquet reads write options through a case-insensitive map, so leaving + * both spellings in place would make the effective value depend on map iteration order. + * + *

This covers the end-to-end outcome only: which of two colliding spellings survives the + * collapse is decided by map iteration order, which a test cannot steer from out here, so this + * passes with or without the normalization. {@code + * DeltaV2WriteContextTest#mergeVariantShreddingOptionsOverridesCallerSpellings} asserts on the + * merge itself and does fail without it. What this one still catches is writer options being + * allowed to override the table-derived value outright. + */ + @Test + public void variantWriteIgnoresMixedCaseInferShreddingOption(@TempDir File deltaTablePath) { + String tablePath = new File(deltaTablePath, "mixedcase").getAbsolutePath(); + withShreddedWritesAllowed( + () -> { + spark.sql( + str( + "CREATE TABLE delta.`%s` (id INT, v VARIANT) USING delta " + + "TBLPROPERTIES ('%s' = 'false')", + tablePath, DeltaConfigs.ENABLE_VARIANT_SHREDDING().key())); + spark + .sql(VARIANT_ROW) + .writeTo(str("dsv2.delta.`%s`", tablePath)) + .option("SPARK.SQL.Variant.InferShreddingSchema", "true") + .append(); + assertFalse( + snapshotHasShreddedVariant(str("delta.`%s`", tablePath)), + "A mixed-case write option must not override the table property"); + }); + } + + private static final String SHREDDING_UNSUPPORTED = + "This Spark version cannot infer a variant shredding schema, so nothing shreds on write"; + + /** + * Whether writes can shred variant columns on this Spark version. Asked of the production shim + * rather than of a version string: the shim yields no inference option where the underlying conf + * does not exist, and without that option the writer is never asked to shred. + */ + private static boolean shreddedWritesSupported() { + return !VariantShreddingShims.getVariantInferShreddingSchemaOptions(true).isEmpty(); + } + + private static final String VARIANT_ROW = + "SELECT 1 AS id, parse_json('{\"a\":1,\"b\":\"xy\"}') AS v"; + + private static final String VARIANT_PROJECTION = + "SELECT id, variant_get(v, '$.a', 'int'), variant_get(v, '$.b', 'string')"; + + /** Runs {@code body} with shredded Parquet writes allowed, restoring the previous setting. */ + private void withShreddedWritesAllowed(ThrowingRunnable body) { + String key = "spark.sql.variant.writeShredding.enabled"; + Option previous = spark.conf().getOption(key); + spark.conf().set(key, "true"); + try { + body.run(); + } catch (Exception e) { + throw new RuntimeException(e); + } finally { + if (previous.isDefined()) { + spark.conf().set(key, previous.get()); + } else { + spark.conf().unset(key); + } + } + } + + private interface ThrowingRunnable { + void run() throws Exception; + } + + /** Whether any data file in {@code table}'s current snapshot stores the variant shredded. */ + private boolean snapshotHasShreddedVariant(String table) throws Exception { + List files = + spark.sql(str("SELECT DISTINCT input_file_name() AS f FROM %s", table)).collectAsList(); + assertFalse(files.isEmpty(), "Expected at least one data file in " + table); + for (org.apache.spark.sql.Row file : files) { + org.apache.parquet.schema.Type variant = + ParquetFileReader.readFooter( + spark.sessionState().newHadoopConf(), + new Path(file.getString(0)), + ParquetMetadataConverter.NO_FILTER) + .getFileMetaData() + .getSchema() + .getType("v"); + assertTrue( + variant instanceof org.apache.parquet.schema.GroupType, + "Expected the variant column to be a Parquet group, got: " + variant); + if (((org.apache.parquet.schema.GroupType) variant).containsField("typed_value")) { + return true; + } + } + return false; + } + + private String tableFeatures(String tbl) { + return spark + .sql(str("DESCRIBE DETAIL %s", tbl)) + .selectExpr("tableFeatures") + .collectAsList() + .toString(); + } + + private Map tableProperties(String tbl) { + Map properties = new java.util.HashMap<>(); + spark + .sql(str("SHOW TBLPROPERTIES %s", tbl)) + .collectAsList() + .forEach(r -> properties.put(r.getString(0), r.getString(1))); + return properties; + } } diff --git a/spark/v2/src/test/scala-shims/spark-4.2/io/delta/spark/internal/v2/V2MetadataOnlyDeleteTest.java b/spark/v2/src/test/scala-shims/spark-4.2/io/delta/spark/internal/v2/V2MetadataOnlyDeleteTest.java new file mode 100644 index 00000000000..2aa9f33818c --- /dev/null +++ b/spark/v2/src/test/scala-shims/spark-4.2/io/delta/spark/internal/v2/V2MetadataOnlyDeleteTest.java @@ -0,0 +1,538 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.spark.internal.v2; + +import static org.junit.jupiter.api.Assertions.*; + +import java.util.Arrays; +import java.util.List; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.RowFactory; +import org.apache.spark.sql.api.java.UDF1; +import org.apache.spark.sql.execution.SparkPlan; +import org.apache.spark.sql.execution.QueryExecution; +import org.apache.spark.sql.execution.datasources.v2.DeleteFromTableExec; +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.util.QueryExecutionListener; +import org.junit.jupiter.api.Test; + +/** + * End-to-end tests for the DSv2 metadata-only DELETE path. + */ +public class V2MetadataOnlyDeleteTest extends DeltaV2TestBase { + + private static final String V2_ENABLE_MODE = "spark.databricks.delta.v2.enableMode"; + + @Test + public void testPartitionPredicateDeleteRemovesMatchingPartition() throws Exception { + String tablePath = newTablePath("delta-v2-mod"); + withTable( + tablePath, + () -> { + createIdPartTable(tablePath); + insertRows(tablePath, "(1, 'a'), (2, 'a'), (3, 'b'), (4, 'c')"); + + executeAndAssertDsv2MetadataOnlyDeleteAtPath(tablePath, "part = 'a'"); + + assertTableRows( + tablePath, "id, part", RowFactory.create(3, "b"), RowFactory.create(4, "c")); + }); + } + + @Test + public void testMetadataOnlyDeletePreservesNumRecordsInRemoveFile() throws Exception { + String tablePath = newTablePath("delta-v2-mod"); + withTable( + tablePath, + () -> { + createIdPartTable(tablePath); + insertRows(tablePath, "(1, 'a'), (2, 'a'), (3, 'b')"); + + executeAndAssertDsv2MetadataOnlyDeleteAtPath(tablePath, "part = 'a'"); + + Dataset transactionLog = spark.read().json(tablePath + "/_delta_log/*.json"); + Dataset addedFileNumRecords = + transactionLog + .where("add IS NOT NULL") + .selectExpr( + "add.path AS path", + "CAST(get_json_object(add.stats, '$.numRecords') AS BIGINT) AS numRecords"); + Dataset removedFileNumRecords = + transactionLog + .where("remove IS NOT NULL") + .selectExpr( + "remove.path AS path", + "CAST(get_json_object(remove.stats, '$.numRecords') AS BIGINT) " + + "AS numRecords"); + + assertFalse(removedFileNumRecords.isEmpty()); + assertEquals(0, removedFileNumRecords.where("numRecords IS NULL").count()); + assertTrue(removedFileNumRecords.exceptAll(addedFileNumRecords).isEmpty()); + }); + } + + @Test + public void testMetadataOnlyDeleteInAutoMode() throws Exception { + String tablePath = newTablePath("delta-v2-mod"); + withTable( + tablePath, + () -> { + createIdPartTable(tablePath); + insertRows(tablePath, "(1, 'a'), (2, 'b')"); + + withSQLConf( + V2_ENABLE_MODE, + "AUTO", + () -> { + String sql = + String.format( + "DELETE FROM %s WHERE part = 'a'", sparkCatalogTable(tablePath)); + boolean ranAsDsv2Delete = + executeAndCapturePlans(sql).stream() + .anyMatch(plan -> plan.exists(p -> p instanceof DeleteFromTableExec)); + assertFalse(ranAsDsv2Delete, "AUTO mode should fall back to the V1 DELETE path"); + }); + + assertTableRows(tablePath, "id, part", RowFactory.create(2, "b")); + }); + } + + @Test + public void testNonMetadataPredicateDoesNotUseMetadataOnlyDelete() throws Exception { + String tablePath = newTablePath("delta-v2-mod"); + withTable( + tablePath, + () -> { + createIdPartTable(tablePath); + insertRows(tablePath, "(1, 'a'), (2, 'b')"); + + assertThrows( + UnsupportedOperationException.class, + () -> + spark.sql( + String.format( + "DELETE FROM %s WHERE part = 'a' OR id = 2", dsv2Table(tablePath)))); + + assertTableRows( + tablePath, "id, part", RowFactory.create(1, "a"), RowFactory.create(2, "b")); + }); + } + + @Test + public void testColumnMappingTableDeleteUsesPhysicalSchema() throws Exception { + String tablePath = newTablePath("delta-v2-mod"); + withTable( + tablePath, + () -> { + spark.sql( + String.format( + "CREATE TABLE delta.`%s` (id INT, part STRING) USING delta " + + "PARTITIONED BY (part) " + + "TBLPROPERTIES ('delta.columnMapping.mode' = 'name')", + tablePath)); + insertRows(tablePath, "(1, 'a'), (2, 'b')"); + + executeAndAssertDsv2MetadataOnlyDeleteAtPath(tablePath, "part = 'a'"); + + assertTableRows(tablePath, "id, part", RowFactory.create(2, "b")); + }); + } + + @Test + public void testWholeTableDeleteRemovesAllRows() throws Exception { + String tablePath = newTablePath("delta-v2-mod"); + withTable( + tablePath, + () -> { + createIdPartTable(tablePath); + insertRows(tablePath, "(1, 'a'), (2, 'b')"); + + executeAndAssertDsv2MetadataOnlyDeleteAtPath(tablePath, null); + + assertTableRows(tablePath, "id, part"); + }); + } + + @Test + public void testPartitionPredicateDeleteWithNoMatchIsNoOp() throws Exception { + String tablePath = newTablePath("delta-v2-mod"); + withTable( + tablePath, + () -> { + createIdPartTable(tablePath); + insertRows(tablePath, "(1, 'a'), (2, 'b')"); + + executeAndAssertDsv2MetadataOnlyDeleteAtPath(tablePath, "part = 'nonExistent'"); + + assertTableRows( + tablePath, "id, part", RowFactory.create(1, "a"), RowFactory.create(2, "b")); + }); + } + + @Test + public void testMetadataOnlyDeleteLeavesUntouchedPartitionFilesStable() throws Exception { + String tablePath = newTablePath("delta-v2-mod"); + withTable( + tablePath, + () -> { + createIdPartTable(tablePath); + // Each partition is written as its own file. + insertRows(tablePath, "(1, 'a')"); + insertRows(tablePath, "(2, 'b')"); + + List initialFiles = filePathsById(tablePath); + assertEquals(2, initialFiles.size()); + String deletedFilePath = initialFiles.get(0).getString(1); + String untouchedFilePath = initialFiles.get(1).getString(1); + assertNotEquals(deletedFilePath, untouchedFilePath); + + executeAndAssertDsv2MetadataOnlyDeleteAtPath(tablePath, "part = 'a'"); + + List remainingFiles = filePathsById(tablePath); + assertEquals(1, remainingFiles.size()); + assertEquals(2, remainingFiles.get(0).getInt(0)); + // The surviving partition's file is untouched (not rewritten): metadata-only delete only + // removes the matching file and never rewrites the others. + assertEquals(untouchedFilePath, remainingFiles.get(0).getString(1)); + assertTableRows(tablePath, "id, part", RowFactory.create(2, "b")); + }); + } + + @Test + public void testInListDeleteRemovesMatchingPartitions() throws Exception { + String tablePath = newTablePath("delta-v2-mod"); + withTable( + tablePath, + () -> { + createIdPartTable(tablePath); + insertRows(tablePath, "(1, 'a'), (2, 'b'), (3, 'c')"); + + executeAndAssertDsv2MetadataOnlyDeleteAtPath(tablePath, "part IN ('a', 'c')"); + + assertTableRows(tablePath, "id, part", RowFactory.create(2, "b")); + }); + } + + @Test + public void testOrOnSamePartitionColumnRemovesBothPartitions() throws Exception { + String tablePath = newTablePath("delta-v2-mod"); + withTable( + tablePath, + () -> { + createIdPartTable(tablePath); + insertRows(tablePath, "(1, 'a'), (2, 'b'), (3, 'c')"); + + executeAndAssertDsv2MetadataOnlyDeleteAtPath( + tablePath, "part = 'a' OR part = 'c'"); + + assertTableRows(tablePath, "id, part", RowFactory.create(2, "b")); + }); + } + + @Test + public void testRangeComparisonDeleteRemovesMatchingPartitions() throws Exception { + String tablePath = newTablePath("delta-v2-mod"); + withTable( + tablePath, + () -> { + createIdPartTable(tablePath); + insertRows(tablePath, "(1, 'a'), (2, 'b'), (3, 'c')"); + + executeAndAssertDsv2MetadataOnlyDeleteAtPath(tablePath, "part >= 'b'"); + + assertTableRows(tablePath, "id, part", RowFactory.create(1, "a")); + }); + } + + @Test + public void testInequalityDeleteRemovesMatchingPartitions() throws Exception { + String tablePath = newTablePath("delta-v2-mod"); + withTable( + tablePath, + () -> { + createIdPartTable(tablePath); + insertRows(tablePath, "(1, 'a'), (2, 'b'), (3, 'c')"); + + executeAndAssertDsv2MetadataOnlyDeleteAtPath(tablePath, "part <> 'a'"); + + assertTableRows(tablePath, "id, part", RowFactory.create(1, "a")); + }); + } + + @Test + public void testLikePrefixDeleteRemovesMatchingPartitions() throws Exception { + String tablePath = newTablePath("delta-v2-mod"); + withTable( + tablePath, + () -> { + createIdPartTable(tablePath); + insertRows(tablePath, "(1, 'apple'), (2, 'apricot'), (3, 'banana')"); + + executeAndAssertDsv2MetadataOnlyDeleteAtPath(tablePath, "part LIKE 'a%'"); + + assertTableRows(tablePath, "id, part", RowFactory.create(3, "banana")); + }); + } + + @Test + public void testSubstringOnPartitionColumnDelete() throws Exception { + String tablePath = newTablePath("delta-v2-mod"); + withTable( + tablePath, + () -> { + createIdPartTable(tablePath); + insertRows(tablePath, "(1, 'apple'), (2, 'avocado'), (3, 'banana')"); + + executeAndAssertDsv2MetadataOnlyDeleteAtPath( + tablePath, "substring(part, 1, 1) = 'a'"); + + assertTableRows(tablePath, "id, part", RowFactory.create(3, "banana")); + }); + } + + @Test + public void testUdfOnPartitionColumnDelete() throws Exception { + String tablePath = newTablePath("delta-v2-mod"); + withTable( + tablePath, + () -> { + spark + .udf() + .register( + "starts_with_a", + (UDF1) s -> s != null && s.startsWith("a"), + DataTypes.BooleanType); + createIdPartTable(tablePath); + insertRows(tablePath, "(1, 'apple'), (2, 'avocado'), (3, 'banana')"); + + executeAndAssertDsv2MetadataOnlyDeleteAtPath(tablePath, "starts_with_a(part)"); + + assertTableRows(tablePath, "id, part", RowFactory.create(3, "banana")); + }); + } + + @Test + public void testArithmeticOnIntPartitionColumnDelete() throws Exception { + String tablePath = newTablePath("delta-v2-mod"); + withTable( + tablePath, + () -> { + spark.sql( + String.format( + "CREATE TABLE delta.`%s` (id INT, pid INT) USING delta PARTITIONED BY (pid)", + tablePath)); + spark.sql( + String.format( + "INSERT INTO delta.`%s` VALUES (1, 1), (2, 2), (3, 3)", tablePath)); + + executeAndAssertDsv2MetadataOnlyDeleteAtPath(tablePath, "pid + 1 = 3"); + + assertTableRows( + tablePath, "id, pid", RowFactory.create(1, 1), RowFactory.create(3, 3)); + }); + } + + @Test + public void testMixedOrWithUntranslatableAndSimpleLeg() throws Exception { + String tablePath = newTablePath("delta-v2-mod"); + withTable( + tablePath, + () -> { + createIdPartTable(tablePath); + insertRows(tablePath, "(1, 'a'), (2, 'b'), (3, 'c')"); + + executeAndAssertDsv2MetadataOnlyDeleteAtPath( + tablePath, "part = 'a' OR substring(part, 1, 1) = 'c'"); + + assertTableRows(tablePath, "id, part", RowFactory.create(2, "b")); + }); + } + + @Test + public void testAndAcrossTwoPartitionColumns() throws Exception { + String tablePath = newTablePath("delta-v2-mod"); + withTable( + tablePath, + () -> { + spark.sql( + String.format( + "CREATE TABLE delta.`%s` (id INT, p0 STRING, p1 STRING) USING delta " + + "PARTITIONED BY (p0, p1)", + tablePath)); + spark.sql( + String.format( + "INSERT INTO delta.`%s` VALUES " + + "(1, 'a', 'x'), (2, 'a', 'y'), (3, 'b', 'x')", + tablePath)); + + executeAndAssertDsv2MetadataOnlyDeleteAtPath( + tablePath, "p0 = 'a' AND p1 = 'x'"); + + assertTableRows( + tablePath, + "id, p0, p1", + RowFactory.create(2, "a", "y"), + RowFactory.create(3, "b", "x")); + }); + } + + @Test + public void testAlwaysTrueConditionRemovesAllRows() throws Exception { + String tablePath = newTablePath("delta-v2-mod"); + withTable( + tablePath, + () -> { + createIdPartTable(tablePath); + insertRows(tablePath, "(1, 'a'), (2, 'b')"); + + executeAndAssertDsv2MetadataOnlyDeleteAtPath(tablePath, "1 = 1"); + + assertTableRows(tablePath, "id, part"); + }); + } + + @Test + public void testIsNullOnPartitionColumnDelete() throws Exception { + String tablePath = newTablePath("delta-v2-mod"); + withTable( + tablePath, + () -> { + createIdPartTable(tablePath); + insertRows(tablePath, "(1, 'a'), (2, null), (3, 'b')"); + + executeAndAssertDsv2MetadataOnlyDeleteAtPath(tablePath, "part IS NULL"); + + assertTableRows( + tablePath, "id, part", RowFactory.create(1, "a"), RowFactory.create(3, "b")); + }); + } + + @Test + public void testIsNotNullOnPartitionColumnDelete() throws Exception { + String tablePath = newTablePath("delta-v2-mod"); + withTable( + tablePath, + () -> { + createIdPartTable(tablePath); + insertRows(tablePath, "(1, 'a'), (2, null), (3, 'b')"); + + executeAndAssertDsv2MetadataOnlyDeleteAtPath(tablePath, "part IS NOT NULL"); + + assertTableRows(tablePath, "id, part", RowFactory.create(2, null)); + }); + } + + private void createIdPartTable(String tablePath) { + spark.sql( + String.format( + "CREATE TABLE delta.`%s` (id INT, part STRING) USING delta PARTITIONED BY (part)", + tablePath)); + } + + private String sparkCatalogTable(String tablePath) { + return String.format("delta.`%s`", tablePath); + } + + private void insertRows(String tablePath, String valuesSql) { + spark.sql(String.format("INSERT INTO delta.`%s` VALUES %s", tablePath, valuesSql)); + } + + private List filePathsById(String tablePath) { + return spark + .sql( + String.format( + "SELECT id, _metadata.file_path AS file_path FROM delta.`%s` ORDER BY id", + tablePath)) + .collectAsList(); + } + + private void assertTableRows(String tablePath, String columns, Row... expectedRows) { + List actualRows = + spark + .sql(String.format("SELECT %s FROM delta.`%s` ORDER BY id", columns, tablePath)) + .collectAsList(); + List expected = Arrays.asList(expectedRows); + assertEquals( + expected, + actualRows, + () -> "Unexpected table contents for " + tablePath + ": " + actualRows); + assertEquals( + spark + .sql(String.format("SELECT %s FROM %s ORDER BY id", columns, dsv2Table(tablePath))) + .collectAsList(), + actualRows, + "DSv2 and V1 reads should observe the same committed rows"); + } + + /** + * Runs a DELETE against the {@code dsv2} catalog and asserts that it executed as a DSv2 + * metadata-only delete: the physical plan is the DSv2 {@link DeleteFromTableExec} node. + * + * @param tablePath the table's filesystem path + * @param whereClause the DELETE predicate without the {@code WHERE} keyword, or {@code null} for + * a whole-table delete + */ + private void executeAndAssertDsv2MetadataOnlyDeleteAtPath(String tablePath, String whereClause) { + executeAndAssertDsv2MetadataOnlyDelete(dsv2Table(tablePath), whereClause); + } + + private void executeAndAssertDsv2MetadataOnlyDelete(String table, String whereClause) { + String sql = + whereClause == null + ? String.format("DELETE FROM %s", table) + : String.format("DELETE FROM %s WHERE %s", table, whereClause); + + boolean ranAsMetadataOnlyDelete = + executeAndCapturePlans(sql).stream() + .anyMatch(plan -> plan.exists(p -> p instanceof DeleteFromTableExec)); + assertTrue( + ranAsMetadataOnlyDelete, + "Expected the DELETE to execute as a DSv2 metadata-only DeleteFromTableExec"); + } + + /** + * Runs {@code sql} and returns every executed physical plan captured via a listener. A DSv2 + * metadata-only delete selects files via {@code filesForScan}, which runs its own nested Spark + * queries (materializing {@code allFiles}). Hence, more than one plan is executed. Callers inspect + * the collected plans for the node they care about rather than assuming a single one. + */ + private List executeAndCapturePlans(String sql) { + List captured = java.util.Collections.synchronizedList(new java.util.ArrayList<>()); + QueryExecutionListener listener = new QueryExecutionListener() { + @Override + public void onSuccess(String funcName, QueryExecution qe, long durationNs) { + captured.add(qe.executedPlan()); + } + + @Override + public void onFailure(String funcName, QueryExecution qe, Exception exception) {} + }; + + spark.listenerManager().register(listener); + try { + spark.sql(sql); + spark.sparkContext().listenerBus().waitUntilEmpty(60_000L); + } catch (java.util.concurrent.TimeoutException e) { + throw new RuntimeException("Timed out waiting for the query-execution listener", e); + } finally { + spark.listenerManager().unregister(listener); + } + assertFalse( + captured.isEmpty(), "QueryExecutionListener did not capture an executed plan for: " + sql); + return captured; + } +} diff --git a/spark/v2/src/test/scala/io/delta/spark/internal/v2/kernel/KernelContextSuite.scala b/spark/v2/src/test/scala/io/delta/spark/internal/v2/kernel/KernelContextSuite.scala new file mode 100644 index 00000000000..e98e9281d3a --- /dev/null +++ b/spark/v2/src/test/scala/io/delta/spark/internal/v2/kernel/KernelContextSuite.scala @@ -0,0 +1,139 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.spark.internal.v2.kernel + +import java.util.concurrent.FutureTask + +import org.apache.spark.sql.delta.storage.LogStore + +import org.apache.spark.SparkFunSuite +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.test.SharedSparkSession + +class KernelContextSuite extends SparkFunSuite with SharedSparkSession { + + private val hadoopConfKey = "kernel.context.test" + private val invariantHadoopConfKey = "kernel.context.invariant.test" + + private def newContext(options: Map[String, String] = Map.empty): KernelContext = + KernelContext(options, LogStore.createLogStore(spark)) + + test("an empty context uses the active session Hadoop configuration") { + withSQLConf(hadoopConfKey -> "session-value") { + assert(newContext().materializeHadoopConf().get(hadoopConfKey) == "session-value") + } + } + + test("session-invariant filesystem options override the active session configuration") { + withSQLConf(hadoopConfKey -> "session-value") { + val context = newContext(Map(hadoopConfKey -> "context-value")) + + assert(context.materializeHadoopConf().get(hadoopConfKey) == "context-value") + } + } + + test("context retains the provided LogStore") { + val logStore = LogStore.createLogStore(spark) + val context = KernelContext(Map.empty, logStore) + + assert(context.logStore eq logStore) + } + + test("materializeHadoopConf resolves the active Spark session on every call") { + val originalSession = SparkSession.active + val otherSession = spark.newSession() + originalSession.conf.set(hadoopConfKey, "original-session") + otherSession.conf.set(hadoopConfKey, "other-session") + val context = newContext() + + try { + SparkSession.setActiveSession(originalSession) + assert(context.materializeHadoopConf().get(hadoopConfKey) == "original-session") + + SparkSession.setActiveSession(otherSession) + assert(context.materializeHadoopConf().get(hadoopConfKey) == "other-session") + } finally { + SparkSession.setActiveSession(originalSession) + originalSession.conf.unset(hadoopConfKey) + otherSession.conf.unset(hadoopConfKey) + } + } + + test("child threads inherit the active session without changing the shared KernelContext") { + val originalSession = SparkSession.active + val sessionA = spark.newSession() + val sessionB = spark.newSession() + val invariantOptions = Map(invariantHadoopConfKey -> "context-value") + val context = newContext(invariantOptions) + + try { + sessionA.conf.set(hadoopConfKey, "session-a") + SparkSession.setActiveSession(sessionA) + val parentAConf = context.materializeHadoopConf() + val childATask = new FutureTask[(SparkSession, String, String)](() => { + val hadoopConf = context.materializeHadoopConf() + (SparkSession.active, hadoopConf.get(hadoopConfKey), hadoopConf.get(invariantHadoopConfKey)) + }) + new Thread(childATask).start() + val (childASession, childASessionValue, childAInvariantValue) = childATask.get() + + assert(childASession eq sessionA) + assert(childASessionValue == parentAConf.get(hadoopConfKey)) + assert(childAInvariantValue == parentAConf.get(invariantHadoopConfKey)) + + sessionB.conf.set(hadoopConfKey, "session-b") + SparkSession.setActiveSession(sessionB) + val parentBConf = context.materializeHadoopConf() + val childBTask = new FutureTask[(SparkSession, String, String)](() => { + val hadoopConf = context.materializeHadoopConf() + (SparkSession.active, hadoopConf.get(hadoopConfKey), hadoopConf.get(invariantHadoopConfKey)) + }) + new Thread(childBTask).start() + val (childBSession, childBSessionValue, childBInvariantValue) = childBTask.get() + + assert(childBSession eq sessionB) + assert(childBSessionValue == parentBConf.get(hadoopConfKey)) + assert(childBSessionValue != childASessionValue) + assert(childBInvariantValue == parentBConf.get(invariantHadoopConfKey)) + assert(childBInvariantValue == childAInvariantValue) + assert(context.sessionInvariantFsOptions == invariantOptions) + } finally { + SparkSession.setActiveSession(originalSession) + sessionA.conf.unset(hadoopConfKey) + sessionB.conf.unset(hadoopConfKey) + } + } + + test("constructing a context with null session-invariant filesystem options throws") { + val logStore = LogStore.createLogStore(spark) + Seq[() => KernelContext]( + () => new KernelContext(null, logStore), + () => KernelContext(null, logStore)).foreach { createContext => + val error = intercept[IllegalArgumentException] { + createContext() + } + assert(error.getMessage == "requirement failed: sessionInvariantFsOptions must not be null") + } + } + + test("constructing a context with a null LogStore throws") { + val error = intercept[IllegalArgumentException] { + KernelContext(Map.empty, null) + } + assert(error.getMessage == "requirement failed: logStore must not be null") + } +} diff --git a/spark/v2/src/test/scala/io/delta/spark/internal/v2/snapshot/unitycatalog/UCManagedTableSnapshotManagerSuite.scala b/spark/v2/src/test/scala/io/delta/spark/internal/v2/snapshot/unitycatalog/UCManagedTableSnapshotManagerSuite.scala index 534edaa4b0f..a1d5cf037a5 100644 --- a/spark/v2/src/test/scala/io/delta/spark/internal/v2/snapshot/unitycatalog/UCManagedTableSnapshotManagerSuite.scala +++ b/spark/v2/src/test/scala/io/delta/spark/internal/v2/snapshot/unitycatalog/UCManagedTableSnapshotManagerSuite.scala @@ -19,11 +19,12 @@ import java.util.Optional import scala.jdk.CollectionConverters._ -import io.delta.kernel.exceptions.KernelException +import io.delta.kernel.exceptions.{KernelException, VersionToLoadAfterLatestCommitException} import io.delta.kernel.unitycatalog.{InMemoryUCClient, UCCatalogManagedClient, UCCatalogManagedTestUtils, UCTableIdentifier} import io.delta.spark.internal.v2.exception.VersionNotFoundException import io.delta.storage.commit.uccommitcoordinator.InvalidTargetTableException +import org.scalatest.Outcome import org.scalatest.funsuite.AnyFunSuite /** Integration tests for [[UCManagedTableSnapshotManager]]. */ @@ -31,6 +32,9 @@ class UCManagedTableSnapshotManagerSuite extends AnyFunSuite with UCCatalogManagedTestUtils { + override protected def withFixture(test: NoArgTest): Outcome = + spark.withActive(super.withFixture(test)) + private val testUcTableId = "testUcTableId" private val testTableIdentifier = new UCTableIdentifier("cat", "sch", "tbl") private val testUcUri = "https://test-uc.example.com" @@ -83,7 +87,7 @@ class UCManagedTableSnapshotManagerSuite val snapshot = manager.loadLatestSnapshot() - assert(snapshot.getVersion == maxRatifiedVersion) + assert(snapshot.version == maxRatifiedVersion) assert(ucClient.getLastGetCommitsTableIdentifier.getNamespace.toSeq == Seq("cat", "sch")) assert(ucClient.getLastGetCommitsTableIdentifier.getName == "tbl") } @@ -112,11 +116,13 @@ class UCManagedTableSnapshotManagerSuite withUCClientAndTestTable { (ucClient, tablePath, maxRatifiedVersion) => val manager = createManager(ucClient, tablePath) - assert(manager.loadSnapshotAt(0L).getVersion == 0L) - assert(manager.loadSnapshotAt(1L).getVersion == 1L) + assert(manager.loadSnapshotAt(0L).version == 0L) + assert(manager.loadSnapshotAt(1L).version == 1L) intercept[IllegalArgumentException] { manager.loadSnapshotAt(-1L) } - intercept[IllegalArgumentException] { manager.loadSnapshotAt(maxRatifiedVersion + 10) } + Seq(maxRatifiedVersion + 1, maxRatifiedVersion + 10).foreach { version => + intercept[VersionToLoadAfterLatestCommitException] { manager.loadSnapshotAt(version) } + } } } @@ -300,9 +306,20 @@ class UCManagedTableSnapshotManagerSuite maxRatifiedVersion, Optional.of(maxRatifiedVersion - 1)) } + } + } - intercept[IllegalArgumentException] { - manager.getTableChanges(defaultEngine, maxRatifiedVersion + 5, Optional.empty()) + test("getTableChanges: throws typed exceptions for future start and end versions") { + withUCClientAndTestTable { (ucClient, tablePath, maxRatifiedVersion) => + val manager = createManager(ucClient, tablePath) + + Seq(maxRatifiedVersion + 1, maxRatifiedVersion + 5).foreach { version => + intercept[VersionToLoadAfterLatestCommitException] { + manager.getTableChanges(defaultEngine, version, Optional.empty()) + } + intercept[VersionToLoadAfterLatestCommitException] { + manager.getTableChanges(defaultEngine, 0L, Optional.of(version)) + } } } } diff --git a/spark/v2/src/test/scala/io/delta/spark/internal/v2/tablemanager/CachedSnapshotManagerSuite.scala b/spark/v2/src/test/scala/io/delta/spark/internal/v2/tablemanager/CachedSnapshotManagerSuite.scala new file mode 100644 index 00000000000..b04a7b36eda --- /dev/null +++ b/spark/v2/src/test/scala/io/delta/spark/internal/v2/tablemanager/CachedSnapshotManagerSuite.scala @@ -0,0 +1,1510 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.spark.internal.v2.tablemanager + +import java.io.File +import java.net.URI +import java.util.{Collections, Optional} +import java.util.concurrent.{ + ConcurrentLinkedQueue, + CopyOnWriteArrayList, + CountDownLatch, + CyclicBarrier, + TimeUnit +} +import java.util.concurrent.atomic.{AtomicLong, AtomicReference} + +import scala.jdk.CollectionConverters._ +import scala.util.control.NonFatal + +// format: off +// scalastyle:off import.ordering.noEmptyLine +// scalastyle:off import.ordering.wrongOrderInGroup +import org.apache.spark.sql.delta.storage.LogStore +import io.delta.spark.internal.v2.catalog.DeltaV2Table +import org.apache.spark.sql.delta.v2.interop.{ + DeltaV2QueryContext, + DeltaV2SnapshotManager +} +import io.delta.spark.internal.v2.kernel.{KernelContext, KernelEngineFactory} +import io.delta.spark.internal.v2.snapshot.SnapshotManagerFactory +import org.mockito.Mockito.mockStatic +import org.mockito.invocation.InvocationOnMock +import org.mockito.stubbing.Answer +import io.delta.kernel.exceptions.KernelException + +import io.delta.sql.{DeltaSparkSessionExtensionV1 => DeltaSparkSessionExtension} + +import org.apache.spark.sql.delta.{DeltaConfigs, DeltaLog, Snapshot} +import org.apache.spark.sql.delta.catalog.{DeltaCatalogV1 => DeltaCatalog} +import org.apache.spark.sql.delta.sources.DeltaSQLConf +import org.apache.spark.sql.delta.util.FileNames + +import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.fs.{FileStatus, FSDataInputStream, Path, RawLocalFileSystem} + +import org.apache.spark.SparkConf +import org.apache.spark.network.util.JavaUtils +import org.apache.spark.sql.{QueryTest, SparkSession} +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf} +import org.apache.spark.sql.catalyst.catalog.{ + CatalogStorageFormat, + CatalogTable, + CatalogTableType +} +import org.apache.spark.sql.connector.catalog.Identifier +import org.apache.spark.sql.test.SharedSparkSession +import org.apache.spark.sql.types.StructType +// scalastyle:on import.ordering.noEmptyLine +// scalastyle:on import.ordering.wrongOrderInGroup + +class CachedSnapshotManagerSuite + extends QueryTest + with SharedSparkSession +{ + + override protected def sparkConf: SparkConf = { + super.sparkConf + .set(StaticSQLConf.SPARK_SESSION_EXTENSIONS.key, classOf[DeltaSparkSessionExtension].getName) + .set(SQLConf.V2_SESSION_CATALOG_IMPLEMENTATION.key, classOf[DeltaCatalog].getName) + } +// format: on + + private def createDeltaTable(dir: File, numRows: Int = 10): Unit = { + spark.range(numRows).write.format("delta").save(dir.getCanonicalPath) + } + + private def appendToDeltaTable(dir: File, numRows: Int = 5): Unit = { + spark.range(numRows).write.format("delta").mode("append").save(dir.getCanonicalPath) + } + + private def createManager( + dir: File, + kernelContext: KernelContext = KernelContext(Map.empty, LogStore.createLogStore(spark)) + ): CachedSnapshotManager = { + new CachedSnapshotManager( + new Path(dir.getCanonicalPath), + kernelContext, + new AtomicReference[CatalogTable]()) + } + + private def setRecordingMarkers(session: SparkSession, sessionMarker: String): Unit = { + session.conf.set( + CachedSnapshotManagerRecordingFileSystem.SessionMarkerKey, + sessionMarker) + session.conf.set( + CachedSnapshotManagerRecordingFileSystem.InvariantMarkerKey, + s"session-$sessionMarker") + } + + private def assertRecordingFileSystemObserved(expectedSessionMarker: String): Unit = { + val observations = CachedSnapshotManagerRecordingFileSystem.currentThreadObservations + assert(observations.nonEmpty) + assert(observations.forall { case (_, sessionMarker, invariantMarker) => + sessionMarker == expectedSessionMarker && invariantMarker == "table-option" + }, s"Unexpected filesystem observations: $observations") + } + + private def startAndJoinThreads(threads: Seq[Thread]): Unit = { + threads.foreach(_.setDaemon(true)) + threads.foreach(_.start()) + threads.foreach(_.join(TimeUnit.SECONDS.toMillis(30L))) + val alive = threads.filter(_.isAlive) + alive.foreach(_.interrupt()) + assert(alive.isEmpty, s"Threads did not terminate: ${alive.map(_.getName).mkString(", ")}") + } + + private def awaitClockAfter(timestampMs: Long): Unit = { + val deadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(30L) + while (System.currentTimeMillis() <= timestampMs && System.nanoTime() < deadlineNanos) { + Thread.`yield`() + } + assert( + System.currentTimeMillis() > timestampMs, + s"Wall clock did not advance beyond $timestampMs") + } + + test("contextual APIs use query catalog metadata while legacy APIs use the latest metadata") { + withTempDir { dir => + val legacyCatalogTable = CatalogTable( + identifier = TableIdentifier("legacy_table"), + tableType = CatalogTableType.EXTERNAL, + storage = CatalogStorageFormat.empty, + schema = new StructType()) + val queryCatalogTable = CatalogTable( + identifier = TableIdentifier("query_table"), + tableType = CatalogTableType.EXTERNAL, + storage = CatalogStorageFormat.empty, + schema = new StructType()) + val observedCatalogTables = new ConcurrentLinkedQueue[Option[CatalogTable]]() + val kernelContext = KernelContext(Map.empty, LogStore.createLogStore(spark)) + val factoryMock = mockStatic( + classOf[SnapshotManagerFactory], + new Answer[DeltaV2SnapshotManager] { + override def answer(invocation: InvocationOnMock): DeltaV2SnapshotManager = { + val catalogTable = invocation.getArgument[Optional[CatalogTable]](2) + observedCatalogTables.add( + if (catalogTable.isPresent) Some(catalogTable.get()) else None) + throw new IllegalStateException("recorded catalog metadata") + } + }) + val manager = new CachedSnapshotManager( + new Path(dir.getCanonicalPath), + kernelContext, + new AtomicReference[CatalogTable](legacyCatalogTable)) + try { + val queryContext = DeltaV2QueryContext(Some(queryCatalogTable)) + val kernelEngine = kernelContext.getDefaultEngine() + + def assertRoutesWith( + expectedCatalogTableOpt: Option[CatalogTable])( + operation: => Any): Unit = { + val error = intercept[IllegalStateException](operation) + assert(error.getMessage === "recorded catalog metadata") + assert(observedCatalogTables.poll() === expectedCatalogTableOpt) + } + + val legacyOperations = Seq[() => Any]( + () => manager.loadLatestSnapshot(), + () => manager.loadSnapshotAt(0L), + () => manager.getActiveCommitAtTime(0L, false, false, false), + () => manager.checkVersionExists(0L, false, false), + () => manager.getTableChanges(kernelEngine, 0L, Optional.empty())) + legacyOperations.foreach(operation => + assertRoutesWith(Some(legacyCatalogTable))(operation())) + + def contextualOperations(context: DeltaV2QueryContext): Seq[() => Any] = Seq( + () => manager.loadLatestSnapshot(context), + () => manager.loadSnapshotAt(0L, context), + () => manager.getActiveCommitAtTime(0L, false, false, false, context), + () => manager.checkVersionExists(0L, false, false, context), + () => manager.getTableChanges(kernelEngine, 0L, Optional.empty(), context)) + + contextualOperations(queryContext).foreach(operation => + assertRoutesWith(Some(queryCatalogTable))(operation())) + contextualOperations(DeltaV2QueryContext.empty).foreach(operation => + assertRoutesWith(None)(operation())) + assert(observedCatalogTables.isEmpty) + } finally { + factoryMock.close() + } + } + } + + test("DeltaV2Table operations do not use another query's published catalog metadata") { + withSQLConf( + DeltaSQLConf.DELTA_LOG_CACHE_SIZE.key -> "1000", + DeltaSQLConf.DELTA_ASYNC_UPDATE_STALENESS_TIME_LIMIT.key -> "0") { + withTempDir { dir => + val originalTableName = "query_context_original" + val laterTableName = "query_context_later" + withTable(originalTableName, laterTableName) { + createDeltaTable(dir) + val versionZeroWrittenAtMs = System.currentTimeMillis() + awaitClockAfter(versionZeroWrittenAtMs) + appendToDeltaTable(dir) + val tablePath = dir.getCanonicalPath + sql(s"CREATE TABLE $originalTableName USING DELTA LOCATION '$tablePath'") + sql(s"CREATE TABLE $laterTableName USING DELTA LOCATION '$tablePath'") + + val originalCatalogTable = spark.sessionState.catalog + .getTableMetadata(TableIdentifier(originalTableName)) + val laterCatalogTable = spark.sessionState.catalog + .getTableMetadata(TableIdentifier(laterTableName)) + val observedCatalogTables = new ConcurrentLinkedQueue[Option[CatalogTable]]() + val factoryMock = mockStatic( + classOf[SnapshotManagerFactory], + new Answer[DeltaV2SnapshotManager] { + override def answer(invocation: InvocationOnMock): DeltaV2SnapshotManager = { + val catalogTable = invocation.getArgument[Optional[CatalogTable]](2) + observedCatalogTables.add( + if (catalogTable.isPresent) Some(catalogTable.get()) else None) + invocation.callRealMethod().asInstanceOf[DeltaV2SnapshotManager] + } + }) + DeltaV2TableManagerCache.clearCache() + try { + val originalTable = new DeltaV2Table( + Identifier.of(Array("default"), originalTableName), + originalCatalogTable, + Collections.emptyMap[String, String]()) + val laterTable = new DeltaV2Table( + Identifier.of(Array("default"), laterTableName), + laterCatalogTable, + Collections.emptyMap[String, String]()) + assert(originalTable.getSnapshotManager eq laterTable.getSnapshotManager) + + val laterPublicationAtMs = System.currentTimeMillis() + awaitClockAfter(laterPublicationAtMs) + observedCatalogTables.clear() + val versionZeroMicros = sql(s"DESCRIBE HISTORY $originalTableName") + .where("version = 0") + .select("timestamp") + .head() + .getTimestamp(0) + .getTime * 1000L + assert(originalTable.withTimestamp(versionZeroMicros).version() == "0") + + val observations = observedCatalogTables.asScala.toSeq + assert(observations.nonEmpty) + assert(observations.forall( + _.exists(_.identifier == originalCatalogTable.identifier))) + } finally { + factoryMock.close() + DeltaV2TableManagerCache.clearCache() + } + } + } + } + } + + private def assertWaitsForSnapshotLock(thread: Thread): Unit = { + val deadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(30L) + var stackTrace = thread.getStackTrace.toSeq + while (thread.isAlive && + !stackTrace.exists(_.getMethodName == "lockInterruptibly") && + System.nanoTime() < deadlineNanos) { + Thread.sleep(10L) + stackTrace = thread.getStackTrace.toSeq + } + assert( + stackTrace.exists(_.getMethodName == "lockInterruptibly"), + s"${thread.getName} did not wait for the snapshot lock: ${stackTrace.mkString(", ")}") + } + + // === Cold start ============================================ + + test("cold start loads latest snapshot") { + withTempDir { dir => + createDeltaTable(dir) + val mgr = createManager(dir) + try { + val snapshot = mgr.loadLatestSnapshot() + assert(snapshot != null) + assert(snapshot.version == 0L) + } finally { + mgr.retire() + } + } + } + + test("cold start with multi-version table loads latest") { + withTempDir { dir => + createDeltaTable(dir) + appendToDeltaTable(dir) + appendToDeltaTable(dir) + val mgr = createManager(dir) + try { + val snapshot = mgr.loadLatestSnapshot() + assert(snapshot.version == 2L) + } finally { + mgr.retire() + } + } + } + + // === Warm hit (cache reuse) ================================= + + test("second loadLatestSnapshot reuses cached snapshot at same version") { + withSQLConf(DeltaSQLConf.DELTA_ASYNC_UPDATE_STALENESS_TIME_LIMIT.key -> "60000") { + withTempDir { dir => + createDeltaTable(dir) + val mgr = createManager(dir) + try { + val first = mgr.loadLatestSnapshot() + val second = mgr.loadLatestSnapshot() + assert(first eq second, "Expected same snapshot instance on warm hit") + val firstFileCount = first.allFiles.count() + assert(firstFileCount > 0L) + assert(second.allFiles.count() == firstFileCount) + } finally { + mgr.retire() + } + } + } + } + + // === Staleness triggers full reload ========================== + + test("stale rebuild after append advances version via full reload") { + withSQLConf(DeltaSQLConf.DELTA_ASYNC_UPDATE_STALENESS_TIME_LIMIT.key -> "0") { + withTempDir { dir => + createDeltaTable(dir) + val mgr = createManager(dir) + try { + val snap1 = mgr.loadLatestSnapshot() + assert(snap1.version == 0L) + + appendToDeltaTable(dir) + + val snap2 = mgr.loadLatestSnapshot() + assert(snap2.version == 1L) + assert(snap2 ne snap1, "A newly loaded snapshot must replace the cached facade") + } finally { + mgr.retire() + } + } + } + } + + test("cached snapshot manager preserves KernelContext session invariance") { + withTempDir { dir => + createDeltaTable(dir) + val originalSession = SparkSession.active + val constructionSession = spark.newSession() + val firstOperationSession = spark.newSession() + val secondOperationSession = spark.newSession() + val invariantOptions = Map( + "fs.file.impl" -> classOf[CachedSnapshotManagerRecordingFileSystem].getName, + "fs.file.impl.disable.cache" -> "true", + CachedSnapshotManagerRecordingFileSystem.InvariantMarkerKey -> "table-option") + var manager: CachedSnapshotManager = null + + try { + SparkSession.setActiveSession(constructionSession) + setRecordingMarkers(constructionSession, "construction") + val kernelContext = + KernelContext(invariantOptions, LogStore.createLogStore(constructionSession)) + manager = createManager(dir, kernelContext) + + SparkSession.setActiveSession(firstOperationSession) + setRecordingMarkers(firstOperationSession, "first-operation") + CachedSnapshotManagerRecordingFileSystem.clear() + assert(manager.loadLatestSnapshot().version == 0L) + assertRecordingFileSystemObserved("first-operation") + val retainedEngine = kernelContext.getDefaultEngine() + + appendToDeltaTable(dir) + SparkSession.setActiveSession(secondOperationSession) + setRecordingMarkers(secondOperationSession, "second-operation") + CachedSnapshotManagerRecordingFileSystem.clear() + assert(manager.loadSnapshotAt(1L).version == 1L) + assert(kernelContext.getDefaultEngine() eq retainedEngine) + val expectedSessionMarker = + "first-operation" + assertRecordingFileSystemObserved(expectedSessionMarker) + } finally { + if (manager != null) manager.retire() + SparkSession.setActiveSession(originalSession) + } + } + } + + test("superseded and retired snapshots release persisted state and remain usable") { + withSQLConf(DeltaSQLConf.DELTA_ASYNC_UPDATE_STALENESS_TIME_LIMIT.key -> "0") { + withTempDir { dir => + createDeltaTable(dir) + val mgr = createManager(dir) + try { + val beforePreviousStatsRddIds = spark.sparkContext.getPersistentRDDs.keySet + val previous = mgr.loadLatestSnapshot() + val previousFileCount = previous.allFiles.count() + val previousStatsCount = previous.withStats.count() + val previousRddIds = + spark.sparkContext.getPersistentRDDs.keySet -- beforePreviousStatsRddIds + assert(previousRddIds.nonEmpty) + + appendToDeltaTable(dir) + val current = mgr.loadLatestSnapshot() + assert(previousRddIds.intersect(spark.sparkContext.getPersistentRDDs.keySet).isEmpty) + val beforeCurrentStatsRddIds = spark.sparkContext.getPersistentRDDs.keySet + val currentStatsCount = current.withStats.count() + val currentRddIds = + spark.sparkContext.getPersistentRDDs.keySet -- beforeCurrentStatsRddIds + assert(currentRddIds.nonEmpty) + + assert(previous.version == 0L) + assert(current.version == 1L) + assert(previous.allFiles.count() == previousFileCount) + assert(previous.withStats.count() == previousStatsCount) + assert(current.allFiles.count() > previousFileCount) + + mgr.retire() + assert(currentRddIds.intersect(spark.sparkContext.getPersistentRDDs.keySet).isEmpty) + assert(current.withStats.count() == currentStatsCount) + } finally { + mgr.retire() + } + } + } + } + + // === Version-specific load ================================== + + test("loadSnapshotAt returns specific version") { + withTempDir { dir => + createDeltaTable(dir) + appendToDeltaTable(dir) + val mgr = createManager(dir) + try { + val snapV0 = mgr.loadSnapshotAt(0L) + val snapV1 = mgr.loadSnapshotAt(1L) + assert(snapV0.version == 0L) + assert(snapV1.version == 1L) + assert(mgr.loadLatestSnapshot() eq snapV1) + } finally { + mgr.retire() + } + } + } + + test("loadSnapshotAt reuses the matching cached snapshot") { + withSQLConf(DeltaSQLConf.DELTA_ASYNC_UPDATE_STALENESS_TIME_LIMIT.key -> "0") { + withTempDir { dir => + createDeltaTable(dir) + appendToDeltaTable(dir) + val mgr = createManager(dir) + try { + val cached = mgr.loadLatestSnapshot() + assert(cached.version == 1L) + appendToDeltaTable(dir) + + val loaded = mgr.loadSnapshotAt(1L) + assert(loaded eq cached, "Matching version should reuse the cached snapshot") + assert(mgr.loadLatestSnapshot().version == 2L) + } finally { + mgr.retire() + } + } + } + } + + test("newer loadSnapshotAt refreshes latest despite a non-stale cached snapshot") { + withSQLConf(DeltaSQLConf.DELTA_ASYNC_UPDATE_STALENESS_TIME_LIMIT.key -> "60000") { + withTempDir { dir => + createDeltaTable(dir) + val mgr = createManager(dir) + try { + val cachedLatest = mgr.loadLatestSnapshot() + appendToDeltaTable(dir) + + val versioned = mgr.loadSnapshotAt(1L) + val latestAgain = mgr.loadLatestSnapshot() + assert(versioned.version == 1L) + assert(versioned eq latestAgain) + assert(latestAgain ne cachedLatest) + } finally { + mgr.retire() + } + } + } + } + + test("newer loadSnapshotAt keeps a refreshed later snapshot cached") { + withSQLConf(DeltaSQLConf.DELTA_ASYNC_UPDATE_STALENESS_TIME_LIMIT.key -> "60000") { + withTempDir { dir => + createDeltaTable(dir) + val mgr = createManager(dir) + try { + val cachedLatest = mgr.loadLatestSnapshot() + appendToDeltaTable(dir) + appendToDeltaTable(dir) + + val versioned = mgr.loadSnapshotAt(1L) + val latestAgain = mgr.loadLatestSnapshot() + assert(versioned.version == 1L) + assert(latestAgain.version == 2L) + assert(latestAgain ne versioned) + assert(latestAgain ne cachedLatest) + } finally { + mgr.retire() + } + } + } + } + + test("older loadSnapshotAt preserves a non-stale cached latest snapshot") { + withSQLConf(DeltaSQLConf.DELTA_ASYNC_UPDATE_STALENESS_TIME_LIMIT.key -> "60000") { + withTempDir { dir => + createDeltaTable(dir) + appendToDeltaTable(dir) + val mgr = createManager(dir) + try { + val cachedLatest = mgr.loadLatestSnapshot() + appendToDeltaTable(dir) + + val versioned = mgr.loadSnapshotAt(0L) + val latestAgain = mgr.loadLatestSnapshot() + assert(versioned.version == 0L) + assert(latestAgain eq cachedLatest) + assert(latestAgain.version == 1L) + } finally { + mgr.retire() + } + } + } + } + + test("older loadSnapshotAt refreshes a stale cached latest snapshot") { + withSQLConf(DeltaSQLConf.DELTA_ASYNC_UPDATE_STALENESS_TIME_LIMIT.key -> "0") { + withTempDir { dir => + createDeltaTable(dir) + appendToDeltaTable(dir) + val mgr = createManager(dir) + try { + val cachedLatest = mgr.loadLatestSnapshot() + appendToDeltaTable(dir) + + val versioned = mgr.loadSnapshotAt(0L) + val latestAgain = mgr.loadLatestSnapshot() + assert(versioned.version == 0L) + assert(latestAgain.version == 2L) + assert(latestAgain ne versioned) + assert(latestAgain ne cachedLatest) + } finally { + mgr.retire() + } + } + } + } + + // === Retire lifecycle ======================================= + + Seq("0", "60000").foreach { stalenessLimit => + test(s"retire preserves newer exact-version refresh with staleness $stalenessLimit") { + withSQLConf(DeltaSQLConf.DELTA_ASYNC_UPDATE_STALENESS_TIME_LIMIT.key -> stalenessLimit) { + withTempDir { dir => + createDeltaTable(dir) + val mgr = createManager(dir) + try { + val beforeRetire = mgr.loadLatestSnapshot() + + mgr.retire() + appendToDeltaTable(dir) + + val afterRetire = mgr.loadSnapshotAt(1L) + assert(beforeRetire.version == 0L) + assert(afterRetire.version == 1L) + assert(afterRetire ne beforeRetire) + assert(mgr.loadLatestSnapshot() eq afterRetire) + } finally { + mgr.retire() + } + } + } + } + + test(s"retire preserves newer historical refresh with staleness $stalenessLimit") { + withSQLConf(DeltaSQLConf.DELTA_ASYNC_UPDATE_STALENESS_TIME_LIMIT.key -> stalenessLimit) { + withTempDir { dir => + createDeltaTable(dir) + val mgr = createManager(dir) + try { + val beforeRetire = mgr.loadLatestSnapshot() + + mgr.retire() + appendToDeltaTable(dir) + appendToDeltaTable(dir) + + val historical = mgr.loadSnapshotAt(1L) + val latest = mgr.loadLatestSnapshot() + assert(beforeRetire.version == 0L) + assert(historical.version == 1L) + assert(latest.version == 2L) + assert(latest ne historical) + assert(latest ne beforeRetire) + } finally { + mgr.retire() + } + } + } + } + + test(s"failed exact load preserves latest refresh: $stalenessLimit") { + withSQLConf(DeltaSQLConf.DELTA_ASYNC_UPDATE_STALENESS_TIME_LIMIT.key -> stalenessLimit) { + withTempDir { dir => + createDeltaTable(dir) + val mgr = createManager(dir) + try { + val beforeRetire = mgr.loadLatestSnapshot() + + mgr.retire() + appendToDeltaTable(dir) + + intercept[KernelException] { + mgr.loadSnapshotAt(2L) + } + val latest = mgr.loadLatestSnapshot() + assert(latest.version == 1L) + assert(latest ne beforeRetire) + } finally { + mgr.retire() + } + } + } + } + } + + // === Table identity validation ============================== + + test("table recreation at the same path replaces the cached snapshot") { + withSQLConf(DeltaSQLConf.DELTA_ASYNC_UPDATE_STALENESS_TIME_LIMIT.key -> "0") { + withTempDir { dir => + createDeltaTable(dir) + val mgr = createManager(dir) + try { + val previous = mgr.loadLatestSnapshot() + JavaUtils.deleteRecursively(dir) + createDeltaTable(dir) + + val replacement = mgr.loadLatestSnapshot() + assert(replacement.version == 0L) + assert(replacement.metadata.id != previous.metadata.id) + assert(replacement ne previous) + assert(mgr.loadLatestSnapshot() eq replacement) + } finally { + mgr.retire() + } + } + } + } + + Seq(false, true).foreach { enableICT => + test(s"missing current commit installs previous version, matching DeltaLog: ICT=$enableICT") { + withSQLConf(DeltaSQLConf.DELTA_ASYNC_UPDATE_STALENESS_TIME_LIMIT.key -> "0") { + withTempDir { dir => + spark.range(10).write.format("delta") + .option(DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.key, enableICT.toString) + .save(dir.getCanonicalPath) + val deltaLog = DeltaLog.forTable(spark, new Path(dir.getCanonicalPath)) + val original = deltaLog.update() + val originalFiles = original.allFiles.collect().map(_.path).toSet + appendToDeltaTable(dir) + val manager = createManager(dir) + try { + val cached = manager.loadLatestSnapshot() + assert(cached.version == 1L) + assert(cached.allFiles.collect().map(_.path).toSet.size > originalFiles.size) + val hadoopConf = deltaLog.newDeltaHadoopConf() + val fs = deltaLog.logPath.getFileSystem(hadoopConf) + assert(fs.delete(FileNames.unsafeDeltaFile(deltaLog.logPath, version = 1L), false)) + fs.delete(new Path(deltaLog.logPath, f"${1L}%020d.crc"), false) + val expected = deltaLog.update() + assert(expected.version == 0L) + assert(expected.allFiles.collect().map(_.path).toSet == originalFiles) + + val refreshed = manager.loadLatestSnapshot() + assert(refreshed.version == expected.version) + assert(refreshed.metadata.id == expected.metadata.id) + assert(refreshed.allFiles.collect().map(_.path).toSet == originalFiles) + assert(refreshed ne cached) + assert(manager.loadSnapshotAt(0L) eq refreshed) + + awaitClockAfter(System.currentTimeMillis()) + assert(manager.loadLatestSnapshot() eq refreshed) + } finally { + manager.retire() + } + } + } + } + } + + // === Refresh deduplication ================================== + + test("same-version refresh reuses the cached snapshot") { + withSQLConf(DeltaSQLConf.DELTA_ASYNC_UPDATE_STALENESS_TIME_LIMIT.key -> "0") { + withTempDir { dir => + createDeltaTable(dir) + val mgr = createManager(dir) + try { + val firstSnap = mgr.loadLatestSnapshot() + assert(firstSnap.version == 0L) + + val secondSnap = mgr.loadLatestSnapshot() + assert(firstSnap eq secondSnap, "Same version should keep existing instance") + } finally { + mgr.retire() + } + } + } + } + + test("history, version, and commit-range operations delegate to an uncached manager") { + withTempDir { dir => + createDeltaTable(dir) + val mgr = createManager(dir) + try { + val activeCommit = mgr.getActiveCommitAtTime( + Long.MaxValue, + /* canReturnLastCommit= */ true, + /* mustBeRecreatable= */ true, + /* canReturnEarliestCommit= */ false) + assert(activeCommit.getVersion == 0L) + + mgr.checkVersionExists(0L, mustBeRecreatable = true, allowOutOfRange = false) + + // scalastyle:off deltahadoopconfiguration + val kernelEngine = + KernelEngineFactory.createDefaultEngine(spark.sessionState.newHadoopConf()) + // scalastyle:on deltahadoopconfiguration + val changes = mgr.getTableChanges(kernelEngine, 0L, Optional.empty()) + assert(changes != null) + } finally { + mgr.retire() + } + } + } + + // === Concurrency correctness ================================ + + test("concurrent refreshes serialize and the later caller advances the cached snapshot") { + withSQLConf(DeltaSQLConf.DELTA_ASYNC_UPDATE_STALENESS_TIME_LIMIT.key -> "0") { + withTempDir { dir => + createDeltaTable(dir) + val kernelContext = KernelContext( + Map( + "fs.file.impl" -> classOf[CachedSnapshotManagerBlockingFileSystem].getName, + "fs.file.impl.disable.cache" -> "true"), + LogStore.createLogStore(spark)) + val manager = createManager(dir, kernelContext) + val failures = new ConcurrentLinkedQueue[Throwable]() + val staleResult = new AtomicReference[Snapshot]() + val newerResult = new AtomicReference[Snapshot]() + val secondaryPhase = new CyclicBarrier(2) + val staleThread = new Thread(() => { + try { + staleResult.set(manager.loadLatestSnapshot()) + } catch { + case NonFatal(failure) => failures.add(failure) + } + }, "stale-refresh") + val newerThread = new Thread(() => { + try { + secondaryPhase.await(30L, TimeUnit.SECONDS) + newerResult.set(manager.loadLatestSnapshot()) + } catch { + case NonFatal(failure) => failures.add(failure) + } + }, "newer-refresh") + staleThread.setDaemon(true) + newerThread.setDaemon(true) + + try { + assert(manager.loadLatestSnapshot().version == 0L) + appendToDeltaTable(dir) + CachedSnapshotManagerBlockingFileSystem.arm(staleThread.getName) + staleThread.start() + assert( + CachedSnapshotManagerBlockingFileSystem.awaitCapturedListing(), + "stale refresh did not capture a transaction-log listing") + val staleListingCapturedAtMs = System.currentTimeMillis() + + newerThread.start() + awaitClockAfter(staleListingCapturedAtMs) + secondaryPhase.await(30L, TimeUnit.SECONDS) + appendToDeltaTable(dir) + assertWaitsForSnapshotLock(newerThread) + assert(newerResult.get() == null) + + CachedSnapshotManagerBlockingFileSystem.releaseListing() + staleThread.join(TimeUnit.SECONDS.toMillis(30L)) + newerThread.join(TimeUnit.SECONDS.toMillis(30L)) + assert(!staleThread.isAlive, "stale refresh thread did not terminate") + assert(!newerThread.isAlive, "newer refresh thread did not terminate") + assert(failures.isEmpty, s"Concurrent refresh failed: ${failures.toArray.mkString(", ")}") + assert(staleResult.get().version == 1L) + assert(newerResult.get().version == 2L) + assert(manager.loadSnapshotAt(2L) eq newerResult.get()) + assert( + CachedSnapshotManagerBlockingFileSystem.listingThreadNames.distinct.sorted == + Seq("newer-refresh", "stale-refresh"), + "the later freshness boundary should trigger a second reconstruction") + } finally { + CachedSnapshotManagerBlockingFileSystem.releaseListing() + staleThread.interrupt() + newerThread.interrupt() + manager.retire() + } + } + } + } + + test("concurrent cold latest loads share one refresh across Spark sessions") { + withSQLConf(DeltaSQLConf.DELTA_ASYNC_UPDATE_STALENESS_TIME_LIMIT.key -> "60000") { + withTempDir { dir => + createDeltaTable(dir) + val kernelContext = KernelContext( + Map( + "fs.file.impl" -> classOf[CachedSnapshotManagerBlockingFileSystem].getName, + "fs.file.impl.disable.cache" -> "true"), + LogStore.createLogStore(spark)) + val manager = createManager(dir, kernelContext) + val operationSessions = Seq(spark.newSession(), spark.newSession()) + operationSessions.foreach( + _.conf.set(DeltaSQLConf.DELTA_ASYNC_UPDATE_STALENESS_TIME_LIMIT.key, "60s")) + val startBarrier = new CyclicBarrier(operationSessions.size) + val snapshots = new ConcurrentLinkedQueue[Snapshot]() + val allFileSessions = new ConcurrentLinkedQueue[SparkSession]() + val failures = new ConcurrentLinkedQueue[Throwable]() + val loadsCompleted = new CountDownLatch(operationSessions.size) + val materializeAllFiles = new CountDownLatch(1) + val threads = operationSessions.zipWithIndex.map { case (operationSession, index) => + new Thread(() => { + SparkSession.setActiveSession(operationSession) + try { + startBarrier.await(30L, TimeUnit.SECONDS) + val snapshot = manager.loadLatestSnapshot() + snapshots.add(snapshot) + loadsCompleted.countDown() + assert(materializeAllFiles.await(30L, TimeUnit.SECONDS)) + val allFiles = snapshot.allFiles + assert(allFiles.count() > 0L) + allFileSessions.add(allFiles.sparkSession) + } catch { + case failure: Throwable => failures.add(failure) + } finally { + SparkSession.clearActiveSession() + } + }, s"cold-latest-$index") + } + threads.foreach(_.setDaemon(true)) + + try { + CachedSnapshotManagerBlockingFileSystem.armFirstListing() + threads.foreach(_.start()) + assert( + CachedSnapshotManagerBlockingFileSystem.awaitCapturedListing(), + "cold refresh did not capture a transaction-log listing") + val lockWinner = CachedSnapshotManagerBlockingFileSystem.listingThreadNames.head + assertWaitsForSnapshotLock(threads.find(_.getName != lockWinner).get) + + CachedSnapshotManagerBlockingFileSystem.releaseListing() + assert( + loadsCompleted.await(30L, TimeUnit.SECONDS), + "concurrent cold refreshes did not complete") + assert(failures.isEmpty, s"Concurrent loads failed: ${failures.toArray.mkString(", ")}") + assert(snapshots.size() == operationSessions.size) + val returnedSnapshots = snapshots.asScala.toSeq + assert(returnedSnapshots.map(_.version).distinct == Seq(0L)) + assert(returnedSnapshots.forall(_ eq returnedSnapshots.head)) + assert( + CachedSnapshotManagerBlockingFileSystem.listingThreadNames.distinct.size == 1, + "only the lock winner should reconstruct the cold snapshot") + + materializeAllFiles.countDown() + threads.foreach(_.join(TimeUnit.SECONDS.toMillis(30L))) + assert( + threads.forall(thread => !thread.isAlive), + "concurrent cold refresh threads did not terminate") + assert(failures.isEmpty, s"Concurrent loads failed: ${failures.toArray.mkString(", ")}") + assert(allFileSessions.size() == operationSessions.size) + assert(allFileSessions.asScala.forall(operationSessions.contains)) + assert(allFileSessions.asScala.toSeq.distinct.size == 1) + assert(!allFileSessions.asScala.exists(_ eq spark)) + } finally { + CachedSnapshotManagerBlockingFileSystem.releaseListing() + materializeAllFiles.countDown() + threads.foreach(_.interrupt()) + manager.retire() + } + } + } + } + + test("concurrent cold latest and time-travel loads share one upper-bound refresh") { + withSQLConf(DeltaSQLConf.DELTA_ASYNC_UPDATE_STALENESS_TIME_LIMIT.key -> "60000") { + withTempDir { dir => + createDeltaTable(dir) + val kernelContext = KernelContext( + Map( + "fs.file.impl" -> classOf[CachedSnapshotManagerBlockingFileSystem].getName, + "fs.file.impl.disable.cache" -> "true"), + LogStore.createLogStore(spark)) + val manager = createManager(dir, kernelContext) + val operationSessions = Seq(spark.newSession(), spark.newSession()) + operationSessions.foreach( + _.conf.set(DeltaSQLConf.DELTA_ASYNC_UPDATE_STALENESS_TIME_LIMIT.key, "60s")) + val startBarrier = new CyclicBarrier(operationSessions.size) + val latestResult = new AtomicReference[Snapshot]() + val timeTravelResult = new AtomicReference[Snapshot]() + val failures = new ConcurrentLinkedQueue[Throwable]() + val threads = operationSessions.zipWithIndex.map { case (operationSession, index) => + new Thread(() => { + SparkSession.setActiveSession(operationSession) + try { + startBarrier.await(30L, TimeUnit.SECONDS) + val result = if (index == 0) { + manager.loadLatestSnapshot() + } else { + manager.loadSnapshotAt(0L) + } + if (index == 0) latestResult.set(result) else timeTravelResult.set(result) + } catch { + case failure: Throwable => failures.add(failure) + } finally { + SparkSession.clearActiveSession() + } + }, if (index == 0) "mixed-latest" else "mixed-time-travel") + } + threads.foreach(_.setDaemon(true)) + + try { + CachedSnapshotManagerBlockingFileSystem.armFirstListing() + threads.foreach(_.start()) + assert( + CachedSnapshotManagerBlockingFileSystem.awaitCapturedListing(), + "mixed refresh did not capture a transaction-log listing") + val lockWinner = CachedSnapshotManagerBlockingFileSystem.listingThreadNames.head + assertWaitsForSnapshotLock(threads.find(_.getName != lockWinner).get) + + CachedSnapshotManagerBlockingFileSystem.releaseListing() + threads.foreach(_.join(TimeUnit.SECONDS.toMillis(30L))) + assert( + threads.forall(thread => !thread.isAlive), + "mixed refresh threads did not terminate") + assert(failures.isEmpty, s"Concurrent loads failed: ${failures.toArray.mkString(", ")}") + assert(latestResult.get().version == 0L) + assert(timeTravelResult.get().version == 0L) + assert(latestResult.get() eq timeTravelResult.get()) + assert(manager.loadSnapshotAt(0L) eq latestResult.get()) + assert( + CachedSnapshotManagerBlockingFileSystem.listingThreadNames.distinct.size == 1, + "the waiter should reuse the upper bound installed by the lock winner") + } finally { + CachedSnapshotManagerBlockingFileSystem.releaseListing() + threads.foreach(_.interrupt()) + manager.retire() + } + } + } + } + + test("exact load advances when latest upper bound misses a concurrent append") { + withSQLConf(DeltaSQLConf.DELTA_ASYNC_UPDATE_STALENESS_TIME_LIMIT.key -> "60000") { + withTempDir { dir => + createDeltaTable(dir) + val kernelContext = KernelContext( + Map( + "fs.file.impl" -> classOf[CachedSnapshotManagerBlockingFileSystem].getName, + "fs.file.impl.disable.cache" -> "true"), + LogStore.createLogStore(spark)) + val manager = createManager(dir, kernelContext) + val result = new AtomicReference[Snapshot]() + val failure = new AtomicReference[Throwable]() + val loadThread = new Thread(() => { + try { + result.set(manager.loadSnapshotAt(2L)) + } catch { + case NonFatal(error) => failure.set(error) + } + }, "stale-upper-bound") + loadThread.setDaemon(true) + + try { + assert(manager.loadLatestSnapshot().version == 0L) + appendToDeltaTable(dir) + CachedSnapshotManagerBlockingFileSystem.arm(loadThread.getName) + loadThread.start() + assert( + CachedSnapshotManagerBlockingFileSystem.awaitCapturedListing(), + "latest load did not capture its stale transaction-log listing") + + appendToDeltaTable(dir) + CachedSnapshotManagerBlockingFileSystem.releaseListing() + loadThread.join(TimeUnit.SECONDS.toMillis(30L)) + assert(!loadThread.isAlive, "exact fallback load did not terminate") + assert(failure.get() == null, s"Exact fallback load failed: ${failure.get()}") + assert(result.get().version == 2L) + assert(manager.loadSnapshotAt(2L) eq result.get()) + + val listingsBeforeLatestValidation = + CachedSnapshotManagerBlockingFileSystem.listingThreadNames.size + assert(manager.loadLatestSnapshot() eq result.get()) + assert( + CachedSnapshotManagerBlockingFileSystem.listingThreadNames.size > + listingsBeforeLatestValidation, + "an exact-version installation must not be treated as a validated latest snapshot") + } finally { + CachedSnapshotManagerBlockingFileSystem.releaseListing() + loadThread.interrupt() + manager.retire() + } + } + } + } + + test("concurrent latest and historical loads preserve the installed upper bound") { + withSQLConf(DeltaSQLConf.DELTA_ASYNC_UPDATE_STALENESS_TIME_LIMIT.key -> "0") { + withTempDir { dir => + createDeltaTable(dir) + val kernelContext = KernelContext( + Map( + "fs.file.impl" -> classOf[CachedSnapshotManagerBlockingFileSystem].getName, + "fs.file.impl.disable.cache" -> "true"), + LogStore.createLogStore(spark)) + val manager = createManager(dir, kernelContext) + val initial = manager.loadLatestSnapshot() + appendToDeltaTable(dir) + appendToDeltaTable(dir) + val latestSession = spark.newSession() + val historicalSession = spark.newSession() + val startBarrier = new CyclicBarrier(2) + val latestResult = new AtomicReference[Snapshot]() + val historicalResult = new AtomicReference[Snapshot]() + val latestAllFilesSession = new AtomicReference[SparkSession]() + val historicalAllFilesSession = new AtomicReference[SparkSession]() + val failures = new ConcurrentLinkedQueue[Throwable]() + val latestThread = new Thread(() => { + SparkSession.setActiveSession(latestSession) + try { + startBarrier.await(30L, TimeUnit.SECONDS) + val snapshot = manager.loadLatestSnapshot() + latestResult.set(snapshot) + latestAllFilesSession.set(snapshot.allFiles.sparkSession) + } catch { + case failure: Throwable => failures.add(failure) + } finally { + SparkSession.clearActiveSession() + } + }, "latest-upper-bound") + val historicalThread = new Thread(() => { + SparkSession.setActiveSession(historicalSession) + try { + startBarrier.await(30L, TimeUnit.SECONDS) + val snapshot = manager.loadSnapshotAt(1L) + historicalResult.set(snapshot) + historicalAllFilesSession.set(snapshot.allFiles.sparkSession) + } catch { + case failure: Throwable => failures.add(failure) + } finally { + SparkSession.clearActiveSession() + } + }, "historical-version") + val threads = Seq(latestThread, historicalThread) + threads.foreach(_.setDaemon(true)) + + try { + CachedSnapshotManagerBlockingFileSystem.armFirstListing() + threads.foreach(_.start()) + assert( + CachedSnapshotManagerBlockingFileSystem.awaitCapturedListing(), + "upper-bound refresh did not capture a transaction-log listing") + val lockWinner = CachedSnapshotManagerBlockingFileSystem.listingThreadNames.head + assertWaitsForSnapshotLock(threads.find(_.getName != lockWinner).get) + + CachedSnapshotManagerBlockingFileSystem.releaseListing() + threads.foreach(_.join(TimeUnit.SECONDS.toMillis(30L))) + assert( + threads.forall(thread => !thread.isAlive), + "latest and historical refresh threads did not terminate") + assert(failures.isEmpty, s"Concurrent loads failed: ${failures.toArray.mkString(", ")}") + assert(latestResult.get().version == 2L) + assert(historicalResult.get().version == 1L) + assert(latestResult.get() ne initial) + assert(historicalResult.get() ne latestResult.get()) + assert(manager.loadSnapshotAt(2L) eq latestResult.get()) + assert(latestResult.get().allFiles.count() > historicalResult.get().allFiles.count()) + assert(latestAllFilesSession.get() eq latestSession) + assert(historicalAllFilesSession.get() eq historicalSession) + } finally { + CachedSnapshotManagerBlockingFileSystem.releaseListing() + threads.foreach(_.interrupt()) + manager.retire() + } + } + } + } + + test("retire waits for an in-flight refresh and preserves manager reuse") { + withSQLConf(DeltaSQLConf.DELTA_ASYNC_UPDATE_STALENESS_TIME_LIMIT.key -> "0") { + withTempDir { dir => + createDeltaTable(dir) + val kernelContext = KernelContext( + Map( + "fs.file.impl" -> classOf[CachedSnapshotManagerBlockingFileSystem].getName, + "fs.file.impl.disable.cache" -> "true"), + LogStore.createLogStore(spark)) + val manager = createManager(dir, kernelContext) + manager.loadLatestSnapshot() + appendToDeltaTable(dir) + val refreshed = new AtomicReference[Snapshot]() + val failures = new ConcurrentLinkedQueue[Throwable]() + val refreshThread = new Thread(() => { + SparkSession.setActiveSession(spark) + try { + refreshed.set(manager.loadLatestSnapshot()) + } catch { + case failure: Throwable => failures.add(failure) + } finally { + SparkSession.clearActiveSession() + } + }, "refresh-before-retire") + val retireThread = new Thread(() => { + try { + manager.retire() + } catch { + case failure: Throwable => failures.add(failure) + } + }, "concurrent-retire") + val threads = Seq(refreshThread, retireThread) + threads.foreach(_.setDaemon(true)) + + try { + CachedSnapshotManagerBlockingFileSystem.arm(refreshThread.getName) + refreshThread.start() + assert( + CachedSnapshotManagerBlockingFileSystem.awaitCapturedListing(), + "refresh did not capture a transaction-log listing") + retireThread.start() + assertWaitsForSnapshotLock(retireThread) + + CachedSnapshotManagerBlockingFileSystem.releaseListing() + threads.foreach(_.join(TimeUnit.SECONDS.toMillis(30L))) + assert( + threads.forall(thread => !thread.isAlive), + "refresh and retire threads did not terminate") + assert( + failures.isEmpty, + s"Concurrent lifecycle failed: ${failures.toArray.mkString(", ")}") + assert(refreshed.get().version == 1L) + assert(manager.loadSnapshotAt(1L) eq refreshed.get()) + assert(refreshed.get().allFiles.count() > 0L) + + appendToDeltaTable(dir) + val next = manager.loadSnapshotAt(2L) + assert(next.version == 2L) + assert(next ne refreshed.get()) + } finally { + CachedSnapshotManagerBlockingFileSystem.releaseListing() + threads.foreach(_.interrupt()) + manager.retire() + } + } + } + } + + test("interrupting a snapshot-lock waiter leaves refresh and lock healthy") { + withSQLConf(DeltaSQLConf.DELTA_ASYNC_UPDATE_STALENESS_TIME_LIMIT.key -> "0") { + withTempDir { dir => + createDeltaTable(dir) + val kernelContext = KernelContext( + Map( + "fs.file.impl" -> classOf[CachedSnapshotManagerBlockingFileSystem].getName, + "fs.file.impl.disable.cache" -> "true"), + LogStore.createLogStore(spark)) + val manager = createManager(dir, kernelContext) + manager.loadLatestSnapshot() + appendToDeltaTable(dir) + val refreshed = new AtomicReference[Snapshot]() + val refreshFailure = new AtomicReference[Throwable]() + val waiterFailure = new AtomicReference[Throwable]() + val refreshThread = new Thread(() => { + SparkSession.setActiveSession(spark) + try { + refreshed.set(manager.loadLatestSnapshot()) + } catch { + case failure: Throwable => refreshFailure.set(failure) + } finally { + SparkSession.clearActiveSession() + } + }, "refresh-lock-owner") + val waiterThread = new Thread(() => { + SparkSession.setActiveSession(spark) + try { + manager.loadLatestSnapshot() + } catch { + case failure: Throwable => waiterFailure.set(failure) + } finally { + SparkSession.clearActiveSession() + } + }, "interrupted-lock-waiter") + val threads = Seq(refreshThread, waiterThread) + threads.foreach(_.setDaemon(true)) + + try { + CachedSnapshotManagerBlockingFileSystem.arm(refreshThread.getName) + refreshThread.start() + assert( + CachedSnapshotManagerBlockingFileSystem.awaitCapturedListing(), + "refresh did not capture a transaction-log listing") + waiterThread.start() + assertWaitsForSnapshotLock(waiterThread) + waiterThread.interrupt() + waiterThread.join(TimeUnit.SECONDS.toMillis(30L)) + assert(!waiterThread.isAlive, "interrupted lock waiter did not terminate") + assert(waiterFailure.get().isInstanceOf[InterruptedException]) + + CachedSnapshotManagerBlockingFileSystem.releaseListing() + refreshThread.join(TimeUnit.SECONDS.toMillis(30L)) + assert(!refreshThread.isAlive, "refresh lock owner did not terminate") + assert(refreshFailure.get() == null) + assert(refreshed.get().version == 1L) + + appendToDeltaTable(dir) + val next = manager.loadSnapshotAt(2L) + assert(next.version == 2L) + assert(next ne refreshed.get()) + } finally { + CachedSnapshotManagerBlockingFileSystem.releaseListing() + threads.foreach(_.interrupt()) + manager.retire() + } + } + } + } + + test("concurrent appends and loadLatestSnapshot calls observe monotonically newer versions") { + withSQLConf(DeltaSQLConf.DELTA_ASYNC_UPDATE_STALENESS_TIME_LIMIT.key -> "0") { + withTempDir { dir => + createDeltaTable(dir) + val mgr = createManager(dir) + try { + val versions = 1L to 3L + val numReaders = 4 + val failures = new ConcurrentLinkedQueue[Throwable]() + val maxVersionSeen = new AtomicLong(mgr.loadLatestSnapshot().version) + val roundStarted = new CyclicBarrier(numReaders + 1) + val roundFinished = new CyclicBarrier(numReaders + 1) + val readers = (1 to numReaders).map { _ => + new Thread(() => { + try { + var lastSeenVersion = -1L + versions.foreach { _ => + roundStarted.await(30L, TimeUnit.SECONDS) + val minimumVersion = maxVersionSeen.get() + try { + val observedVersion = mgr.loadLatestSnapshot().version + assert(observedVersion >= lastSeenVersion) + assert(observedVersion >= minimumVersion) + lastSeenVersion = observedVersion + maxVersionSeen.getAndUpdate(current => math.max(current, observedVersion)) + } finally { + roundFinished.await(30L, TimeUnit.SECONDS) + } + } + } catch { + case failure: Throwable => failures.add(failure) + } + }) + } + val appender = new Thread(() => { + try { + versions.foreach { _ => + roundStarted.await(30L, TimeUnit.SECONDS) + try { + appendToDeltaTable(dir) + } finally { + roundFinished.await(30L, TimeUnit.SECONDS) + } + } + } catch { + case failure: Throwable => failures.add(failure) + } + }) + val threads = appender +: readers + startAndJoinThreads(threads) + + assert(failures.isEmpty, s"Concurrent loads failed: ${failures.toArray.mkString(", ")}") + val latest = mgr.loadLatestSnapshot() + assert(latest.version == 3L) + assert(latest.allFiles.count() > 0L) + assert(latest.version >= maxVersionSeen.get()) + } finally { + mgr.retire() + } + } + } + } + + test("concurrent cold loads return one table identity") { + withSQLConf(DeltaSQLConf.DELTA_ASYNC_UPDATE_STALENESS_TIME_LIMIT.key -> "0") { + withTempDir { dir => + createDeltaTable(dir) + val mgr = createManager(dir) + try { + val startBarrier = new CyclicBarrier(8) + val snapshots = new ConcurrentLinkedQueue[Snapshot]() + val failures = new ConcurrentLinkedQueue[Throwable]() + val threads = (1 to 8).map { _ => + new Thread(() => { + try { + startBarrier.await(30L, TimeUnit.SECONDS) + snapshots.add(mgr.loadLatestSnapshot()) + } catch { + case failure: Throwable => failures.add(failure) + } + }) + } + startAndJoinThreads(threads) + + assert(failures.isEmpty, s"Concurrent loads failed: ${failures.toArray.mkString(", ")}") + assert(snapshots.size() == threads.size) + val tableIds = Seq.newBuilder[String] + while (!snapshots.isEmpty) { + tableIds += snapshots.poll().metadata.id + } + assert(tableIds.result().distinct.size == 1) + } finally { + mgr.retire() + } + } + } + } + + test("concurrent versioned loads advance and preserve the latest cached snapshot") { + withSQLConf(DeltaSQLConf.DELTA_ASYNC_UPDATE_STALENESS_TIME_LIMIT.key -> "60000") { + withTempDir { dir => + createDeltaTable(dir) + val mgr = createManager(dir) + try { + val initial = mgr.loadLatestSnapshot() + appendToDeltaTable(dir) + appendToDeltaTable(dir) + val currentResults = new ConcurrentLinkedQueue[Snapshot]() + val intermediateResults = new ConcurrentLinkedQueue[Snapshot]() + val historicalResults = new ConcurrentLinkedQueue[Snapshot]() + val failures = new ConcurrentLinkedQueue[Throwable]() + val numThreads = 12 + val startBarrier = new CyclicBarrier(numThreads) + + val threads = (1 to numThreads).map { index => + new Thread(() => { + try { + startBarrier.await(30L, TimeUnit.SECONDS) + val requestedVersion = index % 3 + val result = mgr.loadSnapshotAt(requestedVersion) + assert(result.version == requestedVersion) + requestedVersion match { + case 0 => historicalResults.add(result) + case 1 => intermediateResults.add(result) + case 2 => currentResults.add(result) + } + } catch { + case failure: Throwable => failures.add(failure) + } + }) + } + startAndJoinThreads(threads) + + assert(failures.isEmpty, s"Concurrent loads failed: ${failures.toArray.mkString(", ")}") + assert(currentResults.size() == 4) + assert(intermediateResults.size() == 4) + assert(historicalResults.size() == 4) + val cachedLatest = mgr.loadLatestSnapshot() + assert(cachedLatest.version == 2L) + assert(cachedLatest ne initial) + assert(cachedLatest.allFiles.count() > 0L) + while (!currentResults.isEmpty) assert(currentResults.poll() eq cachedLatest) + while (!intermediateResults.isEmpty) assert(intermediateResults.poll().version == 1L) + while (!historicalResults.isEmpty) assert(historicalResults.poll().version == 0L) + assert(mgr.loadLatestSnapshot() eq cachedLatest) + } finally { + mgr.retire() + } + } + } + } +} + +private[tablemanager] object CachedSnapshotManagerRecordingFileSystem { + val SessionMarkerKey = "fs.cached-snapshot-manager-test.session-marker" + val InvariantMarkerKey = "fs.cached-snapshot-manager-test.table-option" + + private val observations = new CopyOnWriteArrayList[(String, String, String)]() + + def clear(): Unit = observations.clear() + + def record(operation: String, sessionMarker: String, invariantMarker: String): Unit = + observations.add((operation, sessionMarker, invariantMarker)) + + def currentThreadObservations: Seq[(String, String, String)] = + observations.asScala.toSeq +} + +private[tablemanager] class CachedSnapshotManagerRecordingFileSystem extends RawLocalFileSystem { + private var sessionMarker: String = _ + private var invariantMarker: String = _ + + override def initialize(name: URI, hadoopConf: Configuration): Unit = { + sessionMarker = + hadoopConf.get(CachedSnapshotManagerRecordingFileSystem.SessionMarkerKey) + invariantMarker = + hadoopConf.get(CachedSnapshotManagerRecordingFileSystem.InvariantMarkerKey) + super.initialize(name, hadoopConf) + } + + override def open(path: Path, bufferSize: Int): FSDataInputStream = { + record("open") + super.open(path, bufferSize) + } + + override def getFileStatus(path: Path): FileStatus = { + record("getFileStatus") + super.getFileStatus(path) + } + + override def listStatus(path: Path): Array[FileStatus] = { + record("listStatus") + super.listStatus(path) + } + + private def record(operation: String): Unit = + CachedSnapshotManagerRecordingFileSystem.record( + operation, + sessionMarker, + invariantMarker) +} + +private[tablemanager] object CachedSnapshotManagerBlockingFileSystem { + private val AnyThread = "*" + private val targetThreadName = new AtomicReference[String]() + private val listingThreads = new CopyOnWriteArrayList[String]() + @volatile private var capturedListing = new CountDownLatch(0) + @volatile private var releaseCapturedListing = new CountDownLatch(0) + + def arm(threadName: String): Unit = synchronized { + listingThreads.clear() + targetThreadName.set(threadName) + capturedListing = new CountDownLatch(1) + releaseCapturedListing = new CountDownLatch(1) + } + + def armFirstListing(): Unit = arm(AnyThread) + + def listingThreadNames: Seq[String] = listingThreads.asScala.toSeq + + def awaitCapturedListing(): Boolean = + capturedListing.await(30L, TimeUnit.SECONDS) + + def releaseListing(): Unit = + releaseCapturedListing.countDown() + + def blockAfterListing(path: Path): Unit = { + if (path.getName == "_delta_log") { + listingThreads.add(Thread.currentThread().getName) + } + val expectedThreadName = targetThreadName.get() + if (path.getName == "_delta_log" && + (expectedThreadName == AnyThread || expectedThreadName == Thread.currentThread().getName) && + targetThreadName.compareAndSet(expectedThreadName, null)) { + capturedListing.countDown() + if (!releaseCapturedListing.await(30L, TimeUnit.SECONDS)) { + throw new IllegalStateException( + "Timed out waiting to release captured transaction-log list") + } + } + } +} + +private[tablemanager] class CachedSnapshotManagerBlockingFileSystem extends RawLocalFileSystem { + override def listStatus(path: Path): Array[FileStatus] = { + val statuses = super.listStatus(path) + CachedSnapshotManagerBlockingFileSystem.blockAfterListing(path) + statuses + } +} diff --git a/spark/v2/src/test/scala/io/delta/spark/internal/v2/tablemanager/DeltaV2TableManagerImplSuite.scala b/spark/v2/src/test/scala/io/delta/spark/internal/v2/tablemanager/DeltaV2TableManagerImplSuite.scala new file mode 100644 index 00000000000..81340aea928 --- /dev/null +++ b/spark/v2/src/test/scala/io/delta/spark/internal/v2/tablemanager/DeltaV2TableManagerImplSuite.scala @@ -0,0 +1,124 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.spark.internal.v2.tablemanager + +import java.io.File +import java.util.Collections + +import org.apache.spark.sql.delta.sources.DeltaSQLConf + +import org.apache.spark.SparkConf +import org.apache.spark.sql.QueryTest +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable, CatalogTableType} +import org.apache.spark.sql.test.SharedSparkSession +import org.apache.spark.sql.types.StructType + +class DeltaV2TableManagerImplSuite + extends QueryTest + with SharedSparkSession { + + private val sparkCatalogKey = "spark.sql.catalog.spark_catalog" + private val deltaCatalogV1ClassName = "org.apache.spark.sql.delta.catalog.DeltaCatalogV1" + + override protected def sparkConf: SparkConf = super.sparkConf + .set("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") + .set(sparkCatalogKey, deltaCatalogV1ClassName) + + override def beforeEach(): Unit = { + super.beforeEach() + DeltaV2TableManagerCache.clearCache() + } + + override def afterEach(): Unit = { + DeltaV2TableManagerCache.clearCache() + super.afterEach() + } + + private def forPathAndCatalogManagers( + path: String)(testFn: (DeltaV2TableManagerImpl, Option[CatalogTable]) => Unit): Unit = { + val catalogTable = CatalogTable( + identifier = TableIdentifier("test_table"), + tableType = CatalogTableType.EXTERNAL, + storage = CatalogStorageFormat.empty.copy(locationUri = Some(new File(path).toURI)), + schema = new StructType()) + Seq("path-based" -> None, "catalog-backed" -> Some(catalogTable)).foreach { + case (label, catalogTableOpt) => + DeltaV2TableManagerCache.clearCache() + withClue(s"$label manager: ") { + val manager = DeltaV2TableManagerCache + .forTable(spark, path, Collections.emptyMap(), catalogTableOpt) + .asInstanceOf[DeltaV2TableManagerImpl] + assert(manager.initialCatalogTableOpt === catalogTableOpt) + testFn(manager, catalogTableOpt) + } + } + } + + test("snapshotManager loads table history through path and catalog managers") { + withTempDir { dir => + val path = dir.getCanonicalPath + spark.range(0, 1, 1, 1).write.format("delta").save(path) + spark.range(1, 2, 1, 1).write.format("delta").mode("append").save(path) + spark.range(2, 3, 1, 1).write.format("delta").mode("append").save(path) + + forPathAndCatalogManagers(path) { (manager, catalogTableOpt) => + val kernelEngine = manager.kernelContext.getDefaultEngine() + val atVersionZeroManager = manager.snapshotManager(catalogTableOpt) + val atVersionOneManager = manager.snapshotManager(catalogTableOpt) + val latestManager = manager.snapshotManager(catalogTableOpt) + + assert(atVersionZeroManager eq atVersionOneManager) + assert(atVersionOneManager eq latestManager) + assert(manager.kernelContext.getDefaultEngine() eq kernelEngine) + + val atVersionZero = atVersionZeroManager.loadSnapshotAt(0) + assert(atVersionZero.version == 0) + assert(atVersionZero.allFiles.count() == 1) + + val atVersionOne = atVersionOneManager.loadSnapshotAt(1) + assert(atVersionOne.version == 1) + assert(atVersionOne.allFiles.count() == 2) + + val latest = latestManager.loadLatestSnapshot() + assert(latest.version == 2) + assert(latest.allFiles.count() == 3) + } + } + } + + test("cached composite reuses the table manager and Kernel Engine") { + withSQLConf(DeltaSQLConf.DELTA_LOG_CACHE_SIZE.key -> "1000") { + withTempDir { dir => + val first = DeltaV2TableManagerCache + .forTable( + spark, + dir.getCanonicalPath, + Collections.emptyMap()) + .asInstanceOf[DeltaV2TableManagerImpl] + val second = DeltaV2TableManagerCache + .forTable( + spark, + dir.getCanonicalPath, + Collections.emptyMap()) + .asInstanceOf[DeltaV2TableManagerImpl] + + assert(first eq second) + assert(first.kernelContext.getDefaultEngine() eq second.kernelContext.getDefaultEngine()) + } + } + } +} diff --git a/spark/v2/src/test/scala/org/apache/spark/sql/delta/v2/CDCDeletionVectorHelperSuite.scala b/spark/v2/src/test/scala/org/apache/spark/sql/delta/v2/CDCDeletionVectorHelperSuite.scala index 66f4b6516b7..03553503cac 100644 --- a/spark/v2/src/test/scala/org/apache/spark/sql/delta/v2/CDCDeletionVectorHelperSuite.scala +++ b/spark/v2/src/test/scala/org/apache/spark/sql/delta/v2/CDCDeletionVectorHelperSuite.scala @@ -15,16 +15,16 @@ */ package org.apache.spark.sql.delta.v2 -import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.fs.Path - -import org.apache.spark.SparkFunSuite import org.apache.spark.sql.delta.actions.{DeletionVectorDescriptor => SparkDvDescriptor} import org.apache.spark.sql.delta.commands.DeletionVectorUtils import org.apache.spark.sql.delta.commands.cdc.CDCReader import org.apache.spark.sql.delta.deletionvectors.{RoaringBitmapArray, RoaringBitmapArrayFormat} import org.apache.spark.sql.delta.storage.dv.DeletionVectorStore +import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.fs.Path +import org.apache.spark.SparkFunSuite + /** Unit tests for [[CDCDeletionVectorHelper.computeDVDiff]]. All tests use inline DVs. */ class CDCDeletionVectorHelperSuite extends SparkFunSuite { diff --git a/storage/src/main/java/io/delta/storage/HDFSLogStore.java b/storage/src/main/java/io/delta/storage/HDFSLogStore.java index af5106bcf2f..d04248835f5 100644 --- a/storage/src/main/java/io/delta/storage/HDFSLogStore.java +++ b/storage/src/main/java/io/delta/storage/HDFSLogStore.java @@ -42,6 +42,7 @@ */ public class HDFSLogStore extends HadoopFileSystemLogStore { private static final Logger LOG = LoggerFactory.getLogger(HDFSLogStore.class); + private static final Object LOCAL_FS_WRITE_LOCK = new Object(); public static final String NO_ABSTRACT_FILE_SYSTEM_EXCEPTION_MESSAGE = "No AbstractFileSystem"; public HDFSLogStore(Configuration hadoopConf) { @@ -58,8 +59,9 @@ public void write( if (isLocalFs) { // We need to add `synchronized` for RawLocalFileSystem as its rename will not throw an // exception when the target file exists. Hence we must make sure `exists + rename` in - // `writeInternal` for RawLocalFileSystem is atomic in our tests. - synchronized(this) { + // `writeInternal` for RawLocalFileSystem is atomic across log store instances in our + // tests. + synchronized(LOCAL_FS_WRITE_LOCK) { writeInternal(path, actions, overwrite, hadoopConf); } } else { diff --git a/storage/src/main/java/io/delta/storage/commit/uccommitcoordinator/UCDeltaModels.java b/storage/src/main/java/io/delta/storage/commit/uccommitcoordinator/UCDeltaModels.java index 2520d80ec13..9393f867272 100644 --- a/storage/src/main/java/io/delta/storage/commit/uccommitcoordinator/UCDeltaModels.java +++ b/storage/src/main/java/io/delta/storage/commit/uccommitcoordinator/UCDeltaModels.java @@ -114,6 +114,7 @@ public static final class TableInfo { private final String location; private final AbstractMetadata metadata; private final Map storageProperties; + private final List clientMaintenanceOperations; private final Optional uniformMetadata; public TableInfo( @@ -122,12 +123,14 @@ public TableInfo( String location, AbstractMetadata metadata, Map storageProperties, + List clientMaintenanceOperations, Optional uniformMetadata) { this.tableId = tableId; this.tableType = tableType; this.location = location; this.metadata = metadata; this.storageProperties = storageProperties; + this.clientMaintenanceOperations = clientMaintenanceOperations; this.uniformMetadata = uniformMetadata; } @@ -153,6 +156,13 @@ public Map getStorageProperties() { return storageProperties == null ? Collections.emptyMap() : storageProperties; } + /** Maintenance operations that the catalog permits the client to run. */ + public List getClientMaintenanceOperations() { + return clientMaintenanceOperations == null + ? Collections.emptyList() + : clientMaintenanceOperations; + } + /** UniForm conversion metadata, or empty if the table has no UniForm enabled. */ public Optional getUniformMetadata() { return uniformMetadata; diff --git a/storage/src/main/java/io/delta/storage/commit/uccommitcoordinator/UCDeltaTokenBasedRestClient.java b/storage/src/main/java/io/delta/storage/commit/uccommitcoordinator/UCDeltaTokenBasedRestClient.java index 36531f2b35d..e5c10845cad 100644 --- a/storage/src/main/java/io/delta/storage/commit/uccommitcoordinator/UCDeltaTokenBasedRestClient.java +++ b/storage/src/main/java/io/delta/storage/commit/uccommitcoordinator/UCDeltaTokenBasedRestClient.java @@ -48,6 +48,7 @@ import io.unitycatalog.client.delta.model.DeltaCreateTableRequest; import io.unitycatalog.client.delta.model.DeltaDomainMetadataUpdates; import io.unitycatalog.client.delta.model.DeltaLoadTableResponse; +import io.unitycatalog.client.delta.model.DeltaMaintenanceOperation; import io.unitycatalog.client.delta.model.DeltaProtocol; import io.unitycatalog.client.delta.model.DeltaRemoveDomainMetadataUpdate; import io.unitycatalog.client.delta.model.DeltaRemovePropertiesUpdate; @@ -88,6 +89,7 @@ import java.util.Optional; import java.util.UUID; import java.util.function.Supplier; +import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; @@ -617,9 +619,23 @@ private TableInfo toTableInfo( AdaptedTableMetadata adapted = new AdaptedTableMetadata(name, m); Optional uniformMetadata = toStorageUniformMetadata(response.getUniform()); + List responseMaintenanceOperations = + response.getAllowedMaintenanceOperations(); + List clientMaintenanceOperations = + responseMaintenanceOperations == null + ? Collections.emptyList() + : responseMaintenanceOperations.stream() + .map(DeltaMaintenanceOperation::getValue) + .collect(Collectors.toList()); if (!credentialVendingEnabled) { return new TableInfo( - ucTableId, tableType, location, adapted, Collections.emptyMap(), uniformMetadata); + ucTableId, + tableType, + location, + adapted, + Collections.emptyMap(), + clientMaintenanceOperations, + uniformMetadata); } Map storageProps; try { @@ -629,13 +645,26 @@ private TableInfo toTableInfo( // recover. The exception carries the catalog-side TableInfo (with empty storageProperties) // so the caller can still build a CatalogTable. TableInfo withoutCreds = new TableInfo( - ucTableId, tableType, location, adapted, Collections.emptyMap(), uniformMetadata); + ucTableId, + tableType, + location, + adapted, + Collections.emptyMap(), + clientMaintenanceOperations, + uniformMetadata); throw new CredentialFetchFailedException( String.format("Credential fetch failed for table %s.%s.%s (HTTP %s): %s", catalog, schema, name, e.getCode(), e.getResponseBody()), e, withoutCreds); } - return new TableInfo(ucTableId, tableType, location, adapted, storageProps, uniformMetadata); + return new TableInfo( + ucTableId, + tableType, + location, + adapted, + storageProps, + clientMaintenanceOperations, + uniformMetadata); } private static Optional toStorageUniformMetadata( diff --git a/storage/src/main/java/io/delta/storage/internal/S3LogStoreUtil.java b/storage/src/main/java/io/delta/storage/internal/S3LogStoreUtil.java index f4a6787842a..c7afa140104 100644 --- a/storage/src/main/java/io/delta/storage/internal/S3LogStoreUtil.java +++ b/storage/src/main/java/io/delta/storage/internal/S3LogStoreUtil.java @@ -27,6 +27,7 @@ import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_MAX_PAGING_KEYS; import static org.apache.hadoop.fs.s3a.Constants.MAX_PAGING_KEYS; import static org.apache.hadoop.fs.s3a.S3AUtils.iteratorToStatuses; +import static org.apache.hadoop.fs.s3a.S3AUtils.maybeAddTrailingSlash; /** @@ -66,7 +67,8 @@ private static RemoteIterator s3ListFrom( ListObjectsV2Request.builder() .bucket(s3afs.getBucket()) .maxKeys(maxKeys) - .prefix(s3afs.pathToKey(parentPath)) + .prefix(maybeAddTrailingSlash(s3afs.pathToKey(parentPath))) + .delimiter("/") .startAfter(keyBefore(s3afs.pathToKey(resolvedPath))) .build() ), ACCEPT_ALL, diff --git a/storage/src/test/scala/io/delta/storage/commit/uccommitcoordinator/UCDeltaTokenBasedRestClientSuite.scala b/storage/src/test/scala/io/delta/storage/commit/uccommitcoordinator/UCDeltaTokenBasedRestClientSuite.scala index 189e1e9642d..c61d78adca4 100644 --- a/storage/src/test/scala/io/delta/storage/commit/uccommitcoordinator/UCDeltaTokenBasedRestClientSuite.scala +++ b/storage/src/test/scala/io/delta/storage/commit/uccommitcoordinator/UCDeltaTokenBasedRestClientSuite.scala @@ -24,6 +24,7 @@ import java.util.concurrent.atomic.AtomicInteger import scala.jdk.CollectionConverters._ import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.node.ObjectNode import com.sun.net.httpserver.{HttpExchange, HttpServer} import io.delta.storage.commit.{Commit, CommitFailedException, TableIdentifier} import io.delta.storage.commit.actions.{AbstractDomainMetadata, AbstractMetadata, AbstractProtocol} @@ -228,6 +229,27 @@ class UCDeltaTokenBasedRestClientSuite assert(parsed.get("fields").size() === 2) assert(parsed.get("fields").get(0).get("name").asText() === "date") assert(parsed.get("fields").get(1).get("type").asText() === "integer") + assert(info.getClientMaintenanceOperations.isEmpty) + assert(!info.getStorageProperties.containsKey("delta.clientMaintenanceOperations")) + } + } + + test("loadTable returns recognized client maintenance operations") { + val response = objectMapper.readTree(loadTableJson()).asInstanceOf[ObjectNode] + response.putArray("allowed-maintenance-operations") + .add("DATA_REORGANIZATION") + .add("DATA_CLEANUP") + .add("METADATA_CLEANUP") + deltaHandler = (exchange, _) => sendJson( + exchange, + HttpStatus.SC_OK, + objectMapper.writeValueAsString(response)) + + withClient { c => + val info = c.loadTable(testIdentifier) + assert(info.getClientMaintenanceOperations.asScala.toSeq === Seq( + "DATA_REORGANIZATION", "DATA_CLEANUP", "METADATA_CLEANUP")) + assert(!info.getStorageProperties.containsKey("delta.clientMaintenanceOperations")) } } diff --git a/version.sbt b/version.sbt index 7f7a7893cf0..69e6e6a43ef 100644 --- a/version.sbt +++ b/version.sbt @@ -1 +1 @@ -ThisBuild / version := "4.4.0-SNAPSHOT" +ThisBuild / version := "4.5.0-SNAPSHOT"