From 60c440189f0c172d96b0b6466b2c735d2a8f0934 Mon Sep 17 00:00:00 2001 From: gitRasheed <99349713+gitRasheed@users.noreply.github.com> Date: Thu, 22 Oct 2026 20:50:55 +0100 Subject: [PATCH 1/5] fix(spark): return the requested field when Spark prunes nested struct fields With spark.sql.optimizer.nestedSchemaPruning.enabled (the Spark default), pruneColumns receives struct columns with unused fields removed. The native scan only projects top-level columns and returns the full struct, but readSchema() reported the pruned type, so Spark bound nested fields by the pruned ordinals and silently returned the wrong field: selecting s.c from s{a,b,c} returned s.a. Keep each required column's full table type, so readSchema() matches the data the reader returns. Nested pruning in the native scan can follow. Signed-off-by: gitRasheed <99349713+gitRasheed@users.noreply.github.com> --- .../vortex/spark/read/VortexScanBuilder.java | 15 ++- .../spark/VortexNestedSchemaPruningTest.java | 115 ++++++++++++++++++ 2 files changed, 127 insertions(+), 3 deletions(-) create mode 100644 java/vortex-spark/src/test/java/dev/vortex/spark/VortexNestedSchemaPruningTest.java diff --git a/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexScanBuilder.java b/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexScanBuilder.java index 62c8085aa0f..b623ee80536 100644 --- a/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexScanBuilder.java +++ b/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexScanBuilder.java @@ -131,15 +131,24 @@ public Scan build() { /** * Prunes the columns to only include those specified in the required schema. * - *

This method clears the current column list and replaces it with columns derived from the required schema. - * Currently only supports top-level schema pruning - deeply nested schema pruning is not yet implemented. + *

Only top-level pruning is supported. With {@code spark.sql.optimizer.nestedSchemaPruning.enabled} (the + * default), Spark may pass a struct column with some of its fields removed, but the native scan projects by + * top-level name and returns the whole struct. Each required column therefore keeps its full table type, so + * {@link VortexScan#readSchema()} matches the data the reader returns and Spark binds nested fields by the right + * ordinal. Reporting the pruned type instead makes Spark read the wrong struct field. * * @param requiredSchema the schema specifying which columns are required */ @Override public void pruneColumns(StructType requiredSchema) { + Map byName = new HashMap<>(); + for (Column column : tableColumns) { + byName.put(column.name(), column); + } readColumns.clear(); - readColumns.addAll(Arrays.asList(CatalogV2Util.structTypeToV2Columns(requiredSchema))); + for (Column required : CatalogV2Util.structTypeToV2Columns(requiredSchema)) { + readColumns.add(byName.getOrDefault(required.name(), required)); + } } /** diff --git a/java/vortex-spark/src/test/java/dev/vortex/spark/VortexNestedSchemaPruningTest.java b/java/vortex-spark/src/test/java/dev/vortex/spark/VortexNestedSchemaPruningTest.java new file mode 100644 index 00000000000..4527dc3a2e1 --- /dev/null +++ b/java/vortex-spark/src/test/java/dev/vortex/spark/VortexNestedSchemaPruningTest.java @@ -0,0 +1,115 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +package dev.vortex.spark; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.nio.file.Path; +import java.util.List; +import java.util.stream.Collectors; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.SaveMode; +import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.functions; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +/** + * Tests that selecting nested fields returns the requested field. + * + *

With {@code spark.sql.optimizer.nestedSchemaPruning.enabled} (the Spark default), Spark passes struct columns to + * {@code pruneColumns} with unused fields removed. The native scan only prunes top-level columns, so the scan must + * report the full struct type; otherwise Spark reads the pruned field ordinals out of the full struct and returns the + * wrong field. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public final class VortexNestedSchemaPruningTest { + + private SparkSession spark; + + @TempDir + Path tempDir; + + @BeforeAll + public void setUp() { + spark = SparkSession.builder() + .appName("VortexNestedSchemaPruningTest") + .master("local[2]") + .config("spark.driver.host", "127.0.0.1") + .config("spark.sql.shuffle.partitions", "2") + .config("spark.sql.adaptive.enabled", "false") + .config("spark.ui.enabled", "false") + .getOrCreate(); + } + + @AfterAll + public void tearDown() { + if (spark != null) { + spark.stop(); + } + } + + private Dataset writeAndRead(String name) { + Dataset df = spark.range(5) + .select( + functions.col("id"), + functions + .struct( + functions.col("id").plus(100).alias("a"), + functions.col("id").plus(200).alias("b"), + functions.col("id").plus(300).alias("c")) + .alias("s")); + df = df.withColumn("arr", functions.array(functions.col("s"))) + .withColumn( + "outer", + functions.struct( + functions.lit("x").alias("p"), + functions.col("s").alias("inner"))); + String path = tempDir.resolve(name).toUri().toString(); + df.write() + .format("vortex") + .option("path", path) + .mode(SaveMode.Overwrite) + .save(); + return spark.read().format("vortex").option("path", path).load(); + } + + private static List rows(Dataset df) { + return df.orderBy("id").collectAsList().stream().map(Row::toString).collect(Collectors.toList()); + } + + @ParameterizedTest(name = "nestedSchemaPruning={0}") + @ValueSource(strings = {"true", "false"}) + @DisplayName("Selecting a non-first struct field returns that field") + public void selectNestedField(String nestedSchemaPruning) { + spark.conf().set("spark.sql.optimizer.nestedSchemaPruning.enabled", nestedSchemaPruning); + Dataset df = writeAndRead("struct_" + nestedSchemaPruning); + + assertEquals(List.of("[0,300]", "[1,301]", "[2,302]", "[3,303]", "[4,304]"), rows(df.select("id", "s.c"))); + assertEquals( + List.of("[0,200,300]", "[1,201,301]", "[2,202,302]", "[3,203,303]", "[4,204,304]"), + rows(df.select("id", "s.b", "s.c"))); + assertEquals(List.of("[3,303]", "[4,304]"), rows(df.filter("s.c >= 303").select("id", "s.c"))); + } + + @Test + @DisplayName("Selecting a field of a struct inside an array or another struct returns that field") + public void selectDeeplyNestedField() { + spark.conf().set("spark.sql.optimizer.nestedSchemaPruning.enabled", "true"); + Dataset df = writeAndRead("deep"); + + assertEquals( + List.of("[0,300]", "[1,301]", "[2,302]", "[3,303]", "[4,304]"), + rows(df.select(functions.col("id"), functions.element_at(functions.col("arr.c"), 1)))); + assertEquals( + List.of("[0,300]", "[1,301]", "[2,302]", "[3,303]", "[4,304]"), rows(df.select("id", "outer.inner.c"))); + } +} From 8287ea02cd88fbe1188d6940d7933770690d74b7 Mon Sep 17 00:00:00 2001 From: gitRasheed <99349713+gitRasheed@users.noreply.github.com> Date: Thu, 22 Oct 2026 21:10:19 +0100 Subject: [PATCH 2/5] fix(spark): cover array and map fields, keep table column order Keep the table's columns that Spark requires instead of rebuilding them from the required schema, so a pruned type can't slip back in. Test struct, array, map and struct-in-struct fields, and document that pruning is top-level only. Signed-off-by: gitRasheed <99349713+gitRasheed@users.noreply.github.com> --- docs/user-guide/spark.md | 2 + .../vortex/spark/read/VortexScanBuilder.java | 20 ++-- .../spark/VortexNestedSchemaPruningTest.java | 103 ++++++++---------- 3 files changed, 56 insertions(+), 69 deletions(-) diff --git a/docs/user-guide/spark.md b/docs/user-guide/spark.md index 0410b709aa6..50692078ba7 100644 --- a/docs/user-guide/spark.md +++ b/docs/user-guide/spark.md @@ -78,6 +78,8 @@ When pointed at a directory, the connector discovers all `.vortex` files and cre partition per file. Column pruning is pushed down — only the columns referenced by the query are read from the file. +Pruning applies to top-level columns: a query that uses one field of a struct, array or map +column reads the whole column. Filter pushdown is also supported. Comparisons (`=`, `<>`, `>`, `>=`, `<`, `<=`), `IS NULL`, `IS NOT NULL`, `IN`, the string predicates `STARTS_WITH`, `ENDS_WITH` and `CONTAINS`, and a bare diff --git a/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexScanBuilder.java b/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexScanBuilder.java index b623ee80536..e9e85962762 100644 --- a/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexScanBuilder.java +++ b/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexScanBuilder.java @@ -15,7 +15,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import org.apache.spark.sql.connector.catalog.CatalogV2Util; import org.apache.spark.sql.connector.catalog.Column; import org.apache.spark.sql.connector.expressions.NamedReference; import org.apache.spark.sql.connector.expressions.Transform; @@ -131,23 +130,20 @@ public Scan build() { /** * Prunes the columns to only include those specified in the required schema. * - *

Only top-level pruning is supported. With {@code spark.sql.optimizer.nestedSchemaPruning.enabled} (the - * default), Spark may pass a struct column with some of its fields removed, but the native scan projects by - * top-level name and returns the whole struct. Each required column therefore keeps its full table type, so - * {@link VortexScan#readSchema()} matches the data the reader returns and Spark binds nested fields by the right - * ordinal. Reporting the pruned type instead makes Spark read the wrong struct field. + *

Spark may also prune fields inside struct, array and map columns, but the native scan only projects + * top-level columns and returns each one whole. Required columns therefore keep their table type, so that + * {@link VortexScan#readSchema()} describes the data the reader returns. * * @param requiredSchema the schema specifying which columns are required */ @Override public void pruneColumns(StructType requiredSchema) { - Map byName = new HashMap<>(); - for (Column column : tableColumns) { - byName.put(column.name(), column); - } + Set required = Set.copyOf(Arrays.asList(requiredSchema.fieldNames())); readColumns.clear(); - for (Column required : CatalogV2Util.structTypeToV2Columns(requiredSchema)) { - readColumns.add(byName.getOrDefault(required.name(), required)); + for (Column column : tableColumns) { + if (required.contains(column.name())) { + readColumns.add(column); + } } } diff --git a/java/vortex-spark/src/test/java/dev/vortex/spark/VortexNestedSchemaPruningTest.java b/java/vortex-spark/src/test/java/dev/vortex/spark/VortexNestedSchemaPruningTest.java index 4527dc3a2e1..cb4b60180ca 100644 --- a/java/vortex-spark/src/test/java/dev/vortex/spark/VortexNestedSchemaPruningTest.java +++ b/java/vortex-spark/src/test/java/dev/vortex/spark/VortexNestedSchemaPruningTest.java @@ -8,6 +8,7 @@ import java.nio.file.Path; import java.util.List; import java.util.stream.Collectors; +import org.apache.spark.sql.Column; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.apache.spark.sql.SaveMode; @@ -19,35 +20,49 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.io.TempDir; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; /** - * Tests that selecting nested fields returns the requested field. + * Tests that reading one field of a nested column returns that field. * - *

With {@code spark.sql.optimizer.nestedSchemaPruning.enabled} (the Spark default), Spark passes struct columns to - * {@code pruneColumns} with unused fields removed. The native scan only prunes top-level columns, so the scan must - * report the full struct type; otherwise Spark reads the pruned field ordinals out of the full struct and returns the - * wrong field. + *

With {@code spark.sql.optimizer.nestedSchemaPruning.enabled}, Spark asks the scan for only the nested fields a + * query uses. Each row of the table holds {@code a = id + 100}, {@code b = id + 200} and {@code c = id + 300}, so a + * field read from the wrong position shows up as the wrong hundred. */ @TestInstance(TestInstance.Lifecycle.PER_CLASS) public final class VortexNestedSchemaPruningTest { - private SparkSession spark; + private static final List C_VALUES = List.of(300L, 301L, 302L, 303L, 304L); - @TempDir - Path tempDir; + private SparkSession spark; + private Dataset table; @BeforeAll - public void setUp() { + public void setUp(@TempDir Path tempDir) { spark = SparkSession.builder() .appName("VortexNestedSchemaPruningTest") .master("local[2]") .config("spark.driver.host", "127.0.0.1") .config("spark.sql.shuffle.partitions", "2") .config("spark.sql.adaptive.enabled", "false") + .config("spark.sql.optimizer.nestedSchemaPruning.enabled", "true") .config("spark.ui.enabled", "false") .getOrCreate(); + + Column abc = functions.struct( + functions.col("id").plus(100).alias("a"), + functions.col("id").plus(200).alias("b"), + functions.col("id").plus(300).alias("c")); + Dataset df = spark.range(5) + .select( + functions.col("id"), + abc.alias("s"), + functions.array(abc).alias("arr"), + functions.map(functions.lit("k"), abc).alias("m"), + functions.struct(functions.lit("x").alias("p"), abc.alias("inner")).alias("outer")); + + String path = tempDir.resolve("nested").toUri().toString(); + df.write().format("vortex").option("path", path).mode(SaveMode.Overwrite).save(); + table = spark.read().format("vortex").option("path", path).load(); } @AfterAll @@ -57,59 +72,33 @@ public void tearDown() { } } - private Dataset writeAndRead(String name) { - Dataset df = spark.range(5) - .select( - functions.col("id"), - functions - .struct( - functions.col("id").plus(100).alias("a"), - functions.col("id").plus(200).alias("b"), - functions.col("id").plus(300).alias("c")) - .alias("s")); - df = df.withColumn("arr", functions.array(functions.col("s"))) - .withColumn( - "outer", - functions.struct( - functions.lit("x").alias("p"), - functions.col("s").alias("inner"))); - String path = tempDir.resolve(name).toUri().toString(); - df.write() - .format("vortex") - .option("path", path) - .mode(SaveMode.Overwrite) - .save(); - return spark.read().format("vortex").option("path", path).load(); + @Test + @DisplayName("Selecting a struct field other than the first returns that field") + public void structField() { + assertEquals(C_VALUES, longsOf(functions.col("s.c"))); } - private static List rows(Dataset df) { - return df.orderBy("id").collectAsList().stream().map(Row::toString).collect(Collectors.toList()); + @Test + @DisplayName("Selecting a field of the structs in an array returns that field") + public void arrayElementField() { + assertEquals(C_VALUES, longsOf(functions.element_at(functions.col("arr.c"), 1))); } - @ParameterizedTest(name = "nestedSchemaPruning={0}") - @ValueSource(strings = {"true", "false"}) - @DisplayName("Selecting a non-first struct field returns that field") - public void selectNestedField(String nestedSchemaPruning) { - spark.conf().set("spark.sql.optimizer.nestedSchemaPruning.enabled", nestedSchemaPruning); - Dataset df = writeAndRead("struct_" + nestedSchemaPruning); - - assertEquals(List.of("[0,300]", "[1,301]", "[2,302]", "[3,303]", "[4,304]"), rows(df.select("id", "s.c"))); - assertEquals( - List.of("[0,200,300]", "[1,201,301]", "[2,202,302]", "[3,203,303]", "[4,204,304]"), - rows(df.select("id", "s.b", "s.c"))); - assertEquals(List.of("[3,303]", "[4,304]"), rows(df.filter("s.c >= 303").select("id", "s.c"))); + @Test + @DisplayName("Selecting a field of the structs in a map returns that field") + public void mapValueField() { + assertEquals(C_VALUES, longsOf(functions.col("m").getItem("k").getField("c"))); } @Test - @DisplayName("Selecting a field of a struct inside an array or another struct returns that field") - public void selectDeeplyNestedField() { - spark.conf().set("spark.sql.optimizer.nestedSchemaPruning.enabled", "true"); - Dataset df = writeAndRead("deep"); + @DisplayName("Selecting a field of a struct inside a struct returns that field") + public void structInStructField() { + assertEquals(C_VALUES, longsOf(functions.col("outer.inner.c"))); + } - assertEquals( - List.of("[0,300]", "[1,301]", "[2,302]", "[3,303]", "[4,304]"), - rows(df.select(functions.col("id"), functions.element_at(functions.col("arr.c"), 1)))); - assertEquals( - List.of("[0,300]", "[1,301]", "[2,302]", "[3,303]", "[4,304]"), rows(df.select("id", "outer.inner.c"))); + private List longsOf(Column column) { + return table.orderBy("id").select(column).collectAsList().stream() + .map(row -> row.getLong(0)) + .collect(Collectors.toList()); } } From c8dcbb3d11e75f4f1b952a5a3ddf9ca165bab401 Mon Sep 17 00:00:00 2001 From: gitRasheed <99349713+gitRasheed@users.noreply.github.com> Date: Thu, 22 Oct 2026 21:01:20 +0000 Subject: [PATCH 3/5] style(spark): apply spotless formatting to the nested pruning fix The Java CI job runs `gradlew check`, which includes spotlessCheck. Signed-off-by: gitRasheed <99349713+gitRasheed@users.noreply.github.com> --- .../java/dev/vortex/spark/read/VortexScanBuilder.java | 4 ++-- .../vortex/spark/VortexNestedSchemaPruningTest.java | 10 ++++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexScanBuilder.java b/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexScanBuilder.java index e9e85962762..8adace80b35 100644 --- a/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexScanBuilder.java +++ b/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexScanBuilder.java @@ -130,8 +130,8 @@ public Scan build() { /** * Prunes the columns to only include those specified in the required schema. * - *

Spark may also prune fields inside struct, array and map columns, but the native scan only projects - * top-level columns and returns each one whole. Required columns therefore keep their table type, so that + *

Spark may also prune fields inside struct, array and map columns, but the native scan only projects top-level + * columns and returns each one whole. Required columns therefore keep their table type, so that * {@link VortexScan#readSchema()} describes the data the reader returns. * * @param requiredSchema the schema specifying which columns are required diff --git a/java/vortex-spark/src/test/java/dev/vortex/spark/VortexNestedSchemaPruningTest.java b/java/vortex-spark/src/test/java/dev/vortex/spark/VortexNestedSchemaPruningTest.java index cb4b60180ca..16bdc59a0f8 100644 --- a/java/vortex-spark/src/test/java/dev/vortex/spark/VortexNestedSchemaPruningTest.java +++ b/java/vortex-spark/src/test/java/dev/vortex/spark/VortexNestedSchemaPruningTest.java @@ -58,10 +58,16 @@ public void setUp(@TempDir Path tempDir) { abc.alias("s"), functions.array(abc).alias("arr"), functions.map(functions.lit("k"), abc).alias("m"), - functions.struct(functions.lit("x").alias("p"), abc.alias("inner")).alias("outer")); + functions + .struct(functions.lit("x").alias("p"), abc.alias("inner")) + .alias("outer")); String path = tempDir.resolve("nested").toUri().toString(); - df.write().format("vortex").option("path", path).mode(SaveMode.Overwrite).save(); + df.write() + .format("vortex") + .option("path", path) + .mode(SaveMode.Overwrite) + .save(); table = spark.read().format("vortex").option("path", path).load(); } From 93d7fa516449c4109be911c0adf30fd6dbae6a46 Mon Sep 17 00:00:00 2001 From: gitRasheed <99349713+gitRasheed@users.noreply.github.com> Date: Thu, 22 Oct 2026 21:01:21 +0000 Subject: [PATCH 4/5] test(spark): check that pruneColumns keeps each column's table type Pin the pruneColumns contract without going through Spark's optimizer: a nested-pruned required schema yields the table's full column types, in table order, and unrequired columns are dropped. Signed-off-by: gitRasheed <99349713+gitRasheed@users.noreply.github.com> --- .../spark/read/VortexScanBuilderTest.java | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/java/vortex-spark/src/test/java/dev/vortex/spark/read/VortexScanBuilderTest.java b/java/vortex-spark/src/test/java/dev/vortex/spark/read/VortexScanBuilderTest.java index 85d8046833f..485c093daff 100644 --- a/java/vortex-spark/src/test/java/dev/vortex/spark/read/VortexScanBuilderTest.java +++ b/java/vortex-spark/src/test/java/dev/vortex/spark/read/VortexScanBuilderTest.java @@ -16,6 +16,7 @@ import org.apache.spark.sql.connector.expressions.Transform; import org.apache.spark.sql.connector.expressions.filter.Predicate; import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.StructType; import org.apache.spark.unsafe.types.UTF8String; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -160,6 +161,31 @@ void pushedPredicatesReturnsCopy() { assertArrayEquals(new Predicate[] {predicate}, builder.pushedPredicates()); } + // --- pruneColumns --- + + @Test + @DisplayName("Required columns keep their full table type, so readSchema matches what the reader returns") + void pruneColumnsKeepsTableTypes() { + StructType abc = new StructType() + .add("a", DataTypes.LongType) + .add("b", DataTypes.LongType) + .add("c", DataTypes.LongType); + VortexScanBuilder builder = builderWithColumns( + new Transform[0], + Column.create("id", DataTypes.IntegerType), + Column.create("s", abc), + Column.create("name", DataTypes.StringType)); + + // With nested schema pruning, Spark asks for s with only the field the query uses. + builder.pruneColumns(new StructType() + .add("s", new StructType().add("c", DataTypes.LongType)) + .add("id", DataTypes.IntegerType)); + + assertEquals( + new StructType().add("id", DataTypes.IntegerType).add("s", abc), + builder.build().readSchema()); + } + // --- build --- @Test From a7744f21b9e7cd0e8761e9ec046df2c1134f8e1c Mon Sep 17 00:00:00 2001 From: gitRasheed <99349713+gitRasheed@users.noreply.github.com> Date: Fri, 23 Oct 2026 14:04:24 +0100 Subject: [PATCH 5/5] docs(spark): shorten the pruneColumns comment and drop the user guide note Signed-off-by: gitRasheed <99349713+gitRasheed@users.noreply.github.com> --- docs/user-guide/spark.md | 2 -- .../main/java/dev/vortex/spark/read/VortexScanBuilder.java | 5 ++--- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/docs/user-guide/spark.md b/docs/user-guide/spark.md index 50692078ba7..0410b709aa6 100644 --- a/docs/user-guide/spark.md +++ b/docs/user-guide/spark.md @@ -78,8 +78,6 @@ When pointed at a directory, the connector discovers all `.vortex` files and cre partition per file. Column pruning is pushed down — only the columns referenced by the query are read from the file. -Pruning applies to top-level columns: a query that uses one field of a struct, array or map -column reads the whole column. Filter pushdown is also supported. Comparisons (`=`, `<>`, `>`, `>=`, `<`, `<=`), `IS NULL`, `IS NOT NULL`, `IN`, the string predicates `STARTS_WITH`, `ENDS_WITH` and `CONTAINS`, and a bare diff --git a/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexScanBuilder.java b/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexScanBuilder.java index 8adace80b35..97da8d42302 100644 --- a/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexScanBuilder.java +++ b/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexScanBuilder.java @@ -130,9 +130,8 @@ public Scan build() { /** * Prunes the columns to only include those specified in the required schema. * - *

Spark may also prune fields inside struct, array and map columns, but the native scan only projects top-level - * columns and returns each one whole. Required columns therefore keep their table type, so that - * {@link VortexScan#readSchema()} describes the data the reader returns. + *

Required columns keep their table type because the native scan returns whole top-level columns, even when + * Spark prunes nested fields. * * @param requiredSchema the schema specifying which columns are required */