From a009ad544ae2b72ae3e5ded26167374d9ad1c976 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 28 Sep 2026 21:07:45 -0600 Subject: [PATCH] fix: make benchmark cases over ShortType columns exercise Comet Comet falls back to Spark for any Parquet scan that reads a ShortType column while spark.comet.scan.unsignedSmallIntSafetyCheck is enabled, because Spark maps both signed INT16 and unsigned UINT_8 to ShortType. Benchmark tables are written by Spark, where ShortType is always a signed INT16, so the check only made Comet cases over a ShortType column, such as the c_short casts in CometCastNumericToNumericBenchmark, measure Spark. Disable it in the shared benchmark session. --- .../org/apache/spark/sql/benchmark/CometBenchmarkBase.scala | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBenchmarkBase.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBenchmarkBase.scala index 385c1eb1f93..afb072c7953 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBenchmarkBase.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBenchmarkBase.scala @@ -79,6 +79,11 @@ trait CometBenchmarkBase sparkSession.conf.set(CometConf.COMET_EXEC_ENABLED.key, "false") // Benchmarks use invalid input values that should produce NULL, not exceptions sparkSession.conf.set(SQLConf.ANSI_ENABLED.key, "false") + // Comet falls back to Spark for any Parquet scan of a ShortType column unless this check is + // disabled, because the column may hold an unsigned UINT_8. Benchmark tables are written by + // Spark, where ShortType is always a signed INT16, so the check only turns Comet cases that + // read a ShortType column into Spark measurements. + sparkSession.conf.set(CometConf.COMET_PARQUET_UNSIGNED_SMALL_INT_CHECK.key, "false") sparkSession }