From 13ae771e5ba9e4250df1e3f1a6628ec78ecccda7 Mon Sep 17 00:00:00 2001 From: Yi Hu Date: Thu, 22 Oct 2026 23:01:25 +0000 Subject: [PATCH 1/2] TPC-DS suite follow ups * Some 10 MB test failed due to filtered dataset has no result, either intermediate or final query result, and may cause ZeroDivision, etc. Fixed dataset to add more items so they query result has non-zero row * Fix known error to check whole error stack instead of the first line * Fix log spam on Flink and Spark runners * Enable query90,98 --- .../beam_PostCommit_Java_Tpcds_Flink.json | 2 +- .../beam_PostCommit_Java_Tpcds_Spark.json | 2 +- .../beam_PostCommit_Python_Tpcds_Prism.json | 1 + sdks/java/testing/tpcds/build.gradle | 1 + .../beam/sdk/tpcds/TpcdsParametersReader.java | 2 +- .../src/main/resources/log4j2.properties | 28 +++++++++++++++++++ .../src/main/resources/logging.properties | 24 ++++++++++++++++ .../benchmarks/tpcds/tpcds_launcher.py | 13 +++++++-- .../testing/benchmarks/tpcds/tpcds_queries.py | 12 ++++---- .../testing/benchmarks/tpcds/tpcds_test.py | 3 -- 10 files changed, 72 insertions(+), 16 deletions(-) create mode 100644 .github/trigger_files/beam_PostCommit_Python_Tpcds_Prism.json create mode 100644 sdks/java/testing/tpcds/src/main/resources/log4j2.properties create mode 100644 sdks/java/testing/tpcds/src/main/resources/logging.properties diff --git a/.github/trigger_files/beam_PostCommit_Java_Tpcds_Flink.json b/.github/trigger_files/beam_PostCommit_Java_Tpcds_Flink.json index b26833333238..c537844dc84a 100644 --- a/.github/trigger_files/beam_PostCommit_Java_Tpcds_Flink.json +++ b/.github/trigger_files/beam_PostCommit_Java_Tpcds_Flink.json @@ -1,4 +1,4 @@ { "comment": "Modify this file in a trivial way to cause this test suite to run", - "modification": 2 + "modification": 3 } diff --git a/.github/trigger_files/beam_PostCommit_Java_Tpcds_Spark.json b/.github/trigger_files/beam_PostCommit_Java_Tpcds_Spark.json index b26833333238..c537844dc84a 100644 --- a/.github/trigger_files/beam_PostCommit_Java_Tpcds_Spark.json +++ b/.github/trigger_files/beam_PostCommit_Java_Tpcds_Spark.json @@ -1,4 +1,4 @@ { "comment": "Modify this file in a trivial way to cause this test suite to run", - "modification": 2 + "modification": 3 } diff --git a/.github/trigger_files/beam_PostCommit_Python_Tpcds_Prism.json b/.github/trigger_files/beam_PostCommit_Python_Tpcds_Prism.json new file mode 100644 index 000000000000..0967ef424bce --- /dev/null +++ b/.github/trigger_files/beam_PostCommit_Python_Tpcds_Prism.json @@ -0,0 +1 @@ +{} diff --git a/sdks/java/testing/tpcds/build.gradle b/sdks/java/testing/tpcds/build.gradle index 60c2f8bfdd8b..4fe0f936fb36 100644 --- a/sdks/java/testing/tpcds/build.gradle +++ b/sdks/java/testing/tpcds/build.gradle @@ -158,6 +158,7 @@ task run(type: JavaExec) { jvmArgs += sparkJvmArgs() } + systemProperty "java.util.logging.config.file", "${project.projectDir}/src/main/resources/logging.properties" mainClass = "org.apache.beam.sdk.tpcds.BeamTpcds" classpath = configurations.gradleRun args tpcdsArgsList.toArray() diff --git a/sdks/java/testing/tpcds/src/main/java/org/apache/beam/sdk/tpcds/TpcdsParametersReader.java b/sdks/java/testing/tpcds/src/main/java/org/apache/beam/sdk/tpcds/TpcdsParametersReader.java index a2a451b554e8..1dc71bdc7347 100644 --- a/sdks/java/testing/tpcds/src/main/java/org/apache/beam/sdk/tpcds/TpcdsParametersReader.java +++ b/sdks/java/testing/tpcds/src/main/java/org/apache/beam/sdk/tpcds/TpcdsParametersReader.java @@ -42,7 +42,7 @@ public class TpcdsParametersReader { "1", "3", "4", "7", "9", "10", "11", "12", "15", "17", "19", "20", "21", "26", "31", "32", "33", "34", "35", "38", "40", "42", "43", "45", "46", "49", "50", "52", "53", "55", "56", "59", "60", "61", "63", "65", "66", "68", "69", "71", "73", "74", "75", - "76", "79", "82", "84", "87", "88", "89", "92", "93", "96", "97", "99") + "76", "79", "82", "84", "87", "88", "89", "90", "92", "93", "96", "97", "98", "99") .map(s -> QUERY_PREFIX + s) .collect(Collectors.toList()); diff --git a/sdks/java/testing/tpcds/src/main/resources/log4j2.properties b/sdks/java/testing/tpcds/src/main/resources/log4j2.properties new file mode 100644 index 000000000000..d11ea2f2291a --- /dev/null +++ b/sdks/java/testing/tpcds/src/main/resources/log4j2.properties @@ -0,0 +1,28 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +rootLogger.level = info +rootLogger.appenderRef.console.ref = console + +appender.console.type = Console +appender.console.name = console +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n%ex + +logger.spark.name = org.apache.spark +logger.spark.level = warn diff --git a/sdks/java/testing/tpcds/src/main/resources/logging.properties b/sdks/java/testing/tpcds/src/main/resources/logging.properties new file mode 100644 index 000000000000..3fb18b13bea1 --- /dev/null +++ b/sdks/java/testing/tpcds/src/main/resources/logging.properties @@ -0,0 +1,24 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +handlers = java.util.logging.ConsoleHandler +.level = INFO +org.apache.flink.level = WARNING +org.apache.flink.runtime.level = SEVERE +org.apache.flink.metrics.level = SEVERE +org.apache.flink.streaming.level = SEVERE diff --git a/sdks/python/apache_beam/testing/benchmarks/tpcds/tpcds_launcher.py b/sdks/python/apache_beam/testing/benchmarks/tpcds/tpcds_launcher.py index 280226cd00e2..df6f16d6598e 100644 --- a/sdks/python/apache_beam/testing/benchmarks/tpcds/tpcds_launcher.py +++ b/sdks/python/apache_beam/testing/benchmarks/tpcds/tpcds_launcher.py @@ -240,7 +240,7 @@ def run_single_query( elapsed = time.monotonic() - t0 end_dt = datetime.datetime.now(datetime.timezone.utc) err_str = str(exc) - first_line = err_str.splitlines()[0] if err_str else repr(exc) + first_line = err_str.splitlines()[0] for line in err_str.splitlines(): if "UnsupportedOperationException" in line or "CannotPlanException" in line: first_line = line.strip() @@ -256,7 +256,7 @@ def run_single_query( start_time=start_dt.strftime("%Y-%m-%dT%H:%M:%SZ"), end_time=end_dt.strftime("%Y-%m-%dT%H:%M:%SZ"), elapsed_sec=round(elapsed, 3), - error_message=first_line, + error_message=err_str, ) @@ -273,7 +273,14 @@ def format_summary_table(results: Sequence[TpcdsQueryResult]) -> str: ] rows = [] for r in results: - note = f"Returned {r.row_count} row(s)" if r.is_successful else r.error_message + if r.is_successful: + note = f"Returned {r.row_count} row(s)" + else: + note = r.error_message.splitlines()[0] if r.error_message else "" + for line in r.error_message.splitlines(): + if "UnsupportedOperationException" in line or "CannotPlanException" in line: + note = line.strip() + break if len(note) > 85: note = note[:82] + "..." rows.append([ diff --git a/sdks/python/apache_beam/testing/benchmarks/tpcds/tpcds_queries.py b/sdks/python/apache_beam/testing/benchmarks/tpcds/tpcds_queries.py index e7d45fc57d48..572eabd7a5ce 100644 --- a/sdks/python/apache_beam/testing/benchmarks/tpcds/tpcds_queries.py +++ b/sdks/python/apache_beam/testing/benchmarks/tpcds/tpcds_queries.py @@ -34,8 +34,8 @@ "3", "7", "13", "15", "19", "26", "42", "43", "48", "52", "55", "91", "96" ] -# The 55 TPC-DS queries supported by Beam Calcite SQL (returning > 0 rows). -# Together with KNOWN_FAILING_QUERIES (48 queries), this partitions all 103 TPC-DS queries. +# TPC-DS queries supported by Beam Calcite SQL (returning > 0 rows). +# Together with KNOWN_FAILING_QUERIES, this partitions all 103 TPC-DS queries. # yapf: disable SUPPORTED_QUERIES: List[str] = [ "1", "3", "4", "7", "9", "10", "11", "12", "15", "17", @@ -43,13 +43,13 @@ "40", "42", "43", "45", "46", "49", "50", "52", "53", "55", "56", "59", "60", "61", "63", "65", "66", "68", "69", "71", "73", "74", "75", "76", "79", "82", "84", "87", "88", "89", - "92", "93", "96", "97", "99", + "90", "92", "93", "96", "97", "98", "99", ] # yapf: enable -# The 48 known failing TPC-DS queries on Beam Calcite SQL with their expected error substring. +# Known failing TPC-DS queries on Beam Calcite SQL with their expected error substring. # Used by `--check_regression` and `:sdks:python:apache_beam:testing:benchmarks:tpcds:checkAllQueries` -# to verify that all 55 SUPPORTED_QUERIES succeed on PrismRunner and that the remaining 48 known +# to verify that all SUPPORTED_QUERIES succeed on PrismRunner and that the remaining 48 known # failing queries fail with their expected diagnostic without regressing. # pylint: disable=line-too-long KNOWN_FAILING_QUERIES: Dict[str, str] = { @@ -96,11 +96,9 @@ "query83": "Error decoding input stream with coder", "query85": "Returned 0 rows", "query86": "Error while applying rule BeamBasicAggregationRule", - "query90": "CalcFn failed to evaluate", "query91": "VolcanoPlanner timeout (7-table join permutation search)", "query94": "Error while applying rule JoinPushExpressionsRule", "query95": "Error while applying rule JoinPushExpressionsRule", - "query98": "IllegalArgumentException: DECIMAL NOT NULL is not nullable", } # pylint: enable=line-too-long diff --git a/sdks/python/apache_beam/testing/benchmarks/tpcds/tpcds_test.py b/sdks/python/apache_beam/testing/benchmarks/tpcds/tpcds_test.py index b19da0f62681..f7630d196160 100644 --- a/sdks/python/apache_beam/testing/benchmarks/tpcds/tpcds_test.py +++ b/sdks/python/apache_beam/testing/benchmarks/tpcds/tpcds_test.py @@ -47,9 +47,6 @@ def test_known_failing_queries_map_valid(self): tpcds_queries.PLANNER_TIMEOUT_QUERIES.issubset(failing_queries)) self.assertEqual(supported_queries & failing_queries, set()) self.assertEqual(supported_queries | failing_queries, all_queries) - # 55 supported + 48 known failing = 103 total queries - self.assertEqual(len(supported_queries), 55) - self.assertEqual(len(failing_queries), 48) def test_query_selection_and_normalization(self): qnames = tpcds_queries.parse_query_selection("canonical13") From 5345eea5f86315c0941ec920fc9c4ef42389d6c6 Mon Sep 17 00:00:00 2001 From: Yi Hu Date: Fri, 23 Oct 2026 09:21:15 -0400 Subject: [PATCH 2/2] Update sdks/python/apache_beam/testing/benchmarks/tpcds/tpcds_queries.py Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .../apache_beam/testing/benchmarks/tpcds/tpcds_queries.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/testing/benchmarks/tpcds/tpcds_queries.py b/sdks/python/apache_beam/testing/benchmarks/tpcds/tpcds_queries.py index 572eabd7a5ce..fd114cefe25d 100644 --- a/sdks/python/apache_beam/testing/benchmarks/tpcds/tpcds_queries.py +++ b/sdks/python/apache_beam/testing/benchmarks/tpcds/tpcds_queries.py @@ -49,7 +49,7 @@ # Known failing TPC-DS queries on Beam Calcite SQL with their expected error substring. # Used by `--check_regression` and `:sdks:python:apache_beam:testing:benchmarks:tpcds:checkAllQueries` -# to verify that all SUPPORTED_QUERIES succeed on PrismRunner and that the remaining 48 known +# to verify that all SUPPORTED_QUERIES succeed on PrismRunner and that the remaining known # failing queries fail with their expected diagnostic without regressing. # pylint: disable=line-too-long KNOWN_FAILING_QUERIES: Dict[str, str] = {