diff --git a/.github/trigger_files/beam_PostCommit_Java_Tpcds_Flink.json b/.github/trigger_files/beam_PostCommit_Java_Tpcds_Flink.json index b26833333238..c537844dc84a 100644 --- a/.github/trigger_files/beam_PostCommit_Java_Tpcds_Flink.json +++ b/.github/trigger_files/beam_PostCommit_Java_Tpcds_Flink.json @@ -1,4 +1,4 @@ { "comment": "Modify this file in a trivial way to cause this test suite to run", - "modification": 2 + "modification": 3 } diff --git a/.github/trigger_files/beam_PostCommit_Java_Tpcds_Spark.json b/.github/trigger_files/beam_PostCommit_Java_Tpcds_Spark.json index b26833333238..c537844dc84a 100644 --- a/.github/trigger_files/beam_PostCommit_Java_Tpcds_Spark.json +++ b/.github/trigger_files/beam_PostCommit_Java_Tpcds_Spark.json @@ -1,4 +1,4 @@ { "comment": "Modify this file in a trivial way to cause this test suite to run", - "modification": 2 + "modification": 3 } diff --git a/.github/trigger_files/beam_PostCommit_Python_Tpcds_Prism.json b/.github/trigger_files/beam_PostCommit_Python_Tpcds_Prism.json new file mode 100644 index 000000000000..0967ef424bce --- /dev/null +++ b/.github/trigger_files/beam_PostCommit_Python_Tpcds_Prism.json @@ -0,0 +1 @@ +{} diff --git a/sdks/java/testing/tpcds/build.gradle b/sdks/java/testing/tpcds/build.gradle index 60c2f8bfdd8b..4fe0f936fb36 100644 --- a/sdks/java/testing/tpcds/build.gradle +++ b/sdks/java/testing/tpcds/build.gradle @@ -158,6 +158,7 @@ task run(type: JavaExec) { jvmArgs += sparkJvmArgs() } + systemProperty "java.util.logging.config.file", "${project.projectDir}/src/main/resources/logging.properties" mainClass = "org.apache.beam.sdk.tpcds.BeamTpcds" classpath = configurations.gradleRun args tpcdsArgsList.toArray() diff --git a/sdks/java/testing/tpcds/src/main/java/org/apache/beam/sdk/tpcds/TpcdsParametersReader.java b/sdks/java/testing/tpcds/src/main/java/org/apache/beam/sdk/tpcds/TpcdsParametersReader.java index a2a451b554e8..1dc71bdc7347 100644 --- a/sdks/java/testing/tpcds/src/main/java/org/apache/beam/sdk/tpcds/TpcdsParametersReader.java +++ b/sdks/java/testing/tpcds/src/main/java/org/apache/beam/sdk/tpcds/TpcdsParametersReader.java @@ -42,7 +42,7 @@ public class TpcdsParametersReader { "1", "3", "4", "7", "9", "10", "11", "12", "15", "17", "19", "20", "21", "26", "31", "32", "33", "34", "35", "38", "40", "42", "43", "45", "46", "49", "50", "52", "53", "55", "56", "59", "60", "61", "63", "65", "66", "68", "69", "71", "73", "74", "75", - "76", "79", "82", "84", "87", "88", "89", "92", "93", "96", "97", "99") + "76", "79", "82", "84", "87", "88", "89", "90", "92", "93", "96", "97", "98", "99") .map(s -> QUERY_PREFIX + s) .collect(Collectors.toList()); diff --git a/sdks/java/testing/tpcds/src/main/resources/log4j2.properties b/sdks/java/testing/tpcds/src/main/resources/log4j2.properties new file mode 100644 index 000000000000..d11ea2f2291a --- /dev/null +++ b/sdks/java/testing/tpcds/src/main/resources/log4j2.properties @@ -0,0 +1,28 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +rootLogger.level = info +rootLogger.appenderRef.console.ref = console + +appender.console.type = Console +appender.console.name = console +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n%ex + +logger.spark.name = org.apache.spark +logger.spark.level = warn diff --git a/sdks/java/testing/tpcds/src/main/resources/logging.properties b/sdks/java/testing/tpcds/src/main/resources/logging.properties new file mode 100644 index 000000000000..3fb18b13bea1 --- /dev/null +++ b/sdks/java/testing/tpcds/src/main/resources/logging.properties @@ -0,0 +1,24 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +handlers = java.util.logging.ConsoleHandler +.level = INFO +org.apache.flink.level = WARNING +org.apache.flink.runtime.level = SEVERE +org.apache.flink.metrics.level = SEVERE +org.apache.flink.streaming.level = SEVERE diff --git a/sdks/python/apache_beam/testing/benchmarks/tpcds/tpcds_launcher.py b/sdks/python/apache_beam/testing/benchmarks/tpcds/tpcds_launcher.py index 280226cd00e2..df6f16d6598e 100644 --- a/sdks/python/apache_beam/testing/benchmarks/tpcds/tpcds_launcher.py +++ b/sdks/python/apache_beam/testing/benchmarks/tpcds/tpcds_launcher.py @@ -240,7 +240,7 @@ def run_single_query( elapsed = time.monotonic() - t0 end_dt = datetime.datetime.now(datetime.timezone.utc) err_str = str(exc) - first_line = err_str.splitlines()[0] if err_str else repr(exc) + first_line = err_str.splitlines()[0] for line in err_str.splitlines(): if "UnsupportedOperationException" in line or "CannotPlanException" in line: first_line = line.strip() @@ -256,7 +256,7 @@ def run_single_query( start_time=start_dt.strftime("%Y-%m-%dT%H:%M:%SZ"), end_time=end_dt.strftime("%Y-%m-%dT%H:%M:%SZ"), elapsed_sec=round(elapsed, 3), - error_message=first_line, + error_message=err_str, ) @@ -273,7 +273,14 @@ def format_summary_table(results: Sequence[TpcdsQueryResult]) -> str: ] rows = [] for r in results: - note = f"Returned {r.row_count} row(s)" if r.is_successful else r.error_message + if r.is_successful: + note = f"Returned {r.row_count} row(s)" + else: + note = r.error_message.splitlines()[0] if r.error_message else "" + for line in r.error_message.splitlines(): + if "UnsupportedOperationException" in line or "CannotPlanException" in line: + note = line.strip() + break if len(note) > 85: note = note[:82] + "..." rows.append([ diff --git a/sdks/python/apache_beam/testing/benchmarks/tpcds/tpcds_queries.py b/sdks/python/apache_beam/testing/benchmarks/tpcds/tpcds_queries.py index e7d45fc57d48..fd114cefe25d 100644 --- a/sdks/python/apache_beam/testing/benchmarks/tpcds/tpcds_queries.py +++ b/sdks/python/apache_beam/testing/benchmarks/tpcds/tpcds_queries.py @@ -34,8 +34,8 @@ "3", "7", "13", "15", "19", "26", "42", "43", "48", "52", "55", "91", "96" ] -# The 55 TPC-DS queries supported by Beam Calcite SQL (returning > 0 rows). -# Together with KNOWN_FAILING_QUERIES (48 queries), this partitions all 103 TPC-DS queries. +# TPC-DS queries supported by Beam Calcite SQL (returning > 0 rows). +# Together with KNOWN_FAILING_QUERIES, this partitions all 103 TPC-DS queries. # yapf: disable SUPPORTED_QUERIES: List[str] = [ "1", "3", "4", "7", "9", "10", "11", "12", "15", "17", @@ -43,13 +43,13 @@ "40", "42", "43", "45", "46", "49", "50", "52", "53", "55", "56", "59", "60", "61", "63", "65", "66", "68", "69", "71", "73", "74", "75", "76", "79", "82", "84", "87", "88", "89", - "92", "93", "96", "97", "99", + "90", "92", "93", "96", "97", "98", "99", ] # yapf: enable -# The 48 known failing TPC-DS queries on Beam Calcite SQL with their expected error substring. +# Known failing TPC-DS queries on Beam Calcite SQL with their expected error substring. # Used by `--check_regression` and `:sdks:python:apache_beam:testing:benchmarks:tpcds:checkAllQueries` -# to verify that all 55 SUPPORTED_QUERIES succeed on PrismRunner and that the remaining 48 known +# to verify that all SUPPORTED_QUERIES succeed on PrismRunner and that the remaining known # failing queries fail with their expected diagnostic without regressing. # pylint: disable=line-too-long KNOWN_FAILING_QUERIES: Dict[str, str] = { @@ -96,11 +96,9 @@ "query83": "Error decoding input stream with coder", "query85": "Returned 0 rows", "query86": "Error while applying rule BeamBasicAggregationRule", - "query90": "CalcFn failed to evaluate", "query91": "VolcanoPlanner timeout (7-table join permutation search)", "query94": "Error while applying rule JoinPushExpressionsRule", "query95": "Error while applying rule JoinPushExpressionsRule", - "query98": "IllegalArgumentException: DECIMAL NOT NULL is not nullable", } # pylint: enable=line-too-long diff --git a/sdks/python/apache_beam/testing/benchmarks/tpcds/tpcds_test.py b/sdks/python/apache_beam/testing/benchmarks/tpcds/tpcds_test.py index b19da0f62681..f7630d196160 100644 --- a/sdks/python/apache_beam/testing/benchmarks/tpcds/tpcds_test.py +++ b/sdks/python/apache_beam/testing/benchmarks/tpcds/tpcds_test.py @@ -47,9 +47,6 @@ def test_known_failing_queries_map_valid(self): tpcds_queries.PLANNER_TIMEOUT_QUERIES.issubset(failing_queries)) self.assertEqual(supported_queries & failing_queries, set()) self.assertEqual(supported_queries | failing_queries, all_queries) - # 55 supported + 48 known failing = 103 total queries - self.assertEqual(len(supported_queries), 55) - self.assertEqual(len(failing_queries), 48) def test_query_selection_and_normalization(self): qnames = tpcds_queries.parse_query_selection("canonical13")