From 85997d8f535737d15944431fa6e9be0cd1c3d88a Mon Sep 17 00:00:00 2001 From: kary zheng Date: Thu, 10 Sep 2026 20:48:33 -0700 Subject: [PATCH 01/32] feat(workflow-operator): export the relational and sampling operators Seventeen operators implement the trait: the three joins, the three set operations, Aggregate, the sort family, Split, the two samplers, and If, Dummy and Sleep. What they have in common is that they rearrange rows rather than read what is inside one. A sampler decides per row whether to keep it, so which rows survive is fixed by the exact sequence java.util.Random produces. Seeding Python's own generator selects a different set, and the script would then report a different sample than the workflow it came from, so SamplingHelpers transcribes the generator rather than approximating it. Split out of #8340 on review, which had grown past what one reading can hold. That change keeps the operators that work on text, and the cast transcription they share. Co-Authored-By: Claude Opus 5 (1M context) --- .../texera/amber/operator/LogicalOp.scala | 12 ++ .../amber/operator/SamplingHelpers.scala | 63 +++++++++++ .../operator/aggregate/AggregateOpDesc.scala | 94 ++++++++++++++- .../CartesianProductOpDesc.scala | 28 ++++- .../difference/DifferenceOpDesc.scala | 13 ++- .../amber/operator/dummy/DummyOpDesc.scala | 17 ++- .../operator/hashJoin/HashJoinOpDesc.scala | 35 +++++- .../amber/operator/ifStatement/IfOpDesc.scala | 21 +++- .../operator/intersect/IntersectOpDesc.scala | 11 +- .../intervalJoin/IntervalJoinOpDesc.scala | 39 ++++++- .../RandomKSamplingOpDesc.scala | 22 +++- .../ReservoirSamplingOpDesc.scala | 27 ++++- .../amber/operator/sleep/SleepOpDesc.scala | 10 +- .../operator/sort/SortCriteriaUnit.scala | 6 +- .../amber/operator/sort/SortOpDesc.scala | 20 +++- .../operator/sort/StableMergeSortOpDesc.scala | 27 ++++- .../sortPartitions/SortPartitionsOpDesc.scala | 20 +++- .../amber/operator/split/SplitOpDesc.scala | 25 +++- .../SymmetricDifferenceOpDesc.scala | 10 +- .../operator/dummy/DummyOpDescSpec.scala | 5 + .../hashJoin/HashJoinOpDescSpec.scala | 107 ++++++++++++++++++ .../intervalJoin/IntervalJoinOpDescSpec.scala | 99 ++++++++++++++++ .../RandomKSamplingOpDescSpec.scala | 24 ++++ .../ReservoirSamplingOpDescSpec.scala | 21 ++++ .../operator/sleep/SleepOpExecSpec.scala | 17 +++ .../operator/split/SplitOpDescSpec.scala | 38 +++++++ 26 files changed, 773 insertions(+), 38 deletions(-) create mode 100644 common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/SamplingHelpers.scala diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/LogicalOp.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/LogicalOp.scala index efa46144180..5e7be51d003 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/LogicalOp.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/LogicalOp.scala @@ -459,6 +459,18 @@ abstract class LogicalOp extends PortDescriptor with Serializable { def operatorInfo: OperatorInfo + /** + * Whether the row ORDER of this operator's output is part of its contract. + * Defaults to false: the engine runs operators across parallel workers, so + * for almost every operator the output row order is an implementation-defined + * interleaving. Only operators whose very purpose is to establish an order + * override this, which here is the sort family: Sort, Stable Merge Sort and + * Sort Partitions. Anything comparing two runs of an operator reads it to + * decide whether the rows have to arrive in the same order or only be the + * same rows. + */ + def orderSensitive: Boolean = false + private def getOperatorVersion: String = { val path = "amber/src/main/scala/" val operatorPath = path + this.getClass.getPackage.getName.replace(".", "/") diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/SamplingHelpers.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/SamplingHelpers.scala new file mode 100644 index 00000000000..21eeeb0b231 --- /dev/null +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/SamplingHelpers.scala @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.operator + +/** The Python a sampler's standalone code needs, emitted once per script via + * [[StandaloneCodeGenerator.standaloneHelpers]]. + */ +object SamplingHelpers { + + /** + * A Python transcription of `java.util.Random`, for operators whose executor + * draws from one. + * + * A sampler decides per row whether to keep it, so which rows survive is + * fixed by the exact sequence the generator produces. Seeding Python's + * `random` or numpy's with the engine's seed selects a different set, and + * the script would then report a different sample than the workflow it came + * from. Only the same generator gives the same rows. + */ + val JavaRandom: String = + """# java.util.Random, transcribed so sampling matches the engine. + |class _TexeraJavaRandom: + | _MASK = (1 << 48) - 1 + | _MULTIPLIER = 0x5DEECE66D + | _ADDEND = 0xB + | + | def __init__(self, seed): + | self._seed = (seed ^ self._MULTIPLIER) & self._MASK + | + | def _next(self, bits): + | self._seed = (self._seed * self._MULTIPLIER + self._ADDEND) & self._MASK + | value = self._seed >> (48 - bits) + | return value - (1 << 32) if value >= (1 << 31) else value + | + | def next_double(self): + | return ((self._next(26) << 27) + self._next(27)) * (2.0 ** -53) + | + | def next_int(self, bound): + | if bound & (-bound) == bound: + | return (bound * self._next(31)) >> 31 + | while True: + | bits = self._next(31) + | value = bits % bound + | if bits - value + (bound - 1) >= 0: + | return value""".stripMargin +} diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDesc.scala index 1bace652c8b..16408b0870c 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDesc.scala @@ -29,14 +29,16 @@ import org.apache.texera.amber.core.virtualidentity.{ WorkflowIdentity } import org.apache.texera.amber.core.workflow._ -import org.apache.texera.amber.operator.LogicalOp +import org.apache.texera.amber.operator.{LogicalOp, StandaloneCodeGenerator} import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeNameList import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral import org.apache.texera.amber.util.JSONUtils.objectMapper import javax.validation.constraints.{NotNull, Size} -class AggregateOpDesc extends LogicalOp { +class AggregateOpDesc extends LogicalOp with StandaloneCodeGenerator { + @JsonProperty(value = "aggregations", required = true) @JsonPropertyDescription("multiple aggregation functions") @NotNull(message = "aggregation cannot be null") @@ -138,4 +140,92 @@ class AggregateOpDesc extends LogicalOp { inputPorts = List(InputPort()), outputPorts = List(OutputPort()) ) + + // The engine aggregates in two phases across partitions; one process needs + // only the one groupby, or a single-row reduction when no key is grouped on. + // + // Must run before `getPhysicalPlan`, which rewrites `aggregations` in place: + // it turns COUNT into SUM for the final phase, and this reads them as written. + override def generateStandaloneCode(): String = { + val keys = Option(groupByKeys).getOrElse(List()) + val aggs = Option(aggregations).getOrElse(List()) + + // Identical helper definition each call — keeps the standalone module + // self-contained without relying on a shared prelude. + val concatHelper = + """def _texera_agg_concat(series): + | # The accumulator starts empty and only earns a separator once it + | # holds something, so a leading empty value adds neither text nor + | # comma: "", "a", "" concatenates to "a," and not ",a,". A null is + | # read as the empty string, which is what makes the two the same + | # here. This is concatAgg's fold, written out. + | partial = "" + | for v in series: + | text = "" if pd.isna(v) else str(v) + | partial = text if partial == "" else partial + "," + text + | return partial""".stripMargin + + if (keys.isEmpty) { + val rowEntries = aggs + .map(agg => s" ${pyStringLiteral(agg.resultAttribute)}: ${aggExprScalar(agg)},") + .mkString("\n") + s"""$concatHelper + |out1df = pd.DataFrame([{ + |$rowEntries + |}])""".stripMargin + } else { + val keysLit = keys.map(pyStringLiteral).mkString("[", ", ", "]") + val aggLines = aggs.zipWithIndex + .map { + case (agg, i) => + s"_texera_agg_s$i = ${aggExprGroupby(agg, "_texera_agg_groups")}" + } + .mkString("\n") + val mergeLines = aggs.indices + .map(i => + s"""out1df = out1df.merge(_texera_agg_s$i.reset_index(), on=$keysLit, how="left")""" + ) + .mkString("\n") + s"""$concatHelper + |_texera_agg_groups = in1df.groupby($keysLit, dropna=False, sort=False) + |out1df = in1df[$keysLit].drop_duplicates().reset_index(drop=True) + |$aggLines + |$mergeLines""".stripMargin + } + } + + private def aggExprScalar(agg: AggregationOperation): String = { + val attrLit = + if (agg.attribute == null || agg.attribute.isEmpty) "None" + else pyStringLiteral(agg.attribute) + agg.aggFunction match { + case AggregationFunction.SUM => s"in1df[$attrLit].sum()" + case AggregationFunction.AVERAGE => s"in1df[$attrLit].mean()" + case AggregationFunction.MIN => s"in1df[$attrLit].min()" + case AggregationFunction.MAX => s"in1df[$attrLit].max()" + case AggregationFunction.COUNT => + if (agg.attribute == null || agg.attribute.isEmpty) "int(len(in1df))" + else s"int(in1df[$attrLit].count())" + case AggregationFunction.CONCAT => s"_texera_agg_concat(in1df[$attrLit])" + } + } + + private def aggExprGroupby(agg: AggregationOperation, groups: String): String = { + val attrLit = + if (agg.attribute == null || agg.attribute.isEmpty) "None" + else pyStringLiteral(agg.attribute) + val resultLit = pyStringLiteral(agg.resultAttribute) + agg.aggFunction match { + case AggregationFunction.SUM => s"$groups[$attrLit].sum().rename($resultLit)" + case AggregationFunction.AVERAGE => s"$groups[$attrLit].mean().rename($resultLit)" + case AggregationFunction.MIN => s"$groups[$attrLit].min().rename($resultLit)" + case AggregationFunction.MAX => s"$groups[$attrLit].max().rename($resultLit)" + case AggregationFunction.COUNT => + if (agg.attribute == null || agg.attribute.isEmpty) + s"$groups.size().rename($resultLit)" + else s"$groups[$attrLit].count().rename($resultLit)" + case AggregationFunction.CONCAT => + s"$groups[$attrLit].apply(_texera_agg_concat).rename($resultLit)" + } + } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/cartesianProduct/CartesianProductOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/cartesianProduct/CartesianProductOpDesc.scala index 16005c67a59..1f6aaf36e24 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/cartesianProduct/CartesianProductOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/cartesianProduct/CartesianProductOpDesc.scala @@ -23,10 +23,11 @@ import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.tuple.{Attribute, Schema} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow._ -import org.apache.texera.amber.operator.LogicalOp +import org.apache.texera.amber.operator.{LogicalOp, StandaloneCodeGenerator} import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} -class CartesianProductOpDesc extends LogicalOp { +class CartesianProductOpDesc extends LogicalOp with StandaloneCodeGenerator { + override def getPhysicalOp( workflowId: WorkflowIdentity, executionId: ExecutionIdentity @@ -103,4 +104,27 @@ class CartesianProductOpDesc extends LogicalOp { ), outputPorts = List(OutputPort()) ) + + // Schema mirrors SchemaPropagationFunc: left columns kept as-is, each right + // column renamed by repeatedly appending "#@1" while the candidate name + // collides with any left column OR any other right column's ORIGINAL name. + // The renamed-name table is recomputed at runtime from the actual DataFrame + // columns. Known divergence: row order — pandas cross-merge varies right + // fastest (L1R1, L1R2, L2R1, L2R2); the JVM op buffers left and emits per + // arriving right tuple (L1R1, L2R1, L1R2, L2R2). Cartesian product is set- + // semantically order-agnostic, so this is acceptable. + override def generateStandaloneCode(): String = { + """_left_cols = list(in1df.columns) + |_right_cols = list(in2df.columns) + |_left_set = set(_left_cols) + |_right_set = set(_right_cols) + |_rename = {} + |for _col in _right_cols: + | _new = _col + | _others = _right_set - {_col} + | while _new in _left_set or _new in _others: + | _new = _new + "#@1" + | _rename[_col] = _new + |out1df = in1df.merge(in2df.rename(columns=_rename), how="cross").reset_index(drop=True)""".stripMargin + } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/difference/DifferenceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/difference/DifferenceOpDesc.scala index c12f057a25b..887c1aa44e1 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/difference/DifferenceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/difference/DifferenceOpDesc.scala @@ -23,11 +23,10 @@ import com.google.common.base.Preconditions import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow._ -import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} +import org.apache.texera.amber.operator.{LogicalOp, StandaloneCodeGenerator} -class DifferenceOpDesc extends LogicalOp { - +class DifferenceOpDesc extends LogicalOp with StandaloneCodeGenerator { override def getPhysicalOp( workflowId: WorkflowIdentity, executionId: ExecutionIdentity @@ -61,4 +60,12 @@ class DifferenceOpDesc extends LogicalOp { ), outputPorts = List(OutputPort(blocking = true)) ) + + // Distinct rows in left (port 0) not in right (port 1). [left, right, right] + + // drop_duplicates(keep=False): right rows appear >=2x and drop, left-only + // survive. concat (not merge) so NaN == NaN matches the JVM HashSet. + override def generateStandaloneCode(): String = + "out1df = pd.concat([in1df.drop_duplicates(), in2df.drop_duplicates(), " + + "in2df.drop_duplicates()], ignore_index=True)" + + ".drop_duplicates(keep=False).reset_index(drop=True)" } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/dummy/DummyOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/dummy/DummyOpDesc.scala index 4a2c4c48d54..bee942cd370 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/dummy/DummyOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/dummy/DummyOpDesc.scala @@ -23,9 +23,14 @@ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle import org.apache.texera.amber.core.workflow.{InputPort, OutputPort, PortIdentity} import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} -import org.apache.texera.amber.operator.{LogicalOp, PortDescription, PortDescriptor} +import org.apache.texera.amber.operator.{ + LogicalOp, + PortDescription, + PortDescriptor, + StandaloneCodeGenerator +} -class DummyOpDesc extends LogicalOp with PortDescriptor { +class DummyOpDesc extends LogicalOp with PortDescriptor with StandaloneCodeGenerator { @JsonProperty @JsonSchemaTitle("Description") @@ -66,4 +71,12 @@ class DummyOpDesc extends LogicalOp with PortDescriptor { allowPortCustomization = true ) } + + override def generateStandaloneCode(): String = { + // Placeholder operator: pass the first input through to the output. + // Multi-port configurations don't fully translate under the current + // single-output placeholder scheme; downstream of extra ports would + // alias the same variable. + "out1df = in1df" + } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDesc.scala index 708546a84ad..469defde967 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDesc.scala @@ -29,13 +29,14 @@ import org.apache.texera.amber.core.virtualidentity.{ WorkflowIdentity } import org.apache.texera.amber.core.workflow._ -import org.apache.texera.amber.operator.LogicalOp +import org.apache.texera.amber.operator.{LogicalOp, StandaloneCodeGenerator} import org.apache.texera.amber.operator.hashJoin.HashJoinOpDesc.HASH_JOIN_INTERNAL_KEY_NAME import org.apache.texera.amber.operator.metadata.annotations.{ AutofillAttributeName, AutofillAttributeNameOnPort1 } import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral import org.apache.texera.amber.util.JSONUtils.objectMapper object HashJoinOpDesc { @@ -53,8 +54,7 @@ object HashJoinOpDesc { } } """) -class HashJoinOpDesc[K] extends LogicalOp { - +class HashJoinOpDesc[K] extends LogicalOp with StandaloneCodeGenerator { @JsonProperty(required = true) @JsonSchemaTitle("Left Input Attribute") @JsonPropertyDescription("attribute to be joined on the Left Input") @@ -200,4 +200,33 @@ class HashJoinOpDesc[K] extends LogicalOp { ), outputPorts = List(OutputPort()) ) + + // Equi-join: drop the probe key (kept only when its name differs from the + // build key), suffix colliding right columns "#@1" — matches JoinUtils. Known + // Texera divergences: row order, null keys (NaN != NaN in merge), outer + // anti-row column placement. + override def generateStandaloneCode(): String = { + val buildKeyLit = objectMapper.writeValueAsString(buildAttributeName) + val probeKeyLit = objectMapper.writeValueAsString(probeAttributeName) + val how = joinType match { + case JoinType.INNER => "inner" + case JoinType.LEFT_OUTER => "left" + case JoinType.RIGHT_OUTER => "right" + case JoinType.FULL_OUTER => "outer" + } + val merge = + s"""out1df = in1df.merge(in2df, how=${pyStringLiteral(how)}, left_on=$buildKeyLit, """ + + s"""right_on=$probeKeyLit, suffixes=("", "#@1"))""" + // When the left frame already carries the probe key's name, the merge + // suffixes the right key and leaves the left column alone, so dropping the + // bare name would take the left payload instead of the key. + val suffixedProbeKeyLit = objectMapper.writeValueAsString(probeAttributeName + "#@1") + val tail = + if (buildAttributeName != probeAttributeName) + s"out1df = out1df.drop(columns=[$suffixedProbeKeyLit if $probeKeyLit in in1df.columns " + + s"else $probeKeyLit]).reset_index(drop=True)" + else + "out1df = out1df.reset_index(drop=True)" + s"$merge\n$tail" + } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/ifStatement/IfOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/ifStatement/IfOpDesc.scala index f811ed34296..7ce5fa07941 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/ifStatement/IfOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/ifStatement/IfOpDesc.scala @@ -24,11 +24,12 @@ import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow._ -import org.apache.texera.amber.operator.LogicalOp +import org.apache.texera.amber.operator.{LogicalOp, StandaloneCodeGenerator} import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral import org.apache.texera.amber.util.JSONUtils.objectMapper -class IfOpDesc extends LogicalOp { +class IfOpDesc extends LogicalOp with StandaloneCodeGenerator { @JsonProperty(required = true) @JsonSchemaTitle("Condition State") @JsonPropertyDescription("name of the state variable to evaluate") @@ -72,4 +73,20 @@ class IfOpDesc extends LogicalOp { ), outputPorts = List(OutputPort(PortIdentity(), "False"), OutputPort(PortIdentity(1), "True")) ) + + // The engine flips the active output port when a State message arrives on the + // Condition port. A script has no State channel, so the condition is read from + // a global named after it, true when nothing set it, which is the port the + // engine starts on. The Condition input carries no rows to read either way. + override def generateStandaloneCode(): String = { + val globalName = "_texera_if_" + Option(conditionName).getOrElse("") + val globalLit = pyStringLiteral(globalName) + s"""_texera_if_cond = bool(globals().get($globalLit, True)) + |if _texera_if_cond: + | out2df = in2df.copy() + | out1df = in2df.iloc[0:0].copy() + |else: + | out1df = in2df.copy() + | out2df = in2df.iloc[0:0].copy()""".stripMargin + } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/intersect/IntersectOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/intersect/IntersectOpDesc.scala index 323d51b0ad5..ff2c564398b 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/intersect/IntersectOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/intersect/IntersectOpDesc.scala @@ -22,11 +22,10 @@ package org.apache.texera.amber.operator.intersect import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow._ -import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} +import org.apache.texera.amber.operator.{LogicalOp, StandaloneCodeGenerator} -class IntersectOpDesc extends LogicalOp { - +class IntersectOpDesc extends LogicalOp with StandaloneCodeGenerator { override def getPhysicalOp( workflowId: WorkflowIdentity, executionId: ExecutionIdentity @@ -52,4 +51,10 @@ class IntersectOpDesc extends LogicalOp { inputPorts = List(InputPort(PortIdentity()), InputPort(PortIdentity(1))), outputPorts = List(OutputPort(blocking = true)) ) + + // Distinct rows in both inputs. concat + duplicated (not pd.merge) so NaN == + // NaN matches the JVM HashSet's null equality. + override def generateStandaloneCode(): String = + """_both = pd.concat([in1df.drop_duplicates(), in2df.drop_duplicates()], ignore_index=True) + |out1df = _both[_both.duplicated(keep="first")].reset_index(drop=True)""".stripMargin } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDesc.scala index 2218fe2cce1..81086827c1e 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDesc.scala @@ -26,7 +26,7 @@ import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.tuple.{Attribute, Schema} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow._ -import org.apache.texera.amber.operator.LogicalOp +import org.apache.texera.amber.operator.{LogicalOp, StandaloneCodeGenerator} import org.apache.texera.amber.operator.metadata.annotations.{ AutofillAttributeName, AutofillAttributeNameOnPort1 @@ -52,8 +52,7 @@ import org.apache.texera.amber.util.JSONUtils.objectMapper } } """) -class IntervalJoinOpDesc extends LogicalOp { - +class IntervalJoinOpDesc extends LogicalOp with StandaloneCodeGenerator { @JsonProperty(required = true) @JsonSchemaTitle("Left Input attr") @JsonPropertyDescription("Choose one attribute in the left table") @@ -147,6 +146,40 @@ class IntervalJoinOpDesc extends LogicalOp { outputPorts = List(OutputPort()) ) + // Inner interval join: left point in [rightKey, rightKey + constant], bounds + // toggled by include{Left,Right}Bound. Cross-join + mask computes the full + // result (no sorted-input assumption, unlike the exec). Runtime dtype check + // picks numeric vs pd.DateOffset (unit from timeIntervalType). + // + // Both keys are read off the merged frame rather than copied into columns of + // the operator's own naming, which would overwrite an input column that + // happens to carry that name and then drop it. Only the right key needs + // locating: the merge leaves every left name alone and suffixes a right one + // that collides. + override def generateStandaloneCode(): String = { + val leftLit = objectMapper.writeValueAsString(leftAttributeName) + val rightLit = objectMapper.writeValueAsString(rightAttributeName) + val loOp = if (includeLeftBound) ">=" else ">" + val hiOp = if (includeRightBound) "<=" else "<" + val offsetUnit = Option(timeIntervalType).flatten match { + case Some(TimeIntervalType.YEAR) => "years" + case Some(TimeIntervalType.MONTH) => "months" + case Some(TimeIntervalType.HOUR) => "hours" + case Some(TimeIntervalType.MINUTE) => "minutes" + case Some(TimeIntervalType.SECOND) => "seconds" + case _ => "days" // DAY or unset + } + s"""_pairs = in1df.merge(in2df, how="cross", suffixes=("", "#@1")) + |_iv_l = _pairs[$leftLit] + |_iv_r = _pairs[$rightLit if $rightLit not in in1df.columns else $rightLit + "#@1"] + |if pd.api.types.is_datetime64_any_dtype(_iv_r): + | _iv_hi = _iv_r + pd.DateOffset($offsetUnit=$constant) + |else: + | _iv_hi = _iv_r + $constant + |_iv_match = (_iv_l $loOp _iv_r) & (_iv_l $hiOp _iv_hi) + |out1df = _pairs[_iv_match].reset_index(drop=True)""".stripMargin + } + def this( leftTableAttributeName: String, rightTableAttributeName: String, diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/randomksampling/RandomKSamplingOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/randomksampling/RandomKSamplingOpDesc.scala index 31cafcbc76b..188d3e38ef4 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/randomksampling/RandomKSamplingOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/randomksampling/RandomKSamplingOpDesc.scala @@ -23,11 +23,12 @@ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.{InputPort, OutputPort, PhysicalOp} +import org.apache.texera.amber.operator.{SamplingHelpers, StandaloneCodeGenerator} import org.apache.texera.amber.operator.filter.FilterOpDesc import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.util.JSONUtils.objectMapper -class RandomKSamplingOpDesc extends FilterOpDesc { +class RandomKSamplingOpDesc extends FilterOpDesc with StandaloneCodeGenerator { @JsonProperty(value = "random k sample percentage", required = true) @JsonPropertyDescription("random k sampling with given percentage") @@ -60,4 +61,23 @@ class RandomKSamplingOpDesc extends FilterOpDesc { outputPorts = List(OutputPort()), supportReconfiguration = true ) + + override def standaloneHelpers(): Seq[String] = Seq(SamplingHelpers.JavaRandom) + + // A per-row Bernoulli filter drawn from the same generator, so a rerun keeps + // the exact rows rather than merely the same proportion of them. + // + // The engine seeds it with the worker count, which makes the surviving rows a + // property of the deployment. One process states the one seed it can and so + // agrees with a single-worker run; a wider one splits the input and samples + // each share from the start of the sequence, which no seed here reproduces. + override def generateStandaloneCode(): String = { + val p = percentage / 100.0 + s"""_texera_rks_rng = _TexeraJavaRandom(1) + |_texera_rks_mask = pd.Series( + | [$p >= _texera_rks_rng.next_double() for _ in range(len(in1df))], + | index=in1df.index, dtype=bool + |) + |out1df = in1df[_texera_rks_mask].reset_index(drop=True)""".stripMargin + } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/reservoirsampling/ReservoirSamplingOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/reservoirsampling/ReservoirSamplingOpDesc.scala index eda2aa299fe..efa43d253a6 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/reservoirsampling/ReservoirSamplingOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/reservoirsampling/ReservoirSamplingOpDesc.scala @@ -23,11 +23,11 @@ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.{InputPort, OutputPort, PhysicalOp} -import org.apache.texera.amber.operator.LogicalOp +import org.apache.texera.amber.operator.{LogicalOp, SamplingHelpers, StandaloneCodeGenerator} import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.util.JSONUtils.objectMapper -class ReservoirSamplingOpDesc extends LogicalOp { +class ReservoirSamplingOpDesc extends LogicalOp with StandaloneCodeGenerator { @JsonProperty(value = "number of item sampled in reservoir sampling", required = true) @JsonPropertyDescription("reservoir sampling with k items being kept randomly") @@ -60,4 +60,27 @@ class ReservoirSamplingOpDesc extends LogicalOp { outputPorts = List(OutputPort()) ) } + + override def standaloneHelpers(): Seq[String] = Seq(SamplingHelpers.JavaRandom) + + // The executor runs Algorithm R (Vitter): fill the reservoir with the first k + // tuples, then for tuple m+1 (m >= k) draw i = rand.nextInt(m), uniform in + // [0, m), and replace reservoir[i] iff i < k. + // + // Its generator is seeded with the worker count, so the rows agree with a + // single-worker run and not with a wider one. See RandomKSamplingOpDesc for + // why no seed closes that gap. + override def generateStandaloneCode(): String = { + s"""_texera_rs_rng = _TexeraJavaRandom(1) + |_texera_rs_k = $k + |_texera_rs_reservoir = [] + |for _texera_rs_n, _texera_rs_row in enumerate(in1df.itertuples(index=False, name=None)): + | if _texera_rs_n < _texera_rs_k: + | _texera_rs_reservoir.append(_texera_rs_row) + | else: + | _texera_rs_i = _texera_rs_rng.next_int(_texera_rs_n) + | if _texera_rs_i < _texera_rs_k: + | _texera_rs_reservoir[_texera_rs_i] = _texera_rs_row + |out1df = pd.DataFrame(_texera_rs_reservoir, columns=list(in1df.columns)).reset_index(drop=True)""".stripMargin + } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sleep/SleepOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sleep/SleepOpDesc.scala index 3eee3cd33b7..1986eb52d83 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sleep/SleepOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sleep/SleepOpDesc.scala @@ -24,11 +24,11 @@ import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.{InputPort, OutputPort, PhysicalOp} -import org.apache.texera.amber.operator.LogicalOp +import org.apache.texera.amber.operator.{LogicalOp, StandaloneCodeGenerator} import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.util.JSONUtils.objectMapper -class SleepOpDesc extends LogicalOp { +class SleepOpDesc extends LogicalOp with StandaloneCodeGenerator { @JsonProperty(required = true) @JsonSchemaTitle("Sleep Time (seconds)") @@ -63,4 +63,10 @@ class SleepOpDesc extends LogicalOp { inputPorts = List(InputPort()), outputPorts = List(OutputPort()) ) + + override def generateStandaloneCode(): String = { + // JVM op sleeps between each tuple; pandas operates in batch, not row-by-row, + // so per-row sleep has no equivalent. Translate as a passthrough. + "out1df = in1df" + } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sort/SortCriteriaUnit.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sort/SortCriteriaUnit.scala index f0e851110b8..9b34fe44b1a 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sort/SortCriteriaUnit.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sort/SortCriteriaUnit.scala @@ -21,13 +21,17 @@ package org.apache.texera.amber.operator.sort import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString -import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName +import org.apache.texera.amber.operator.metadata.annotations.{AutofillAttributeName, SampleColumn} class SortCriteriaUnit { + // Verification picks a column with TIE GROUPS. Sorting on a unique key never + // puts two equal keys next to each other, so the stability this operator is + // named for goes unchecked. @JsonProperty(value = "attribute", required = true) @JsonPropertyDescription("Attribute name to sort by") @AutofillAttributeName + @SampleColumn("name") var attributeName: EncodableString = _ @JsonProperty(value = "sortPreference", required = true) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sort/SortOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sort/SortOpDesc.scala index db2ae21e7f6..662fd37377d 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sort/SortOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sort/SortOpDesc.scala @@ -21,11 +21,18 @@ package org.apache.texera.amber.operator.sort import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import org.apache.texera.amber.core.tuple.Schema -import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ + PythonTemplateBuilderStringContext, + pyStringLiteral +} import org.apache.texera.amber.core.workflow.{InputPort, OutputPort, PortIdentity} -import org.apache.texera.amber.operator.PythonOperatorDescriptor +import org.apache.texera.amber.operator.{PythonOperatorDescriptor, StandaloneCodeGenerator} import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} -class SortOpDesc extends PythonOperatorDescriptor { +class SortOpDesc extends PythonOperatorDescriptor with StandaloneCodeGenerator { + + // Sorting is this operator's contract: output row order is meaningful. + override def orderSensitive: Boolean = true + @JsonProperty(required = true) @JsonPropertyDescription("column to perform sorting on") var attributes: List[SortCriteriaUnit] = List.empty @@ -78,4 +85,11 @@ class SortOpDesc extends PythonOperatorDescriptor { outputPorts = List(OutputPort(blocking = true)) ) + override def generateStandaloneCode(): String = { + val cols = attributes.map(c => pyStringLiteral(c.attributeName)).mkString("[", ", ", "]") + val ascending = attributes + .map(c => if (c.sortPreference == SortPreference.ASC) "True" else "False") + .mkString("[", ", ", "]") + s"out1df = in1df.sort_values(by=$cols, ascending=$ascending)" + } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpDesc.scala index 318089da5e3..aedf6a0f6cb 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpDesc.scala @@ -24,8 +24,9 @@ import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.{InputPort, OutputPort, PhysicalOp} -import org.apache.texera.amber.operator.LogicalOp +import org.apache.texera.amber.operator.{LogicalOp, StandaloneCodeGenerator} import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral import org.apache.texera.amber.util.JSONUtils.objectMapper import scala.collection.mutable.ListBuffer @@ -36,7 +37,10 @@ import scala.collection.mutable.ListBuffer * the lexicographic order and per-key direction (ASC/DESC). */ //TODO(#3922): disallowing sorting on binary type -class StableMergeSortOpDesc extends LogicalOp { +class StableMergeSortOpDesc extends LogicalOp with StandaloneCodeGenerator { + + // Sorting is this operator's contract: output row order is meaningful. + override def orderSensitive: Boolean = true @JsonProperty(value = "keys", required = true) @JsonSchemaTitle("Sort Keys") @@ -69,4 +73,23 @@ class StableMergeSortOpDesc extends LogicalOp { List(InputPort()), List(OutputPort(blocking = true)) ) + + // JVM op runs an incremental stable merge sort with NULLS-LAST regardless of + // direction. pandas' sort_values(kind="mergesort") is stable, and + // na_position="last" is direction-independent — matches the JVM null policy + // exactly. Known divergence: floating-point NaN. JVM uses Double.compare, + // which orders NaN > +Inf (so NaN sorts last in ASC, first in DESC). pandas + // treats NaN as a missing value and always places it at na_position, so DESC + // ordering of a Double column with NaNs will differ. + override def generateStandaloneCode(): String = { + val criteria = Option(keys).getOrElse(ListBuffer.empty) + if (criteria.isEmpty) return "out1df = in1df.copy()" + val cols = criteria + .map(c => pyStringLiteral(c.attributeName)) + .mkString("[", ", ", "]") + val ascending = criteria + .map(c => if (c.sortPreference == SortPreference.ASC) "True" else "False") + .mkString("[", ", ", "]") + s"""out1df = in1df.sort_values(by=$cols, ascending=$ascending, kind="mergesort", na_position="last").reset_index(drop=True)""" + } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sortPartitions/SortPartitionsOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sortPartitions/SortPartitionsOpDesc.scala index 9290d549f2e..7edbb9a782f 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sortPartitions/SortPartitionsOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sortPartitions/SortPartitionsOpDesc.scala @@ -24,9 +24,10 @@ import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchema import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.{InputPort, OutputPort, PhysicalOp, RangePartition} -import org.apache.texera.amber.operator.LogicalOp +import org.apache.texera.amber.operator.{LogicalOp, StandaloneCodeGenerator} import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral import org.apache.texera.amber.util.JSONUtils.objectMapper @JsonSchemaInject(json = """ @@ -38,7 +39,10 @@ import org.apache.texera.amber.util.JSONUtils.objectMapper } } """) -class SortPartitionsOpDesc extends LogicalOp { +class SortPartitionsOpDesc extends LogicalOp with StandaloneCodeGenerator { + + // Sorting is this operator's contract: output row order is meaningful. + override def orderSensitive: Boolean = true @JsonProperty(required = true) @JsonSchemaTitle("Attribute") @@ -84,4 +88,16 @@ class SortPartitionsOpDesc extends LogicalOp { inputPorts = List(InputPort()), outputPorts = List(OutputPort(blocking = true)) ) + + // Row order is this operator's answer, so the sort matches the engine's: + // stable, ascending, nulls first. The domain bounds are partitioning hints + // that the sort itself never reads. + // + // NaN is where the two part. The engine orders it after positive infinity; + // pandas treats it as missing and puts it where the nulls go, which is first. + override def generateStandaloneCode(): String = { + val col = pyStringLiteral(Option(sortAttributeName).getOrElse("")) + s"""out1df = in1df.sort_values(by=$col, ascending=True, kind="mergesort", na_position="first").reset_index(drop=True)""" + } + } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/split/SplitOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/split/SplitOpDesc.scala index c3c34df948c..aa3c377da59 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/split/SplitOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/split/SplitOpDesc.scala @@ -29,12 +29,12 @@ import com.kjetland.jackson.jsonSchema.annotations.{ import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow._ -import org.apache.texera.amber.operator.LogicalOp +import org.apache.texera.amber.operator.{LogicalOp, SamplingHelpers, StandaloneCodeGenerator} import org.apache.texera.amber.operator.metadata.annotations.HideAnnotation import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.util.JSONUtils.objectMapper -class SplitOpDesc extends LogicalOp { +class SplitOpDesc extends LogicalOp with StandaloneCodeGenerator { @JsonSchemaTitle("Split Percentage") @JsonProperty(defaultValue = "80") @@ -99,4 +99,25 @@ class SplitOpDesc extends LogicalOp { ) } + override def standaloneHelpers(): Seq[String] = Seq(SamplingHelpers.JavaRandom) + + override def generateStandaloneCode(): String = { + // The executor sends each tuple to the upper port iff nextInt(100) < k, + // drawing from one generator per run. Reproducing that draw keeps the same + // rows on the same side; the mask drives both ports, so `out1df` takes the + // upper k% and `out2df` the remainder and the two partition the input. + // + // With "Auto-Generate Seed" the executor seeds from the clock, so that run + // is not reproducible by anything, itself included — the script then seeds + // from its own clock, matching the intent rather than a particular run. + val seedExpr = if (random) "int(_texera_time.time() * 1000)" else seed.toString + val timeImport = if (random) "import time as _texera_time\n" else "" + s"""${timeImport}_texera_split_rng = _TexeraJavaRandom($seedExpr) + |_texera_split_mask = pd.Series( + | [_texera_split_rng.next_int(100) < $k for _ in range(len(in1df))], + | index=in1df.index, dtype=bool + |) + |out1df = in1df[_texera_split_mask].reset_index(drop=True) + |out2df = in1df[~_texera_split_mask].reset_index(drop=True)""".stripMargin + } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/symmetricDifference/SymmetricDifferenceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/symmetricDifference/SymmetricDifferenceOpDesc.scala index d0d6deecbe6..ad073d53628 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/symmetricDifference/SymmetricDifferenceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/symmetricDifference/SymmetricDifferenceOpDesc.scala @@ -23,11 +23,10 @@ import com.google.common.base.Preconditions import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow._ -import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} +import org.apache.texera.amber.operator.{LogicalOp, StandaloneCodeGenerator} -class SymmetricDifferenceOpDesc extends LogicalOp { - +class SymmetricDifferenceOpDesc extends LogicalOp with StandaloneCodeGenerator { override def getPhysicalOp( workflowId: WorkflowIdentity, executionId: ExecutionIdentity @@ -62,4 +61,9 @@ class SymmetricDifferenceOpDesc extends LogicalOp { outputPorts = List(OutputPort(blocking = true)) ) + // Distinct rows in exactly one input. drop_duplicates(keep=False) drops rows + // seen on both sides. concat (not merge) so NaN == NaN matches the JVM HashSet. + override def generateStandaloneCode(): String = + "out1df = pd.concat([in1df.drop_duplicates(), in2df.drop_duplicates()], " + + "ignore_index=True).drop_duplicates(keep=False).reset_index(drop=True)" } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/dummy/DummyOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/dummy/DummyOpDescSpec.scala index 6f23261b271..4ee8cfe7fce 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/dummy/DummyOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/dummy/DummyOpDescSpec.scala @@ -71,6 +71,11 @@ class DummyOpDescSpec extends AnyFlatSpec with Matchers { (new DummyOpDesc).dummyOperator shouldBe "" } + "DummyOpDesc.generateStandaloneCode" should + "pass the first input through to the output" in { + (new DummyOpDesc).generateStandaloneCode() shouldBe "out1df = in1df" + } + "DummyOpDesc.getPhysicalOp" should "be the unimplemented LogicalOp stub (throws NotImplementedError)" in { intercept[NotImplementedError] { diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDescSpec.scala index 4ead90b6456..fe3fed8899b 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDescSpec.scala @@ -19,6 +19,7 @@ package org.apache.texera.amber.operator.hashJoin +import com.typesafe.config.ConfigFactory import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema} import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.operator.LogicalOp @@ -27,6 +28,12 @@ import org.apache.texera.amber.util.JSONUtils.objectMapper import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.util.concurrent.TimeUnit +import scala.io.Source +import scala.util.Try + class HashJoinOpDescSpec extends AnyFlatSpec with Matchers { private def leftRight(): (Schema, Schema) = @@ -75,6 +82,77 @@ class HashJoinOpDescSpec extends AnyFlatSpec with Matchers { out(d.operatorInfo.outputPorts.head.id).getAttributeNames shouldBe List("a", "k", "k#@1") } + "HashJoinOpDesc.generateStandaloneCode" should + "drop the probe key under the name the merge left it with" in { + val d = new HashJoinOpDesc[String] + d.buildAttributeName = "id" + d.probeAttributeName = "key" + // "key" is only suffixed when the left frame has a column of that name, and + // the frame is not known until the script runs, so the choice is in Python. + d.generateStandaloneCode() should include( + """drop(columns=["key#@1" if "key" in in1df.columns else "key"])""" + ) + } + + it should "keep the shared key when both sides name it the same" in { + val d = new HashJoinOpDesc[String] + d.buildAttributeName = "k" + d.probeAttributeName = "k" + d.generateStandaloneCode() should not include "drop(columns=" + } + + /** The bug this pins: with a left `key` payload and a right `key` join column, + * dropping the bare `key` took the payload and kept the right key. Run the + * generated pandas and hold its columns against the schema the operator + * itself promises for the same two inputs. + */ + it should "return the columns getExternalOutputSchemas promises" in { + val python = resolvePython().getOrElse( + cancel("No runnable python executable (udf.conf python.path, python3, python, py)") + ) + if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + + val left = Schema() + .add(new Attribute("id", AttributeType.INTEGER)) + .add(new Attribute("key", AttributeType.STRING)) + val right = Schema() + .add(new Attribute("key", AttributeType.INTEGER)) + .add(new Attribute("value", AttributeType.INTEGER)) + + val d = new HashJoinOpDesc[Integer] + d.buildAttributeName = "id" + d.probeAttributeName = "key" + val promised = d + .getExternalOutputSchemas(Map(PortIdentity() -> left, PortIdentity(1) -> right))( + d.operatorInfo.outputPorts.head.id + ) + .getAttributeNames + + val driver = + s"""import pandas as pd + | + |in1df = pd.DataFrame({"id": [1], "key": ["payload"]}) + |in2df = pd.DataFrame({"key": [1], "value": [9]}) + |${d.generateStandaloneCode()} + |print(",".join(map(str, out1df.columns))) + |print(",".join(map(str, out1df.iloc[0].tolist()))) + |""".stripMargin + + val script = Files.createTempFile("hashjoin-columns-", ".py") + script.toFile.deleteOnExit() + Files.write(script, driver.getBytes(StandardCharsets.UTF_8)) + val process = new ProcessBuilder(python, script.toString).redirectErrorStream(true).start() + val out = Source.fromInputStream(process.getInputStream).mkString + process.waitFor(120, TimeUnit.SECONDS) + withClue(s"python said:\n$out\nscript:\n$driver") { + process.exitValue() shouldBe 0 + val lines = out.trim.linesIterator.toSeq + lines.head.split(",").toList shouldBe promised + // The payload the drop used to take, not the right key it used to keep. + lines(1) shouldBe "1,payload,9" + } + } + "HashJoinOpDesc" should "round-trip its config fields through the polymorphic base" in { val d = new HashJoinOpDesc[String] d.buildAttributeName = "lk" @@ -89,4 +167,33 @@ class HashJoinOpDescSpec extends AnyFlatSpec with Matchers { r.probeAttributeName shouldBe "rk" r.joinType shouldBe JoinType.LEFT_OUTER } + + // Python resolution follows FilledAreaPlotOpDescSpec: udf.conf python.path + // (UDF_PYTHON_PATH), then python3 / python / py. + private def resolvePython(): Option[String] = { + def fromConfig: Option[String] = + Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption + .orElse(Try(ConfigFactory.load()).toOption) + .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) + .map(_.trim) + .filter(_.nonEmpty) + + def runnable(exe: String): Boolean = + Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption + .exists { p => + if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false } + else p.exitValue() == 0 + } + + (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(runnable) + } + + private def canImportPandas(python: String): Boolean = + Try( + new ProcessBuilder(python, "-c", "import pandas").redirectErrorStream(true).start() + ).toOption + .exists { p => + if (!p.waitFor(60, TimeUnit.SECONDS)) { p.destroyForcibly(); false } + else p.exitValue() == 0 + } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDescSpec.scala index 01d9d2e10b5..cba15368c5e 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDescSpec.scala @@ -19,6 +19,7 @@ package org.apache.texera.amber.operator.intervalJoin +import com.typesafe.config.ConfigFactory import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} @@ -29,6 +30,12 @@ import org.apache.texera.amber.util.JSONUtils.objectMapper import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.util.concurrent.TimeUnit +import scala.io.Source +import scala.util.Try + class IntervalJoinOpDescSpec extends AnyFlatSpec with Matchers { private val workflowId = WorkflowIdentity(1L) @@ -104,6 +111,69 @@ class IntervalJoinOpDescSpec extends AnyFlatSpec with Matchers { List("a", "k", "b", "k#@1") } + "IntervalJoinOpDesc.generateStandaloneCode" should + "read the right key under the name the merge left it with" in { + // "rk" is only suffixed when the left frame has a column of that name, and + // the frame is not known until the script runs, so the choice is in Python. + configured().generateStandaloneCode() should include( + """_iv_r = _pairs["rk" if "rk" not in in1df.columns else "rk" + "#@1"]""" + ) + } + + /** The bug this pins: the keys used to be copied into `_iv_l` and `_iv_r` + * columns, so an input column already carrying one of those names was + * overwritten and then dropped. Run the generated pandas and hold its + * columns against the schema the operator promises for the same two inputs. + */ + it should "keep an input column that carries a temporary's name" in { + val python = resolvePython().getOrElse( + cancel("No runnable python executable (udf.conf python.path, python3, python, py)") + ) + if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + + val left = Schema() + .add(new Attribute("x", AttributeType.INTEGER)) + .add(new Attribute("_iv_l", AttributeType.STRING)) + val right = Schema().add(new Attribute("r", AttributeType.INTEGER)) + + val d = new IntervalJoinOpDesc + d.leftAttributeName = "x" + d.rightAttributeName = "r" + d.constant = 3L + d.timeIntervalType = None + val promised = d + .getPhysicalOp(workflowId, executionId) + .propagateSchema + .func(Map(PortIdentity() -> left, PortIdentity(1) -> right))( + d.operatorInfo.outputPorts.head.id + ) + .getAttributeNames + + val driver = + s"""import pandas as pd + | + |in1df = pd.DataFrame({"x": [2], "_iv_l": ["payload"]}) + |in2df = pd.DataFrame({"r": [1]}) + |${d.generateStandaloneCode()} + |print(",".join(map(str, out1df.columns))) + |print(",".join(map(str, out1df.iloc[0].tolist()))) + |""".stripMargin + + val script = Files.createTempFile("intervaljoin-columns-", ".py") + script.toFile.deleteOnExit() + Files.write(script, driver.getBytes(StandardCharsets.UTF_8)) + val process = new ProcessBuilder(python, script.toString).redirectErrorStream(true).start() + val out = Source.fromInputStream(process.getInputStream).mkString + process.waitFor(120, TimeUnit.SECONDS) + withClue(s"python said:\n$out\nscript:\n$driver") { + process.exitValue() shouldBe 0 + val lines = out.trim.linesIterator.toSeq + lines.head.split(",").toList shouldBe promised + // The payload the temporary used to overwrite, and the row still joins. + lines(1) shouldBe "2,payload,1" + } + } + "IntervalJoinOpDesc" should "round-trip its fields through the polymorphic base" in { val d = configured() d.constant = 42L @@ -118,4 +188,33 @@ class IntervalJoinOpDescSpec extends AnyFlatSpec with Matchers { ij.includeLeftBound shouldBe false ij.includeRightBound shouldBe false } + + // Python resolution follows FilledAreaPlotOpDescSpec: udf.conf python.path + // (UDF_PYTHON_PATH), then python3 / python / py. + private def resolvePython(): Option[String] = { + def fromConfig: Option[String] = + Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption + .orElse(Try(ConfigFactory.load()).toOption) + .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) + .map(_.trim) + .filter(_.nonEmpty) + + def runnable(exe: String): Boolean = + Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption + .exists { p => + if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false } + else p.exitValue() == 0 + } + + (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(runnable) + } + + private def canImportPandas(python: String): Boolean = + Try( + new ProcessBuilder(python, "-c", "import pandas").redirectErrorStream(true).start() + ).toOption + .exists { p => + if (!p.waitFor(60, TimeUnit.SECONDS)) { p.destroyForcibly(); false } + else p.exitValue() == 0 + } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/randomksampling/RandomKSamplingOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/randomksampling/RandomKSamplingOpDescSpec.scala index 94d450a9559..7d6f3781abb 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/randomksampling/RandomKSamplingOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/randomksampling/RandomKSamplingOpDescSpec.scala @@ -59,6 +59,30 @@ class RandomKSamplingOpDescSpec extends AnyFlatSpec with Matchers { restored.asInstanceOf[RandomKSamplingOpDesc].percentage shouldBe 25 } + "RandomKSamplingOpDesc.generateStandaloneCode" should + "draw a per-row Bernoulli mask from a seeded RNG" in { + val d = new RandomKSamplingOpDesc + d.percentage = 25 + d.generateStandaloneCode() shouldBe + """_texera_rks_rng = _TexeraJavaRandom(1) + |_texera_rks_mask = pd.Series( + | [0.25 >= _texera_rks_rng.next_double() for _ in range(len(in1df))], + | index=in1df.index, dtype=bool + |) + |out1df = in1df[_texera_rks_mask].reset_index(drop=True)""".stripMargin + } + + // percentage is a whole number on the descriptor but a probability in the + // generated code; an integer division here would keep no rows at all. + it should "convert the percentage to a probability" in { + Seq(0 -> "0.0", 30 -> "0.3", 100 -> "1.0").foreach { + case (pct, prob) => + val d = new RandomKSamplingOpDesc + d.percentage = pct + d.generateStandaloneCode() should include(s"[$prob >= _texera_rks_rng.next_double()") + } + } + "RandomKSamplingOpDesc.getPhysicalOp" should "wire the RandomKSamplingOpExec class name and carry ports" in { val d = new RandomKSamplingOpDesc diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/reservoirsampling/ReservoirSamplingOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/reservoirsampling/ReservoirSamplingOpDescSpec.scala index 0c903d6ce88..faf0b60aa97 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/reservoirsampling/ReservoirSamplingOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/reservoirsampling/ReservoirSamplingOpDescSpec.scala @@ -59,6 +59,27 @@ class ReservoirSamplingOpDescSpec extends AnyFlatSpec with Matchers { restored.asInstanceOf[ReservoirSamplingOpDesc].k shouldBe 100 } + // Algorithm R: fill the reservoir with the first k rows, then replace a + // uniformly-drawn slot for each later row. Pin the whole snippet — the + // generated Python is indentation-sensitive. + "ReservoirSamplingOpDesc.generateStandaloneCode" should + "emit a seeded Algorithm R reservoir over the input rows" in { + val d = new ReservoirSamplingOpDesc + d.k = 3 + d.generateStandaloneCode() shouldBe + """_texera_rs_rng = _TexeraJavaRandom(1) + |_texera_rs_k = 3 + |_texera_rs_reservoir = [] + |for _texera_rs_n, _texera_rs_row in enumerate(in1df.itertuples(index=False, name=None)): + | if _texera_rs_n < _texera_rs_k: + | _texera_rs_reservoir.append(_texera_rs_row) + | else: + | _texera_rs_i = _texera_rs_rng.next_int(_texera_rs_n) + | if _texera_rs_i < _texera_rs_k: + | _texera_rs_reservoir[_texera_rs_i] = _texera_rs_row + |out1df = pd.DataFrame(_texera_rs_reservoir, columns=list(in1df.columns)).reset_index(drop=True)""".stripMargin + } + "ReservoirSamplingOpDesc.getPhysicalOp" should "wire the ReservoirSamplingOpExec class name and carry ports" in { val d = new ReservoirSamplingOpDesc diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sleep/SleepOpExecSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sleep/SleepOpExecSpec.scala index 2aa143f2948..c5280d30994 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sleep/SleepOpExecSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sleep/SleepOpExecSpec.scala @@ -54,4 +54,21 @@ class SleepOpExecSpec extends AnyFlatSpec { val emitted = (0 until 5).flatMap(i => exec.processTuple(tuple(i), 0).toList) assert(emitted == (0 until 5).map(tuple).toList) } + + // The one thing the operator is for. Every other case here configures no delay, + // so the seconds-to-millis conversion was the only line nothing asked about: + // dropping it, or losing the factor of a thousand, left them all passing. + // + // One second is the smallest delay the field can ask for, sleepTime being whole + // seconds, and one tuple is enough to observe it. The lower bound is asserted + // rather than a window: a loaded machine may take longer, and a delay that + // overshoots is not the failure this guards against. + it should "delay each tuple by the configured number of seconds" in { + val exec = new SleepOpExec(descString(1)) + val startNanos = System.nanoTime() + val emitted = exec.processTuple(tuple(1), 0).toList + val elapsedMillis = (System.nanoTime() - startNanos) / 1000000 + assert(emitted == List(tuple(1))) + assert(elapsedMillis >= 1000, s"expected at least 1000 ms of sleep, took $elapsedMillis ms") + } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/split/SplitOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/split/SplitOpDescSpec.scala index d1fecdf7834..d0a769e9472 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/split/SplitOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/split/SplitOpDescSpec.scala @@ -126,6 +126,44 @@ class SplitOpDescSpec extends AnyFlatSpec with Matchers { } } + // --------------------------------------------------------------------------- + // generateStandaloneCode + // --------------------------------------------------------------------------- + + // The generator is a transcription of java.util.Random, so the script draws the + // rows the executor drew rather than a differently-shaped sample from the same + // distribution. With auto-generate on, the seed comes from the clock, as it does + // on the executor. + "SplitOpDesc.generateStandaloneCode" should + "draw an unseeded mask and partition the input across both outputs" in { + (new SplitOpDesc).generateStandaloneCode() shouldBe + """import time as _texera_time + |_texera_split_rng = _TexeraJavaRandom(int(_texera_time.time() * 1000)) + |_texera_split_mask = pd.Series( + | [_texera_split_rng.next_int(100) < 80 for _ in range(len(in1df))], + | index=in1df.index, dtype=bool + |) + |out1df = in1df[_texera_split_mask].reset_index(drop=True) + |out2df = in1df[~_texera_split_mask].reset_index(drop=True)""".stripMargin + } + + it should "pass the configured seed to the generator when auto-generate is off" in { + val d = new SplitOpDesc + d.random = false + d.seed = 42 + d.generateStandaloneCode() should include("_texera_split_rng = _TexeraJavaRandom(42)") + } + + // k is a percentage on both sides, compared as the executor compares it: + // nextInt(100) < k, not a uniform draw against k/100. + it should "compare the draw against the percentage the executor uses" in { + Seq(0, 30, 80, 100).foreach { k => + val d = new SplitOpDesc + d.k = k + d.generateStandaloneCode() should include(s"_texera_split_rng.next_int(100) < $k") + } + } + // --------------------------------------------------------------------------- // Independent instances // --------------------------------------------------------------------------- From ef38771886e00da2c1e26ccac5df3935f0c27117 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Fri, 18 Sep 2026 11:33:08 -0700 Subject: [PATCH 02/32] test(operator): run both routes of the exported If The engine picks If's route from a State message on the Condition port, and the verification harness writes rows per port with no State channel, so a fixture can only ever reach the default route, True. The exported block reads the same decision from a global instead, which the spec can set, so both routes are run there: the rows leave by one output and the other comes back empty. Co-Authored-By: Claude Opus 5 (1M context) --- .../operator/ifStatement/IfOpDescSpec.scala | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/ifStatement/IfOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/ifStatement/IfOpDescSpec.scala index 49cd325d572..2699886ca7b 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/ifStatement/IfOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/ifStatement/IfOpDescSpec.scala @@ -19,6 +19,7 @@ package org.apache.texera.amber.operator.ifStatement +import com.typesafe.config.ConfigFactory import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} @@ -27,6 +28,12 @@ import org.apache.texera.amber.operator.metadata.OperatorGroupConstants import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.util.concurrent.TimeUnit +import scala.io.Source +import scala.util.Try + class IfOpDescSpec extends AnyFlatSpec with Matchers { private val workflowId = WorkflowIdentity(1L) @@ -76,4 +83,80 @@ class IfOpDescSpec extends AnyFlatSpec with Matchers { ) out shouldBe Map(PortIdentity() -> dataSchema, PortIdentity(1) -> dataSchema) } + + "IfOpDesc.generateStandaloneCode" should + "name the switch after the condition so two Ifs in one script do not share it" in { + val op = new IfOpDesc + op.conditionName = "ready" + op.generateStandaloneCode() should include("\"_texera_if_ready\"") + } + + // The engine picks the route from a State message, which the verification harness + // has no channel for, so the exported block is where both routes can be run. False + // is the half a fixture cannot reach: with no State the engine keeps its default, + // and that default is True. + it should "send the rows to True by default and to False when the switch is off" in { + val python = resolvePythonExecutable().getOrElse(cancel("No runnable python executable")) + if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + + val op = new IfOpDesc + op.conditionName = "ready" + val body = op.generateStandaloneCode().linesIterator.map(" " + _).mkString("\n") + + val driver = + s"""import pandas as pd + | + |for _switch in (None, True, False): + | in2df = pd.DataFrame({"id": [1, 2, 3]}) + | if _switch is None: + | globals().pop("_texera_if_ready", None) + | else: + | globals()["_texera_if_ready"] = _switch + |$body + | print(_switch, list(out1df["id"]), list(out2df["id"])) + |""".stripMargin + + val script = Files.createTempFile("if-standalone-", ".py") + script.toFile.deleteOnExit() + Files.write(script, driver.getBytes(StandardCharsets.UTF_8)) + val process = new ProcessBuilder(python, script.toString).redirectErrorStream(true).start() + val out = Source.fromInputStream(process.getInputStream).mkString + process.waitFor(120, TimeUnit.SECONDS) + + withClue(s"python said:\n$out\nscript:\n$driver") { + process.exitValue() shouldBe 0 + val lines = out.trim.linesIterator.toSeq + // out1df is the False port, out2df the True port. Each route has to be + // exclusive: the rows leave by one and the other comes back empty. + lines should contain("None [] [1, 2, 3]") + lines should contain("True [] [1, 2, 3]") + lines should contain("False [1, 2, 3] []") + } + } + + private def resolvePythonExecutable(): Option[String] = { + def fromConfig: Option[String] = + Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption + .orElse(Try(ConfigFactory.load()).toOption) + .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) + .map(_.trim) + .filter(_.nonEmpty) + + def isRunnable(exe: String): Boolean = + Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption + .exists { p => + if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false } + else p.exitValue() == 0 + } + + (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(isRunnable) + } + + private def canImportPandas(python: String): Boolean = + Try( + new ProcessBuilder(python, "-c", "import pandas").redirectErrorStream(true).start() + ).toOption.exists { p => + if (!p.waitFor(60, TimeUnit.SECONDS)) { p.destroyForcibly(); false } + else p.exitValue() == 0 + } } From 64600d70c07b9120d166c26fefb9e96409926624 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Fri, 18 Sep 2026 12:26:19 -0700 Subject: [PATCH 03/32] docs(operator): say where a sorted string column parts from the engine The engine compares with String.compareTo, which reads UTF-16 code units, while pandas compares by code point. They agree across the basic plane and differ only where a character above U+FFFF meets one in U+E000..U+FFFF, because the surrogate pair encoding the first holds units below that range. U+1F600 against U+FFFD is the shape of it: pandas reads the first as greater, the engine as smaller. Co-Authored-By: Claude Opus 5 (1M context) --- .../texera/amber/operator/sort/StableMergeSortOpDesc.scala | 7 +++++++ .../operator/sortPartitions/SortPartitionsOpDesc.scala | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpDesc.scala index aedf6a0f6cb..94e28775055 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpDesc.scala @@ -81,6 +81,13 @@ class StableMergeSortOpDesc extends LogicalOp with StandaloneCodeGenerator { // which orders NaN > +Inf (so NaN sorts last in ASC, first in DESC). pandas // treats NaN as a missing value and always places it at na_position, so DESC // ordering of a Double column with NaNs will differ. + // + // A string column parts on one narrower case. The engine compares with + // String.compareTo, which reads UTF-16 code units, while pandas compares + // Python strings by code point. The two agree across the basic plane and + // differ only where a character above U+FFFF meets one in U+E000..U+FFFF, + // since the surrogate pair that encodes the first holds units below that + // range. override def generateStandaloneCode(): String = { val criteria = Option(keys).getOrElse(ListBuffer.empty) if (criteria.isEmpty) return "out1df = in1df.copy()" diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sortPartitions/SortPartitionsOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sortPartitions/SortPartitionsOpDesc.scala index 7edbb9a782f..9408b823d8d 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sortPartitions/SortPartitionsOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sortPartitions/SortPartitionsOpDesc.scala @@ -95,6 +95,10 @@ class SortPartitionsOpDesc extends LogicalOp with StandaloneCodeGenerator { // // NaN is where the two part. The engine orders it after positive infinity; // pandas treats it as missing and puts it where the nulls go, which is first. + // + // A string column parts once more, and more narrowly: the engine reads UTF-16 + // code units and pandas reads code points, which agree across the basic plane + // and differ only where a character above U+FFFF meets one in U+E000..U+FFFF. override def generateStandaloneCode(): String = { val col = pyStringLiteral(Option(sortAttributeName).getOrElse("")) s"""out1df = in1df.sort_values(by=$col, ascending=True, kind="mergesort", na_position="first").reset_index(drop=True)""" From 76b469005002f18bbd88e0d3d13572dccd8bf606 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Fri, 18 Sep 2026 12:26:19 -0700 Subject: [PATCH 04/32] fix(operator): reject the draws Java rejects, and a reservoir of zero The transcribed nextInt carried its rejection sum as a Python integer, where Java lets an int overflow: a draw within bound of 2**31 wraps negative there and is taken again. Carrying it keeps the value Java throws away and shifts every later draw, which seed 12345 at bound 1000 reaches on draw 2126624. It also never refused a bound of zero, which Reservoir Sampling hands it when the reservoir holds nothing. The engine ends the run there; the script answered with an empty table instead. Co-Authored-By: Claude Opus 5 (1M context) --- .../amber/operator/SamplingHelpers.scala | 10 +- .../amber/operator/SamplingHelpersSpec.scala | 122 ++++++++++++++++++ .../ReservoirSamplingOpDescSpec.scala | 64 ++++++++- 3 files changed, 194 insertions(+), 2 deletions(-) create mode 100644 common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/SamplingHelpersSpec.scala diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/SamplingHelpers.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/SamplingHelpers.scala index 21eeeb0b231..5866fba82b2 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/SamplingHelpers.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/SamplingHelpers.scala @@ -53,11 +53,19 @@ object SamplingHelpers { | return ((self._next(26) << 27) + self._next(27)) * (2.0 ** -53) | | def next_int(self, bound): + | if bound <= 0: + | raise ValueError("bound must be positive") | if bound & (-bound) == bound: | return (bound * self._next(31)) >> 31 | while True: | bits = self._next(31) | value = bits % bound - | if bits - value + (bound - 1) >= 0: + | # Java rejects a draw by letting this sum overflow: it is an int + | # there, so a draw within bound of 2**31 wraps negative and is + | # taken again. Python would carry the sum instead and never + | # reject, which parts the two sequences on the draw that should + | # have been thrown away. + | probe = bits - value + (bound - 1) + | if ((probe + (1 << 31)) % (1 << 32)) - (1 << 31) >= 0: | return value""".stripMargin } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/SamplingHelpersSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/SamplingHelpersSpec.scala new file mode 100644 index 00000000000..f2a56fce0cb --- /dev/null +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/SamplingHelpersSpec.scala @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.operator + +import com.typesafe.config.ConfigFactory +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.util.concurrent.TimeUnit +import scala.io.Source +import scala.util.Try + +/** The transcribed java.util.Random, checked against what Java would answer. + * + * Only the generator is exercised here. What each sampling operator does with + * the numbers is its own spec's business. + */ +class SamplingHelpersSpec extends AnyFlatSpec with Matchers { + + /** Run `body` after the helper class, and return what it printed. */ + private def runWithHelper(body: String): (Int, String) = { + val python = resolvePython().getOrElse(cancel("No runnable python executable")) + val script = Files.createTempFile("sampling-helpers-", ".py") + script.toFile.deleteOnExit() + Files.write( + script, + (SamplingHelpers.JavaRandom + "\n\n" + body).getBytes(StandardCharsets.UTF_8) + ) + val process = new ProcessBuilder(python, script.toString).redirectErrorStream(true).start() + val out = Source.fromInputStream(process.getInputStream).mkString + process.waitFor(60, TimeUnit.SECONDS) + (process.exitValue(), out) + } + + // A bound of 2**30 + 1 puts about half of the draws in the rejection zone, so + // the very first one with this seed is a draw Java throws away. Java decides + // that by letting `u - r + m` overflow an int; carrying the sum instead keeps + // the discarded value and shifts every later draw by one, which is what these + // five numbers pin. They are what java.util.Random answers for this seed. + "SamplingHelpers.JavaRandom" should "reject the draws Java's overflow check rejects" in { + val (exit, out) = runWithHelper( + """r = _TexeraJavaRandom(1) + |print([r.next_int((1 << 30) + 1) for _ in range(5)]) + |""".stripMargin + ) + withClue(s"python said:\n$out") { + exit shouldBe 0 + out.trim shouldBe "[215764588, 880641847, 874970313, 446064254, 77814904]" + } + } + + // java.util.Random#nextInt(int) opens with this check, and Reservoir Sampling + // reaches it with a reservoir of zero: the engine ends the run there, so a + // script that answered with an empty table instead would be reporting a result + // the run never had. + it should "refuse a bound that is not positive, the way Java does" in { + val (exit, out) = runWithHelper( + """r = _TexeraJavaRandom(1) + |try: + | r.next_int(0) + | print("no error") + |except ValueError as e: + | print("ValueError:", e) + |""".stripMargin + ) + withClue(s"python said:\n$out") { + exit shouldBe 0 + out.trim shouldBe "ValueError: bound must be positive" + } + } + + // The power-of-two bound takes the other branch, which has no rejection at + // all, so it would keep working even if the check above were dropped. + it should "take the power-of-two shortcut without rejecting" in { + val (exit, out) = runWithHelper( + """r = _TexeraJavaRandom(42) + |print(all(0 <= r.next_int(256) < 256 for _ in range(1000))) + |""".stripMargin + ) + withClue(s"python said:\n$out") { + exit shouldBe 0 + out.trim shouldBe "True" + } + } + + private def resolvePython(): Option[String] = { + def fromConfig: Option[String] = + Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption + .orElse(Try(ConfigFactory.load()).toOption) + .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) + .map(_.trim) + .filter(_.nonEmpty) + + def runnable(exe: String): Boolean = + Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption + .exists { p => + if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false } + else p.exitValue() == 0 + } + + (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(runnable) + } +} diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/reservoirsampling/ReservoirSamplingOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/reservoirsampling/ReservoirSamplingOpDescSpec.scala index faf0b60aa97..52b43300ff1 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/reservoirsampling/ReservoirSamplingOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/reservoirsampling/ReservoirSamplingOpDescSpec.scala @@ -19,14 +19,21 @@ package org.apache.texera.amber.operator.reservoirsampling +import com.typesafe.config.ConfigFactory import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} -import org.apache.texera.amber.operator.LogicalOp +import org.apache.texera.amber.operator.{LogicalOp, SamplingHelpers} import org.apache.texera.amber.operator.metadata.OperatorGroupConstants import org.apache.texera.amber.util.JSONUtils.objectMapper import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.util.concurrent.TimeUnit +import scala.io.Source +import scala.util.Try + class ReservoirSamplingOpDescSpec extends AnyFlatSpec with Matchers { private val workflowId = WorkflowIdentity(1L) @@ -80,6 +87,35 @@ class ReservoirSamplingOpDescSpec extends AnyFlatSpec with Matchers { |out1df = pd.DataFrame(_texera_rs_reservoir, columns=list(in1df.columns)).reset_index(drop=True)""".stripMargin } + // A reservoir of zero ends the engine's run on the first row: the executor + // skips the fill branch and hands nextInt a bound of zero, which Java refuses. + // The script has to refuse it there too. Answering with an empty table would + // report a result the run never produced. + it should "fail on the first row when the reservoir holds nothing" in { + val python = resolvePython().getOrElse(cancel("No runnable python executable")) + if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + + val d = new ReservoirSamplingOpDesc + d.k = 0 + val script = Files.createTempFile("reservoir-zero-", ".py") + script.toFile.deleteOnExit() + val driver = + s"""import pandas as pd + |${SamplingHelpers.JavaRandom} + |in1df = pd.DataFrame({"id": [1, 2, 3]}) + |${d.generateStandaloneCode()} + |""".stripMargin + Files.write(script, driver.getBytes(StandardCharsets.UTF_8)) + + val process = new ProcessBuilder(python, script.toString).redirectErrorStream(true).start() + val out = Source.fromInputStream(process.getInputStream).mkString + process.waitFor(60, TimeUnit.SECONDS) + withClue(s"python said:\n$out\nscript:\n$driver") { + process.exitValue() should not be 0 + out should include("bound must be positive") + } + } + "ReservoirSamplingOpDesc.getPhysicalOp" should "wire the ReservoirSamplingOpExec class name and carry ports" in { val d = new ReservoirSamplingOpDesc @@ -94,4 +130,30 @@ class ReservoirSamplingOpDescSpec extends AnyFlatSpec with Matchers { physical.inputPorts.keySet shouldBe d.operatorInfo.inputPorts.map(_.id).toSet physical.outputPorts.keySet shouldBe d.operatorInfo.outputPorts.map(_.id).toSet } + + private def resolvePython(): Option[String] = { + def fromConfig: Option[String] = + Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption + .orElse(Try(ConfigFactory.load()).toOption) + .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) + .map(_.trim) + .filter(_.nonEmpty) + + def runnable(exe: String): Boolean = + Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption + .exists { p => + if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false } + else p.exitValue() == 0 + } + + (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(runnable) + } + + private def canImportPandas(python: String): Boolean = + Try( + new ProcessBuilder(python, "-c", "import pandas").redirectErrorStream(true).start() + ).toOption.exists { p => + if (!p.waitFor(60, TimeUnit.SECONDS)) { p.destroyForcibly(); false } + else p.exitValue() == 0 + } } From 85227864a49a328c92911cbe3b8cc0c5156b3291 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Fri, 18 Sep 2026 12:30:58 -0700 Subject: [PATCH 05/32] fix(operator): answer SUM, AVERAGE and CONCAT the way the engine answers them Four places where pandas is asked the same question and answers differently. SUM keeps the column's declared type, so an INTEGER sum is Java int arithmetic and wraps at 2**31 where pandas widens, and a TIMESTAMP sum is a timestamp built from the added epoch milliseconds where pandas refuses to reduce datetime64 at all. AVERAGE is declared DOUBLE whatever it reads, so a timestamp average is the mean of those milliseconds and not a timestamp. CONCAT folds with Java's toString, which spells a boolean in lower case. The first three need the declared type rather than the dtype pandas inferred: a holed INTEGER column arrives as a float, and an INTEGER and a LONG arrive alike. The schema-free form keeps pandas' own answers rather than guessing. Co-Authored-By: Claude Opus 5 (1M context) --- .../operator/aggregate/AggregateOpDesc.scala | 119 ++++++++++++++---- .../aggregate/AggregateOpDescSpec.scala | 96 ++++++++++++++ 2 files changed, 194 insertions(+), 21 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDesc.scala index 16408b0870c..d1f7134b372 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDesc.scala @@ -22,7 +22,7 @@ package org.apache.texera.amber.operator.aggregate import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle import org.apache.texera.amber.core.executor.OpExecWithClassName -import org.apache.texera.amber.core.tuple.Schema +import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.core.virtualidentity.{ ExecutionIdentity, PhysicalOpIdentity, @@ -36,6 +36,7 @@ import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral import org.apache.texera.amber.util.JSONUtils.objectMapper import javax.validation.constraints.{NotNull, Size} +import scala.util.Try class AggregateOpDesc extends LogicalOp with StandaloneCodeGenerator { @@ -141,12 +142,26 @@ class AggregateOpDesc extends LogicalOp with StandaloneCodeGenerator { outputPorts = List(OutputPort()) ) - // The engine aggregates in two phases across partitions; one process needs - // only the one groupby, or a single-row reduction when no key is grouped on. - // - // Must run before `getPhysicalPlan`, which rewrites `aggregations` in place: - // it turns COUNT into SUM for the final phase, and this reads them as written. - override def generateStandaloneCode(): String = { + /** The engine aggregates in two phases across partitions; one process needs + * only the one groupby, or a single-row reduction when no key is grouped on. + * + * Must run before `getPhysicalPlan`, which rewrites `aggregations` in place: + * it turns COUNT into SUM for the final phase, and this reads them as + * written. + * + * What SUM wraps and what AVERAGE divides both follow the column's DECLARED + * type, not the dtype pandas inferred: a holed INTEGER column arrives as a + * float, and an INTEGER and a LONG arrive alike. Without a schema the two + * fall back to what pandas does on its own. + */ + override def generateStandaloneCode(inputSchemas: Map[PortIdentity, Schema]): String = { + val schema = inputSchemas.get(operatorInfo.inputPorts.head.id) + build(name => schema.flatMap(s => Try(s.getAttribute(name).getType).toOption)) + } + + override def generateStandaloneCode(): String = build(_ => None) + + private def build(declaredType: String => Option[AttributeType]): String = { val keys = Option(groupByKeys).getOrElse(List()) val aggs = Option(aggregations).getOrElse(List()) @@ -161,13 +176,45 @@ class AggregateOpDesc extends LogicalOp with StandaloneCodeGenerator { | # here. This is concatAgg's fold, written out. | partial = "" | for v in series: - | text = "" if pd.isna(v) else str(v) + | if pd.isna(v): + | text = "" + | elif isinstance(v, bool) or (hasattr(v, "dtype") and v.dtype == bool): + | # The engine folds with Java's toString, which spells a + | # boolean in lower case. Python's spells it capitalised. + | text = "true" if v else "false" + | else: + | text = str(v) | partial = text if partial == "" else partial + "," + text - | return partial""".stripMargin + | return partial + | + |def _texera_agg_int_sum(series): + | # The engine adds an INTEGER column as Java ints, which wrap at + | # 2**31. pandas widens to a larger integer instead, so 2147483647 + | # and 1 answer 2147483648 where the run reported -2147483648. + | total = int(series.sum()) + | return ((total + (1 << 31)) % (1 << 32)) - (1 << 31) + | + |def _texera_agg_ts_sum(series): + | # SUM keeps the column's own type, so a sum of timestamps is a + | # timestamp: the engine adds the epoch milliseconds and builds one + | # from the total. pandas refuses to reduce datetime64 at all. + | kept = series.dropna() + | return pd.Timestamp(int(kept.astype("int64").sum() // 10**6), unit="ms") + | + |def _texera_agg_ts_mean(series): + | # AVERAGE is declared DOUBLE whatever column it reads, so the + | # average of a timestamp column is the mean of its epoch + | # milliseconds. pandas would answer with a timestamp. + | kept = series.dropna() + | if len(kept) == 0: + | return None + | return float(kept.astype("int64").mean() / 10**6)""".stripMargin if (keys.isEmpty) { val rowEntries = aggs - .map(agg => s" ${pyStringLiteral(agg.resultAttribute)}: ${aggExprScalar(agg)},") + .map(agg => + s" ${pyStringLiteral(agg.resultAttribute)}: ${aggExprScalar(agg, declaredType)}," + ) .mkString("\n") s"""$concatHelper |out1df = pd.DataFrame([{ @@ -178,7 +225,7 @@ class AggregateOpDesc extends LogicalOp with StandaloneCodeGenerator { val aggLines = aggs.zipWithIndex .map { case (agg, i) => - s"_texera_agg_s$i = ${aggExprGroupby(agg, "_texera_agg_groups")}" + s"_texera_agg_s$i = ${aggExprGroupby(agg, "_texera_agg_groups", declaredType)}" } .mkString("\n") val mergeLines = aggs.indices @@ -194,15 +241,28 @@ class AggregateOpDesc extends LogicalOp with StandaloneCodeGenerator { } } - private def aggExprScalar(agg: AggregationOperation): String = { + private def aggExprScalar( + agg: AggregationOperation, + declaredType: String => Option[AttributeType] + ): String = { val attrLit = if (agg.attribute == null || agg.attribute.isEmpty) "None" else pyStringLiteral(agg.attribute) + val declared = Option(agg.attribute).filter(_.nonEmpty).flatMap(declaredType) agg.aggFunction match { - case AggregationFunction.SUM => s"in1df[$attrLit].sum()" - case AggregationFunction.AVERAGE => s"in1df[$attrLit].mean()" - case AggregationFunction.MIN => s"in1df[$attrLit].min()" - case AggregationFunction.MAX => s"in1df[$attrLit].max()" + case AggregationFunction.SUM => + declared match { + case Some(AttributeType.INTEGER) => s"_texera_agg_int_sum(in1df[$attrLit])" + case Some(AttributeType.TIMESTAMP) => s"_texera_agg_ts_sum(in1df[$attrLit])" + case _ => s"in1df[$attrLit].sum()" + } + case AggregationFunction.AVERAGE => + declared match { + case Some(AttributeType.TIMESTAMP) => s"_texera_agg_ts_mean(in1df[$attrLit])" + case _ => s"in1df[$attrLit].mean()" + } + case AggregationFunction.MIN => s"in1df[$attrLit].min()" + case AggregationFunction.MAX => s"in1df[$attrLit].max()" case AggregationFunction.COUNT => if (agg.attribute == null || agg.attribute.isEmpty) "int(len(in1df))" else s"int(in1df[$attrLit].count())" @@ -210,16 +270,33 @@ class AggregateOpDesc extends LogicalOp with StandaloneCodeGenerator { } } - private def aggExprGroupby(agg: AggregationOperation, groups: String): String = { + private def aggExprGroupby( + agg: AggregationOperation, + groups: String, + declaredType: String => Option[AttributeType] + ): String = { val attrLit = if (agg.attribute == null || agg.attribute.isEmpty) "None" else pyStringLiteral(agg.attribute) val resultLit = pyStringLiteral(agg.resultAttribute) + val declared = Option(agg.attribute).filter(_.nonEmpty).flatMap(declaredType) agg.aggFunction match { - case AggregationFunction.SUM => s"$groups[$attrLit].sum().rename($resultLit)" - case AggregationFunction.AVERAGE => s"$groups[$attrLit].mean().rename($resultLit)" - case AggregationFunction.MIN => s"$groups[$attrLit].min().rename($resultLit)" - case AggregationFunction.MAX => s"$groups[$attrLit].max().rename($resultLit)" + case AggregationFunction.SUM => + declared match { + case Some(AttributeType.INTEGER) => + s"$groups[$attrLit].apply(_texera_agg_int_sum).rename($resultLit)" + case Some(AttributeType.TIMESTAMP) => + s"$groups[$attrLit].apply(_texera_agg_ts_sum).rename($resultLit)" + case _ => s"$groups[$attrLit].sum().rename($resultLit)" + } + case AggregationFunction.AVERAGE => + declared match { + case Some(AttributeType.TIMESTAMP) => + s"$groups[$attrLit].apply(_texera_agg_ts_mean).rename($resultLit)" + case _ => s"$groups[$attrLit].mean().rename($resultLit)" + } + case AggregationFunction.MIN => s"$groups[$attrLit].min().rename($resultLit)" + case AggregationFunction.MAX => s"$groups[$attrLit].max().rename($resultLit)" case AggregationFunction.COUNT => if (agg.attribute == null || agg.attribute.isEmpty) s"$groups.size().rename($resultLit)" diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDescSpec.scala index 0f0e372ad66..67bcf39eae1 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDescSpec.scala @@ -23,10 +23,16 @@ import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorMetadataGenerator} +import com.typesafe.config.ConfigFactory import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.util.concurrent.TimeUnit +import scala.io.Source import scala.jdk.CollectionConverters._ +import scala.util.Try class AggregateOpDescSpec extends AnyFlatSpec with Matchers { @@ -108,6 +114,70 @@ class AggregateOpDescSpec extends AnyFlatSpec with Matchers { } } + // Four places where pandas answers a question the engine answers differently, + // run rather than asserted as text: the arithmetic is the point, not the + // spelling of the call. + it should "answer SUM, AVERAGE and CONCAT the way the engine answers them" in { + val python = resolvePython().getOrElse(cancel("No runnable python executable")) + if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + + val input = Schema() + .add("i", AttributeType.INTEGER) + .add("t", AttributeType.TIMESTAMP) + .add("b", AttributeType.BOOLEAN) + val desc = descWith( + List.empty, + aggOp(AggregationFunction.SUM, "i", "int_total"), + aggOp(AggregationFunction.SUM, "t", "ts_total"), + aggOp(AggregationFunction.AVERAGE, "t", "ts_avg"), + aggOp(AggregationFunction.CONCAT, "b", "flags") + ) + val block = desc.generateStandaloneCode(Map(PortIdentity() -> input)) + + val driver = + s"""import pandas as pd + |in1df = pd.DataFrame({ + | "i": pd.Series([2147483647, 1], dtype="int64"), + | "t": pd.to_datetime(["2020-01-01", "2020-01-02"]), + | "b": pd.Series([True, False], dtype=bool), + |}) + |$block + |row = out1df.iloc[0] + |# The engine adds INTEGER as a Java int, which wraps; sums timestamps + |# into a timestamp; declares AVERAGE a DOUBLE whatever it read; and + |# folds CONCAT with Java's toString, which lowercases a boolean. + |print(int(row["int_total"])) + |print(pd.Timestamp(row["ts_total"]).isoformat()) + |print(repr(float(row["ts_avg"]))) + |print(row["flags"]) + |""".stripMargin + + val script = Files.createTempFile("aggregate-engine-answers-", ".py") + script.toFile.deleteOnExit() + Files.write(script, driver.getBytes(StandardCharsets.UTF_8)) + val process = new ProcessBuilder(python, script.toString).redirectErrorStream(true).start() + val out = Source.fromInputStream(process.getInputStream).mkString + process.waitFor(120, TimeUnit.SECONDS) + + withClue(s"python said:\n$out\nscript:\n$driver") { + process.exitValue() shouldBe 0 + val lines = out.trim.linesIterator.toSeq + lines.head shouldBe "-2147483648" + // 1577836800000 + 1577923200000 milliseconds, read back as a timestamp. + lines(1) shouldBe "2070-01-01T00:00:00" + lines(2) shouldBe "1577880000000.0" + lines(3) shouldBe "true,false" + } + } + + // Without a schema the two type-led branches cannot be chosen, so the block + // stays what pandas does on its own rather than guessing. + it should "fall back to pandas' own answers when no schema is given" in { + val desc = descWith(List.empty, aggOp(AggregationFunction.SUM, "i", "int_total")) + desc.generateStandaloneCode() should include("in1df[\"i\"].sum()") + desc.generateStandaloneCode() should not include "_texera_agg_int_sum(in1df" + } + "AggregateOpDesc JSON schema" should "make the attribute optional only for count and required for every other function" in { val aggDef = OperatorMetadataGenerator @@ -131,4 +201,30 @@ class AggregateOpDescSpec extends AnyFlatSpec with Matchers { val elseRequired = rule.get("else").get("required").elements().asScala.map(_.asText()).toList elseRequired should contain("attribute") } + + private def resolvePython(): Option[String] = { + def fromConfig: Option[String] = + Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption + .orElse(Try(ConfigFactory.load()).toOption) + .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) + .map(_.trim) + .filter(_.nonEmpty) + + def runnable(exe: String): Boolean = + Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption + .exists { p => + if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false } + else p.exitValue() == 0 + } + + (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(runnable) + } + + private def canImportPandas(python: String): Boolean = + Try( + new ProcessBuilder(python, "-c", "import pandas").redirectErrorStream(true).start() + ).toOption.exists { p => + if (!p.waitFor(60, TimeUnit.SECONDS)) { p.destroyForcibly(); false } + else p.exitValue() == 0 + } } From f4ba91d1abb831ddc69c36131f949a2634b30ce1 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Fri, 18 Sep 2026 12:34:18 -0700 Subject: [PATCH 06/32] fix(operator): rename a twice-colliding join column, and keep an integer integral The engine appends "#@1" to a colliding right column until the name is free, so one that meets an already-suffixed column lands on "#@1#@1". The export asked pandas for `suffixes` instead, which appends once and then refuses the duplicate it just made, ending the run. Compute the rename the way HashJoinProbeOpExec computes it, the way the Cartesian product export already did, and hand the merge columns that no longer collide. An outer join also leaves holes, and pandas pays for a hole in an integer column by reading the whole column as float. The engine writes a null and the column stays INTEGER, so the declared integer columns are read back as the nullable integer that says the same thing. Only the declared type can say which those are, and without a schema the widening stands. Co-Authored-By: Claude Opus 5 (1M context) --- .../operator/hashJoin/HashJoinOpDesc.scala | 70 ++++++++++++-- .../hashJoin/HashJoinOpDescSpec.scala | 95 ++++++++++++++++++- 2 files changed, 151 insertions(+), 14 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDesc.scala index 469defde967..6bca94e2475 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDesc.scala @@ -22,7 +22,7 @@ package org.apache.texera.amber.operator.hashJoin import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle} import org.apache.texera.amber.core.executor.OpExecWithClassName -import org.apache.texera.amber.core.tuple.{Attribute, Schema} +import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema} import org.apache.texera.amber.core.virtualidentity.{ ExecutionIdentity, PhysicalOpIdentity, @@ -205,6 +205,40 @@ class HashJoinOpDesc[K] extends LogicalOp with StandaloneCodeGenerator { // build key), suffix colliding right columns "#@1" — matches JoinUtils. Known // Texera divergences: row order, null keys (NaN != NaN in merge), outer // anti-row column placement. + /** An outer join leaves holes, and pandas pays for a hole in an integer column + * by widening it to float: the engine keeps the column INTEGER and writes a + * null. Only the declared type can say which columns to put back, so without + * a schema the widening stands. + */ + override def generateStandaloneCode(inputSchemas: Map[PortIdentity, Schema]): String = { + val block = generateStandaloneCode() + val integerColumns = inputSchemas.values + .flatMap(_.getAttributes) + .filter(a => a.getType == AttributeType.INTEGER || a.getType == AttributeType.LONG) + .map(_.getName) + .toSeq + .distinct + if (integerColumns.isEmpty || joinType == JoinType.INNER) block + else { + val namesLit = integerColumns.map(pyStringLiteral).mkString("[", ", ", "]") + s"""$block + |# A row the outer join did not match leaves a hole, and pandas answers a + |# hole in an integer column by reading the whole column as float. The + |# engine leaves it INTEGER and writes a null, so read the declared ones + |# back as the nullable integer that says the same thing. + |_texera_int_names = $namesLit + |for _texera_int_col in out1df.columns: + | # A renamed right column wears one "#@1" per collision it lost, so + | # take them off one at a time. A column that was declared under the + | # name as it stands is matched before any of them come off. + | _texera_base = _texera_int_col + | while _texera_base not in _texera_int_names and _texera_base.endswith("#@1"): + | _texera_base = _texera_base[:-3] + | if _texera_base in _texera_int_names: + | out1df[_texera_int_col] = out1df[_texera_int_col].astype("Int64")""".stripMargin + } + } + override def generateStandaloneCode(): String = { val buildKeyLit = objectMapper.writeValueAsString(buildAttributeName) val probeKeyLit = objectMapper.writeValueAsString(probeAttributeName) @@ -214,17 +248,35 @@ class HashJoinOpDesc[K] extends LogicalOp with StandaloneCodeGenerator { case JoinType.RIGHT_OUTER => "right" case JoinType.FULL_OUTER => "outer" } + // The engine renames a colliding right column by appending "#@1" until the + // candidate is free, so a frame that already carries `x#@1` gets `x#@1#@1`. + // pandas' `suffixes` appends once and then refuses the duplicate it just + // made, so the rename is computed here the way HashJoinProbeOpExec computes + // it, and the merge is handed columns that no longer collide. val merge = - s"""out1df = in1df.merge(in2df, how=${pyStringLiteral(how)}, left_on=$buildKeyLit, """ + - s"""right_on=$probeKeyLit, suffixes=("", "#@1"))""" - // When the left frame already carries the probe key's name, the merge - // suffixes the right key and leaves the left column alone, so dropping the - // bare name would take the left payload instead of the key. - val suffixedProbeKeyLit = objectMapper.writeValueAsString(probeAttributeName + "#@1") + s"""_left_cols = set(in1df.columns) + |_right_cols = list(in2df.columns) + |_right_set = set(_right_cols) + |_rename = {} + |for _col in _right_cols: + | _new = _col + | _others = _right_set - {_col} + | while _new in _left_cols or _new in _others: + | _new = _new + "#@1" + | _rename[_col] = _new + |_probe_key = _rename.get($probeKeyLit, $probeKeyLit) + |out1df = in1df.merge( + | in2df.rename(columns=_rename), + | how=${pyStringLiteral(how)}, + | left_on=$buildKeyLit, + | right_on=_probe_key, + |)""".stripMargin + // The probe key is not carried into the output: the build key already holds + // it. `_probe_key` is whatever the rename above settled on, so this drops + // the right column rather than a left one that happened to share its name. val tail = if (buildAttributeName != probeAttributeName) - s"out1df = out1df.drop(columns=[$suffixedProbeKeyLit if $probeKeyLit in in1df.columns " + - s"else $probeKeyLit]).reset_index(drop=True)" + "out1df = out1df.drop(columns=[_probe_key]).reset_index(drop=True)" else "out1df = out1df.reset_index(drop=True)" s"$merge\n$tail" diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDescSpec.scala index fe3fed8899b..8df2c18025d 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDescSpec.scala @@ -87,11 +87,11 @@ class HashJoinOpDescSpec extends AnyFlatSpec with Matchers { val d = new HashJoinOpDesc[String] d.buildAttributeName = "id" d.probeAttributeName = "key" - // "key" is only suffixed when the left frame has a column of that name, and - // the frame is not known until the script runs, so the choice is in Python. - d.generateStandaloneCode() should include( - """drop(columns=["key#@1" if "key" in in1df.columns else "key"])""" - ) + // "key" is renamed only when the left frame has a column of that name, and + // the frame is not known until the script runs, so the name the rename + // settled on is what gets dropped. + d.generateStandaloneCode() should include("""_probe_key = _rename.get("key", "key")""") + d.generateStandaloneCode() should include("drop(columns=[_probe_key])") } it should "keep the shared key when both sides name it the same" in { @@ -153,6 +153,91 @@ class HashJoinOpDescSpec extends AnyFlatSpec with Matchers { } } + // The engine appends "#@1" until the name is free, so a right column that + // meets an already-suffixed one lands on "#@1#@1". pandas' `suffixes` appends + // once and then refuses the duplicate it just made, which ended the run. + it should "rename a right column that collides twice, the way the engine does" in { + val python = resolvePython().getOrElse(cancel("No runnable python executable")) + if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + + val d = new HashJoinOpDesc[Integer] + d.buildAttributeName = "k" + d.probeAttributeName = "k" + + val driver = + s"""import pandas as pd + | + |in1df = pd.DataFrame({"k": [1], "x": [10]}) + |in2df = pd.DataFrame({"k": [1], "x": [20], "x#@1": [30]}) + |${d.generateStandaloneCode()} + |print(",".join(map(str, out1df.columns))) + |""".stripMargin + + val script = Files.createTempFile("hashjoin-chained-rename-", ".py") + script.toFile.deleteOnExit() + Files.write(script, driver.getBytes(StandardCharsets.UTF_8)) + val process = new ProcessBuilder(python, script.toString).redirectErrorStream(true).start() + val out = Source.fromInputStream(process.getInputStream).mkString + process.waitFor(120, TimeUnit.SECONDS) + withClue(s"python said:\n$out\nscript:\n$driver") { + process.exitValue() shouldBe 0 + // Left names first and untouched, then the right ones in their own order: + // "k" and "x" each lost a collision, and the right's own "x#@1" was free. + out.trim shouldBe "k,x,k#@1,x#@1#@1,x#@1" + } + } + + // An outer join leaves holes, and a hole costs a pandas integer column its + // type. The engine writes a null and the column stays INTEGER. + it should "keep a declared integer column integral across an outer join" in { + val python = resolvePython().getOrElse(cancel("No runnable python executable")) + if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + + val left = Schema() + .add(new Attribute("k", AttributeType.INTEGER)) + .add(new Attribute("x", AttributeType.INTEGER)) + val right = Schema() + .add(new Attribute("kk", AttributeType.INTEGER)) + .add(new Attribute("y", AttributeType.INTEGER)) + + val d = new HashJoinOpDesc[Integer] + d.buildAttributeName = "k" + d.probeAttributeName = "kk" + d.joinType = JoinType.FULL_OUTER + val block = + d.generateStandaloneCode(Map(PortIdentity() -> left, PortIdentity(1) -> right)) + + val driver = + s"""import pandas as pd + | + |in1df = pd.DataFrame({"k": [1, 2], "x": [10, 20]}) + |in2df = pd.DataFrame({"kk": [1, 3], "y": [30, 40]}) + |$block + |print(",".join(f"{c}:{out1df[c].dtype}" for c in out1df.columns)) + |""".stripMargin + + val script = Files.createTempFile("hashjoin-outer-int-", ".py") + script.toFile.deleteOnExit() + Files.write(script, driver.getBytes(StandardCharsets.UTF_8)) + val process = new ProcessBuilder(python, script.toString).redirectErrorStream(true).start() + val out = Source.fromInputStream(process.getInputStream).mkString + process.waitFor(120, TimeUnit.SECONDS) + withClue(s"python said:\n$out\nscript:\n$driver") { + process.exitValue() shouldBe 0 + out.trim shouldBe "k:Int64,x:Int64,y:Int64" + } + } + + // Without a schema there is nothing to say which columns were integers, so + // the block stays what pandas does on its own. + it should "leave the widening alone when no schema is given" in { + val d = new HashJoinOpDesc[Integer] + d.buildAttributeName = "k" + d.probeAttributeName = "kk" + d.joinType = JoinType.FULL_OUTER + d.generateStandaloneCode() should not include "astype(\"Int64\")" + } + "HashJoinOpDesc" should "round-trip its config fields through the polymorphic base" in { val d = new HashJoinOpDesc[String] d.buildAttributeName = "lk" From a240f01e7a78e6b28e5a5055fa5760e8cc2de18b Mon Sep 17 00:00:00 2001 From: kary zheng Date: Fri, 18 Sep 2026 12:51:57 -0700 Subject: [PATCH 07/32] docs(operator): cut today's comments back to what the code cannot say The reasoning behind each fix was written twice, once in the commit that made it and again beside the line. Keep the half a reader needs to follow the code and drop the worked examples and the derivations. Co-Authored-By: Claude Opus 5 (1M context) --- .../amber/operator/SamplingHelpers.scala | 7 ++--- .../operator/aggregate/AggregateOpDesc.scala | 23 +++++---------- .../operator/hashJoin/HashJoinOpDesc.scala | 29 +++++++------------ .../operator/sort/StableMergeSortOpDesc.scala | 8 ++--- .../sortPartitions/SortPartitionsOpDesc.scala | 5 ++-- .../amber/operator/SamplingHelpersSpec.scala | 17 ++++------- .../aggregate/AggregateOpDescSpec.scala | 7 ++--- .../hashJoin/HashJoinOpDescSpec.scala | 11 +++---- .../operator/ifStatement/IfOpDescSpec.scala | 6 ++-- 9 files changed, 38 insertions(+), 75 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/SamplingHelpers.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/SamplingHelpers.scala index 5866fba82b2..1fa9210898b 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/SamplingHelpers.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/SamplingHelpers.scala @@ -60,11 +60,8 @@ object SamplingHelpers { | while True: | bits = self._next(31) | value = bits % bound - | # Java rejects a draw by letting this sum overflow: it is an int - | # there, so a draw within bound of 2**31 wraps negative and is - | # taken again. Python would carry the sum instead and never - | # reject, which parts the two sequences on the draw that should - | # have been thrown away. + | # Java rejects a draw by letting this sum overflow an int. + | # Python would carry it and never reject. | probe = bits - value + (bound - 1) | if ((probe + (1 << 31)) % (1 << 32)) - (1 << 31) >= 0: | return value""".stripMargin diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDesc.scala index d1f7134b372..789c0debe0b 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDesc.scala @@ -149,10 +149,8 @@ class AggregateOpDesc extends LogicalOp with StandaloneCodeGenerator { * it turns COUNT into SUM for the final phase, and this reads them as * written. * - * What SUM wraps and what AVERAGE divides both follow the column's DECLARED - * type, not the dtype pandas inferred: a holed INTEGER column arrives as a - * float, and an INTEGER and a LONG arrive alike. Without a schema the two - * fall back to what pandas does on its own. + * SUM and AVERAGE follow the column's DECLARED type: a holed INTEGER column + * arrives as a float, and an INTEGER and a LONG arrive alike. */ override def generateStandaloneCode(inputSchemas: Map[PortIdentity, Schema]): String = { val schema = inputSchemas.get(operatorInfo.inputPorts.head.id) @@ -179,8 +177,7 @@ class AggregateOpDesc extends LogicalOp with StandaloneCodeGenerator { | if pd.isna(v): | text = "" | elif isinstance(v, bool) or (hasattr(v, "dtype") and v.dtype == bool): - | # The engine folds with Java's toString, which spells a - | # boolean in lower case. Python's spells it capitalised. + | # Java's toString spells a boolean in lower case. | text = "true" if v else "false" | else: | text = str(v) @@ -188,23 +185,19 @@ class AggregateOpDesc extends LogicalOp with StandaloneCodeGenerator { | return partial | |def _texera_agg_int_sum(series): - | # The engine adds an INTEGER column as Java ints, which wrap at - | # 2**31. pandas widens to a larger integer instead, so 2147483647 - | # and 1 answer 2147483648 where the run reported -2147483648. + | # The engine adds an INTEGER column as Java ints, which wrap. | total = int(series.sum()) | return ((total + (1 << 31)) % (1 << 32)) - (1 << 31) | |def _texera_agg_ts_sum(series): - | # SUM keeps the column's own type, so a sum of timestamps is a - | # timestamp: the engine adds the epoch milliseconds and builds one - | # from the total. pandas refuses to reduce datetime64 at all. + | # SUM keeps the column's own type, so the engine adds the epoch + | # milliseconds and builds a timestamp from the total. | kept = series.dropna() | return pd.Timestamp(int(kept.astype("int64").sum() // 10**6), unit="ms") | |def _texera_agg_ts_mean(series): - | # AVERAGE is declared DOUBLE whatever column it reads, so the - | # average of a timestamp column is the mean of its epoch - | # milliseconds. pandas would answer with a timestamp. + | # AVERAGE is declared DOUBLE whatever column it reads, so this is + | # the mean of the epoch milliseconds and not a timestamp. | kept = series.dropna() | if len(kept) == 0: | return None diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDesc.scala index 6bca94e2475..b1c97df48b7 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDesc.scala @@ -205,10 +205,8 @@ class HashJoinOpDesc[K] extends LogicalOp with StandaloneCodeGenerator { // build key), suffix colliding right columns "#@1" — matches JoinUtils. Known // Texera divergences: row order, null keys (NaN != NaN in merge), outer // anti-row column placement. - /** An outer join leaves holes, and pandas pays for a hole in an integer column - * by widening it to float: the engine keeps the column INTEGER and writes a - * null. Only the declared type can say which columns to put back, so without - * a schema the widening stands. + /** Only the declared type can say which columns an outer join widened, so + * without a schema the widening stands. */ override def generateStandaloneCode(inputSchemas: Map[PortIdentity, Schema]): String = { val block = generateStandaloneCode() @@ -222,15 +220,11 @@ class HashJoinOpDesc[K] extends LogicalOp with StandaloneCodeGenerator { else { val namesLit = integerColumns.map(pyStringLiteral).mkString("[", ", ", "]") s"""$block - |# A row the outer join did not match leaves a hole, and pandas answers a - |# hole in an integer column by reading the whole column as float. The - |# engine leaves it INTEGER and writes a null, so read the declared ones - |# back as the nullable integer that says the same thing. + |# A hole costs a pandas integer column its type; the engine writes a + |# null and leaves it INTEGER. |_texera_int_names = $namesLit |for _texera_int_col in out1df.columns: - | # A renamed right column wears one "#@1" per collision it lost, so - | # take them off one at a time. A column that was declared under the - | # name as it stands is matched before any of them come off. + | # A renamed right column wears one "#@1" per collision it lost. | _texera_base = _texera_int_col | while _texera_base not in _texera_int_names and _texera_base.endswith("#@1"): | _texera_base = _texera_base[:-3] @@ -248,11 +242,9 @@ class HashJoinOpDesc[K] extends LogicalOp with StandaloneCodeGenerator { case JoinType.RIGHT_OUTER => "right" case JoinType.FULL_OUTER => "outer" } - // The engine renames a colliding right column by appending "#@1" until the - // candidate is free, so a frame that already carries `x#@1` gets `x#@1#@1`. - // pandas' `suffixes` appends once and then refuses the duplicate it just - // made, so the rename is computed here the way HashJoinProbeOpExec computes - // it, and the merge is handed columns that no longer collide. + // HashJoinProbeOpExec's rename, written out: append "#@1" until the name is + // free. pandas' `suffixes` appends once and then refuses the duplicate it + // just made. val merge = s"""_left_cols = set(in1df.columns) |_right_cols = list(in2df.columns) @@ -271,9 +263,8 @@ class HashJoinOpDesc[K] extends LogicalOp with StandaloneCodeGenerator { | left_on=$buildKeyLit, | right_on=_probe_key, |)""".stripMargin - // The probe key is not carried into the output: the build key already holds - // it. `_probe_key` is whatever the rename above settled on, so this drops - // the right column rather than a left one that happened to share its name. + // `_probe_key` is whatever the rename settled on, so this drops the right + // column rather than a left one that happened to share its name. val tail = if (buildAttributeName != probeAttributeName) "out1df = out1df.drop(columns=[_probe_key]).reset_index(drop=True)" diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpDesc.scala index 94e28775055..50c094b8149 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpDesc.scala @@ -82,12 +82,8 @@ class StableMergeSortOpDesc extends LogicalOp with StandaloneCodeGenerator { // treats NaN as a missing value and always places it at na_position, so DESC // ordering of a Double column with NaNs will differ. // - // A string column parts on one narrower case. The engine compares with - // String.compareTo, which reads UTF-16 code units, while pandas compares - // Python strings by code point. The two agree across the basic plane and - // differ only where a character above U+FFFF meets one in U+E000..U+FFFF, - // since the surrogate pair that encodes the first holds units below that - // range. + // A string column parts more narrowly: the engine reads UTF-16 code units and + // pandas reads code points, which agree below U+FFFF and can differ above it. override def generateStandaloneCode(): String = { val criteria = Option(keys).getOrElse(ListBuffer.empty) if (criteria.isEmpty) return "out1df = in1df.copy()" diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sortPartitions/SortPartitionsOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sortPartitions/SortPartitionsOpDesc.scala index 9408b823d8d..b50f269a7cf 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sortPartitions/SortPartitionsOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sortPartitions/SortPartitionsOpDesc.scala @@ -96,9 +96,8 @@ class SortPartitionsOpDesc extends LogicalOp with StandaloneCodeGenerator { // NaN is where the two part. The engine orders it after positive infinity; // pandas treats it as missing and puts it where the nulls go, which is first. // - // A string column parts once more, and more narrowly: the engine reads UTF-16 - // code units and pandas reads code points, which agree across the basic plane - // and differ only where a character above U+FFFF meets one in U+E000..U+FFFF. + // A string column parts more narrowly: the engine reads UTF-16 code units and + // pandas reads code points, which agree below U+FFFF and can differ above it. override def generateStandaloneCode(): String = { val col = pyStringLiteral(Option(sortAttributeName).getOrElse("")) s"""out1df = in1df.sort_values(by=$col, ascending=True, kind="mergesort", na_position="first").reset_index(drop=True)""" diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/SamplingHelpersSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/SamplingHelpersSpec.scala index f2a56fce0cb..8bf4f8f6f22 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/SamplingHelpersSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/SamplingHelpersSpec.scala @@ -51,11 +51,9 @@ class SamplingHelpersSpec extends AnyFlatSpec with Matchers { (process.exitValue(), out) } - // A bound of 2**30 + 1 puts about half of the draws in the rejection zone, so - // the very first one with this seed is a draw Java throws away. Java decides - // that by letting `u - r + m` overflow an int; carrying the sum instead keeps - // the discarded value and shifts every later draw by one, which is what these - // five numbers pin. They are what java.util.Random answers for this seed. + // A bound of 2**30 + 1 puts about half the draws in the rejection zone, so the + // first one with this seed is already a draw Java throws away. The five + // numbers are what java.util.Random answers. "SamplingHelpers.JavaRandom" should "reject the draws Java's overflow check rejects" in { val (exit, out) = runWithHelper( """r = _TexeraJavaRandom(1) @@ -68,10 +66,8 @@ class SamplingHelpersSpec extends AnyFlatSpec with Matchers { } } - // java.util.Random#nextInt(int) opens with this check, and Reservoir Sampling - // reaches it with a reservoir of zero: the engine ends the run there, so a - // script that answered with an empty table instead would be reporting a result - // the run never had. + // Reservoir Sampling reaches this check with a reservoir of zero, and the + // engine ends the run there. it should "refuse a bound that is not positive, the way Java does" in { val (exit, out) = runWithHelper( """r = _TexeraJavaRandom(1) @@ -88,8 +84,7 @@ class SamplingHelpersSpec extends AnyFlatSpec with Matchers { } } - // The power-of-two bound takes the other branch, which has no rejection at - // all, so it would keep working even if the check above were dropped. + // The power-of-two bound takes the branch that never rejects. it should "take the power-of-two shortcut without rejecting" in { val (exit, out) = runWithHelper( """r = _TexeraJavaRandom(42) diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDescSpec.scala index 67bcf39eae1..5d06df9c3d9 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDescSpec.scala @@ -114,9 +114,7 @@ class AggregateOpDescSpec extends AnyFlatSpec with Matchers { } } - // Four places where pandas answers a question the engine answers differently, - // run rather than asserted as text: the arithmetic is the point, not the - // spelling of the call. + // Run rather than asserted as text: the arithmetic is the point. it should "answer SUM, AVERAGE and CONCAT the way the engine answers them" in { val python = resolvePython().getOrElse(cancel("No runnable python executable")) if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") @@ -170,8 +168,7 @@ class AggregateOpDescSpec extends AnyFlatSpec with Matchers { } } - // Without a schema the two type-led branches cannot be chosen, so the block - // stays what pandas does on its own rather than guessing. + // Without a schema the type-led branches cannot be chosen. it should "fall back to pandas' own answers when no schema is given" in { val desc = descWith(List.empty, aggOp(AggregationFunction.SUM, "i", "int_total")) desc.generateStandaloneCode() should include("in1df[\"i\"].sum()") diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDescSpec.scala index 8df2c18025d..0c4f1df2f36 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDescSpec.scala @@ -153,9 +153,8 @@ class HashJoinOpDescSpec extends AnyFlatSpec with Matchers { } } - // The engine appends "#@1" until the name is free, so a right column that - // meets an already-suffixed one lands on "#@1#@1". pandas' `suffixes` appends - // once and then refuses the duplicate it just made, which ended the run. + // A right column meeting an already-suffixed one lands on "#@1#@1". pandas' + // `suffixes` refused the duplicate it made and ended the run. it should "rename a right column that collides twice, the way the engine does" in { val python = resolvePython().getOrElse(cancel("No runnable python executable")) if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") @@ -187,8 +186,7 @@ class HashJoinOpDescSpec extends AnyFlatSpec with Matchers { } } - // An outer join leaves holes, and a hole costs a pandas integer column its - // type. The engine writes a null and the column stays INTEGER. + // A hole costs a pandas integer column its type; the engine keeps INTEGER. it should "keep a declared integer column integral across an outer join" in { val python = resolvePython().getOrElse(cancel("No runnable python executable")) if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") @@ -228,8 +226,7 @@ class HashJoinOpDescSpec extends AnyFlatSpec with Matchers { } } - // Without a schema there is nothing to say which columns were integers, so - // the block stays what pandas does on its own. + // Without a schema nothing says which columns were integers. it should "leave the widening alone when no schema is given" in { val d = new HashJoinOpDesc[Integer] d.buildAttributeName = "k" diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/ifStatement/IfOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/ifStatement/IfOpDescSpec.scala index 2699886ca7b..0709ed450e5 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/ifStatement/IfOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/ifStatement/IfOpDescSpec.scala @@ -91,10 +91,8 @@ class IfOpDescSpec extends AnyFlatSpec with Matchers { op.generateStandaloneCode() should include("\"_texera_if_ready\"") } - // The engine picks the route from a State message, which the verification harness - // has no channel for, so the exported block is where both routes can be run. False - // is the half a fixture cannot reach: with no State the engine keeps its default, - // and that default is True. + // The engine picks the route from a State message, which the verification + // harness has no channel for, so False is only reachable here. it should "send the rows to True by default and to False when the switch is off" in { val python = resolvePythonExecutable().getOrElse(cancel("No runnable python executable")) if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") From 42404db42a8207ac7cdbcf1247688b61c1de03b0 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Fri, 18 Sep 2026 13:01:02 -0700 Subject: [PATCH 08/32] fix(operator): sort a null and a NaN where the engine sorts them The engine puts a null last whichever way a key points, and compares a NaN above every number, so a NaN goes last ascending and first descending. The export asked for na_position="last", which is right for the null and wrong for the NaN on the way down. Sort each key in three tiers instead: whether the value is missing, always ascending; whether it is a NaN, following the key; then the value. A column read into a numpy dtype holds a null and a NaN in the same slot, and there both land in the missing tier, which is where they were before. Telling them apart needs the nullable dtypes the Arrow source now reads into. Co-Authored-By: Claude Opus 5 (1M context) --- .../operator/sort/StableMergeSortOpDesc.scala | 35 ++++++-- .../sort/StableMergeSortOpDescSpec.scala | 90 +++++++++++++++++++ 2 files changed, 117 insertions(+), 8 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpDesc.scala index 50c094b8149..e498b2df8ab 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpDesc.scala @@ -74,13 +74,9 @@ class StableMergeSortOpDesc extends LogicalOp with StandaloneCodeGenerator { List(OutputPort(blocking = true)) ) - // JVM op runs an incremental stable merge sort with NULLS-LAST regardless of - // direction. pandas' sort_values(kind="mergesort") is stable, and - // na_position="last" is direction-independent — matches the JVM null policy - // exactly. Known divergence: floating-point NaN. JVM uses Double.compare, - // which orders NaN > +Inf (so NaN sorts last in ASC, first in DESC). pandas - // treats NaN as a missing value and always places it at na_position, so DESC - // ordering of a Double column with NaNs will differ. + // The engine runs an incremental stable merge sort with nulls last whichever + // way a key points. pandas' mergesort is stable too, so the ordering below is + // the same one. // // A string column parts more narrowly: the engine reads UTF-16 code units and // pandas reads code points, which agree below U+FFFF and can differ above it. @@ -93,6 +89,29 @@ class StableMergeSortOpDesc extends LogicalOp with StandaloneCodeGenerator { val ascending = criteria .map(c => if (c.sortPreference == SortPreference.ASC) "True" else "False") .mkString("[", ", ", "]") - s"""out1df = in1df.sort_values(by=$cols, ascending=$ascending, kind="mergesort", na_position="last").reset_index(drop=True)""" + // Sort each key in three tiers, because the engine treats a null and a NaN + // differently: a null goes last whichever way the key points, while a NaN + // compares above every number, so it goes last ascending and first + // descending. A column read into a numpy dtype has one slot for both, and + // there both land in the null tier, which is where they were before. + s"""_texera_sorted = in1df.copy() + |_texera_by = [] + |_texera_asc = [] + |_texera_helpers = [] + |for _texera_col, _texera_a in zip($cols, $ascending): + | _texera_null = "_texera_null_" + _texera_col + | _texera_nan = "_texera_nan_" + _texera_col + | _texera_sorted[_texera_null] = _texera_sorted[_texera_col].isna() + | _texera_sorted[_texera_nan] = ( + | _texera_sorted[_texera_col] != _texera_sorted[_texera_col] + | ).fillna(False) + | _texera_helpers += [_texera_null, _texera_nan] + | _texera_by += [_texera_null, _texera_nan, _texera_col] + | _texera_asc += [True, _texera_a, _texera_a] + |out1df = ( + | _texera_sorted.sort_values(by=_texera_by, ascending=_texera_asc, kind="mergesort") + | .drop(columns=_texera_helpers) + | .reset_index(drop=True) + |)""".stripMargin } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpDescSpec.scala index 933549a3b39..e2dbcdac192 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpDescSpec.scala @@ -19,6 +19,7 @@ package org.apache.texera.amber.operator.sort +import com.typesafe.config.ConfigFactory import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.operator.LogicalOp @@ -27,6 +28,13 @@ import org.apache.texera.amber.util.JSONUtils.objectMapper import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.util.concurrent.TimeUnit +import scala.collection.mutable.ListBuffer +import scala.io.Source +import scala.util.Try + class StableMergeSortOpDescSpec extends AnyFlatSpec with Matchers { private val workflowId = WorkflowIdentity(1L) @@ -67,4 +75,86 @@ class StableMergeSortOpDescSpec extends AnyFlatSpec with Matchers { desc.keys.head.attributeName shouldBe "age" desc.keys.head.sortPreference shouldBe SortPreference.DESC } + + // A null goes last whichever way the key points; a NaN compares above every + // number, so it goes last ascending and first descending. Only a column read + // into a nullable dtype can hold both, which is where this is run. + "StableMergeSortOpDesc.generateStandaloneCode" should + "put a null last both ways and a NaN where the key points" in { + val python = resolvePython().getOrElse(cancel("No runnable python executable")) + if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + + def blockFor(pref: SortPreference): String = { + val unit = new SortCriteriaUnit + unit.attributeName = "v" + unit.sortPreference = pref + val d = new StableMergeSortOpDesc + d.keys = ListBuffer(unit) + d.generateStandaloneCode().linesIterator.map(" " + _).mkString("\n") + } + + val driver = + s"""import pandas as pd, numpy as np + | + |def frame(): + | # 3.0, a null, a stored NaN, 1.0, +inf + | col = pd.arrays.FloatingArray( + | np.array([3.0, 0.0, np.nan, 1.0, np.inf]), + | np.array([False, True, False, False, False]), + | ) + | return pd.DataFrame({"v": col}) + | + |def show(df): + | return [("NULL" if x is pd.NA else repr(float(x))) for x in df["v"]] + | + |if True: + | in1df = frame() + |${blockFor(SortPreference.ASC)} + | print("asc", show(out1df)) + |if True: + | in1df = frame() + |${blockFor(SortPreference.DESC)} + | print("desc", show(out1df)) + |""".stripMargin + + val script = Files.createTempFile("sort-nan-", ".py") + script.toFile.deleteOnExit() + Files.write(script, driver.getBytes(StandardCharsets.UTF_8)) + val process = new ProcessBuilder(python, script.toString).redirectErrorStream(true).start() + val out = Source.fromInputStream(process.getInputStream).mkString + process.waitFor(120, TimeUnit.SECONDS) + + withClue(s"python said:\n$out\nscript:\n$driver") { + process.exitValue() shouldBe 0 + val lines = out.trim.linesIterator.toSeq + lines.head shouldBe "asc ['1.0', '3.0', 'inf', 'nan', 'NULL']" + lines(1) shouldBe "desc ['nan', 'inf', '3.0', '1.0', 'NULL']" + } + } + + private def resolvePython(): Option[String] = { + def fromConfig: Option[String] = + Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption + .orElse(Try(ConfigFactory.load()).toOption) + .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) + .map(_.trim) + .filter(_.nonEmpty) + + def runnable(exe: String): Boolean = + Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption + .exists { p => + if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false } + else p.exitValue() == 0 + } + + (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(runnable) + } + + private def canImportPandas(python: String): Boolean = + Try( + new ProcessBuilder(python, "-c", "import pandas, numpy").redirectErrorStream(true).start() + ).toOption.exists { p => + if (!p.waitFor(60, TimeUnit.SECONDS)) { p.destroyForcibly(); false } + else p.exitValue() == 0 + } } From 0ac403968f067733b77e2b3b272f0bc4cadf1735 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Fri, 18 Sep 2026 13:10:40 -0700 Subject: [PATCH 09/32] test(operator): keep a null and a NaN apart in the set, join and keyed paths The engine reads a missing value and a stored NaN as two different values, so a difference keeps the NaN row a null row does not cancel, a join matches the missing key and not the NaN one, and a group keyed on each stays two groups. Each is run on the nullable dtype an Arrow file is read into, which is the only place the two are distinct. Co-Authored-By: Claude Opus 5 (1M context) --- .../aggregate/AggregateOpDescSpec.scala | 35 +++++++++- .../difference/DifferenceOpDescSpec.scala | 67 +++++++++++++++++++ .../hashJoin/HashJoinOpDescSpec.scala | 39 ++++++++++- 3 files changed, 139 insertions(+), 2 deletions(-) diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDescSpec.scala index 5d06df9c3d9..262ac1db2ac 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDescSpec.scala @@ -168,6 +168,39 @@ class AggregateOpDescSpec extends AnyFlatSpec with Matchers { } } + // A group keyed on a missing value and one keyed on a NaN are two groups to + // the engine. The two are only distinct in a nullable dtype, which is what an + // Arrow file is read into. + it should "group a missing key apart from a NaN one" in { + val python = resolvePython().getOrElse(cancel("No runnable python executable")) + if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + + val desc = descWith(List("k"), aggOp(AggregationFunction.SUM, "v", "total")) + + val driver = + s"""import pandas as pd, numpy as np + | + |in1df = pd.DataFrame({ + | "k": pd.arrays.FloatingArray(np.array([0.0, np.nan]), np.array([True, False])), + | "v": [1, 2], + |}) + |${desc.generateStandaloneCode()} + |print(len(out1df), sorted(int(x) for x in out1df["total"])) + |""".stripMargin + + val script = Files.createTempFile("aggregate-null-nan-", ".py") + script.toFile.deleteOnExit() + Files.write(script, driver.getBytes(StandardCharsets.UTF_8)) + val process = new ProcessBuilder(python, script.toString).redirectErrorStream(true).start() + val out = Source.fromInputStream(process.getInputStream).mkString + process.waitFor(120, TimeUnit.SECONDS) + withClue(s"python said:\n$out\nscript:\n$driver") { + process.exitValue() shouldBe 0 + // Two groups of one row each, not one group totalling 3. + out.trim shouldBe "2 [1, 2]" + } + } + // Without a schema the type-led branches cannot be chosen. it should "fall back to pandas' own answers when no schema is given" in { val desc = descWith(List.empty, aggOp(AggregationFunction.SUM, "i", "int_total")) @@ -219,7 +252,7 @@ class AggregateOpDescSpec extends AnyFlatSpec with Matchers { private def canImportPandas(python: String): Boolean = Try( - new ProcessBuilder(python, "-c", "import pandas").redirectErrorStream(true).start() + new ProcessBuilder(python, "-c", "import pandas, numpy").redirectErrorStream(true).start() ).toOption.exists { p => if (!p.waitFor(60, TimeUnit.SECONDS)) { p.destroyForcibly(); false } else p.exitValue() == 0 diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/difference/DifferenceOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/difference/DifferenceOpDescSpec.scala index f6d17e94ebc..98824ba6a1e 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/difference/DifferenceOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/difference/DifferenceOpDescSpec.scala @@ -19,6 +19,7 @@ package org.apache.texera.amber.operator.difference +import com.typesafe.config.ConfigFactory import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} @@ -32,6 +33,12 @@ import org.apache.texera.amber.operator.metadata.OperatorGroupConstants import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.util.concurrent.TimeUnit +import scala.io.Source +import scala.util.Try + class DifferenceOpDescSpec extends AnyFlatSpec with Matchers { private val workflowId = WorkflowIdentity(1L) @@ -140,4 +147,64 @@ class DifferenceOpDescSpec extends AnyFlatSpec with Matchers { val b = new DifferenceOpDesc a.operatorIdentifier should not equal b.operatorIdentifier } + + // A missing value and a stored NaN are two different values to the engine, so + // a left holding both against a right holding only the missing one keeps the + // NaN row. The two are only distinct in a nullable dtype, which is what an + // Arrow file is read into. + "DifferenceOpDesc.generateStandaloneCode" should + "keep the NaN row that a null row does not cancel" in { + val python = resolvePython().getOrElse(cancel("No runnable python executable")) + if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + + val driver = + s"""import pandas as pd, numpy as np + | + |def col(vals, mask): + | return pd.arrays.FloatingArray(np.array(vals), np.array(mask)) + | + |in1df = pd.DataFrame({"v": col([0.0, np.nan], [True, False])}) + |in2df = pd.DataFrame({"v": col([0.0], [True])}) + |${(new DifferenceOpDesc).generateStandaloneCode()} + |print(len(out1df), [("NULL" if x is pd.NA else repr(float(x))) for x in out1df["v"]]) + |""".stripMargin + + val script = Files.createTempFile("difference-null-nan-", ".py") + script.toFile.deleteOnExit() + Files.write(script, driver.getBytes(StandardCharsets.UTF_8)) + val process = new ProcessBuilder(python, script.toString).redirectErrorStream(true).start() + val out = Source.fromInputStream(process.getInputStream).mkString + process.waitFor(120, TimeUnit.SECONDS) + + withClue(s"python said:\n$out\nscript:\n$driver") { + process.exitValue() shouldBe 0 + out.trim shouldBe "1 ['nan']" + } + } + + private def resolvePython(): Option[String] = { + def fromConfig: Option[String] = + Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption + .orElse(Try(ConfigFactory.load()).toOption) + .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) + .map(_.trim) + .filter(_.nonEmpty) + + def runnable(exe: String): Boolean = + Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption + .exists { p => + if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false } + else p.exitValue() == 0 + } + + (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(runnable) + } + + private def canImportPandas(python: String): Boolean = + Try( + new ProcessBuilder(python, "-c", "import pandas, numpy").redirectErrorStream(true).start() + ).toOption.exists { p => + if (!p.waitFor(60, TimeUnit.SECONDS)) { p.destroyForcibly(); false } + else p.exitValue() == 0 + } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDescSpec.scala index 0c4f1df2f36..6ad1518e287 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDescSpec.scala @@ -226,6 +226,43 @@ class HashJoinOpDescSpec extends AnyFlatSpec with Matchers { } } + // A key that is missing and a key holding a NaN are two different keys to the + // engine, so only the missing one matches a missing right key. The two are + // only distinct in a nullable dtype, which is what an Arrow file is read into. + it should "match a missing key but not a NaN one" in { + val python = resolvePython().getOrElse(cancel("No runnable python executable")) + if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + + val d = new HashJoinOpDesc[Integer] + d.buildAttributeName = "k" + d.probeAttributeName = "k" + d.joinType = JoinType.LEFT_OUTER + + val driver = + s"""import pandas as pd, numpy as np + | + |def col(vals, mask): + | return pd.arrays.FloatingArray(np.array(vals), np.array(mask)) + | + |in1df = pd.DataFrame({"k": col([0.0, np.nan], [True, False]), "lv": [1, 2]}) + |in2df = pd.DataFrame({"k": col([0.0], [True]), "rv": [9]}) + |${d.generateStandaloneCode()} + |print(len(out1df), int(out1df["rv"].notna().sum())) + |""".stripMargin + + val script = Files.createTempFile("hashjoin-null-nan-", ".py") + script.toFile.deleteOnExit() + Files.write(script, driver.getBytes(StandardCharsets.UTF_8)) + val process = new ProcessBuilder(python, script.toString).redirectErrorStream(true).start() + val out = Source.fromInputStream(process.getInputStream).mkString + process.waitFor(120, TimeUnit.SECONDS) + withClue(s"python said:\n$out\nscript:\n$driver") { + process.exitValue() shouldBe 0 + // Both left rows come back, and exactly one of them found a partner. + out.trim shouldBe "2 1" + } + } + // Without a schema nothing says which columns were integers. it should "leave the widening alone when no schema is given" in { val d = new HashJoinOpDesc[Integer] @@ -272,7 +309,7 @@ class HashJoinOpDescSpec extends AnyFlatSpec with Matchers { private def canImportPandas(python: String): Boolean = Try( - new ProcessBuilder(python, "-c", "import pandas").redirectErrorStream(true).start() + new ProcessBuilder(python, "-c", "import pandas, numpy").redirectErrorStream(true).start() ).toOption .exists { p => if (!p.waitFor(60, TimeUnit.SECONDS)) { p.destroyForcibly(); false } From c18b1e5ef42d2a9d27cfc6340fd9f5c92d11d8aa Mon Sep 17 00:00:00 2001 From: kary zheng Date: Fri, 18 Sep 2026 16:21:52 -0700 Subject: [PATCH 10/32] fix(operator): aggregate a timestamp in milliseconds, and cast only the column that declared an integer The integers behind a timestamp column count nanoseconds or microseconds depending on the resolution it was read at, so reading them and dividing by a million answers a microsecond column with a 1970 date. A nanosecond total also leaves the range of a 64-bit integer after six modern dates. Cast to milliseconds first, sum as Python integers, and wrap the total the way the engine's Java longs wrap. The outer-join integer cast pooled the column names of both inputs, so a double on the right that collided with an integer on the left was cast back to an integer and raised. Name the right columns as the merge will name them, and cast those names alone. Co-Authored-By: Claude Opus 5 (1M context) --- .../operator/aggregate/AggregateOpDesc.scala | 19 +++++-- .../operator/hashJoin/HashJoinOpDesc.scala | 35 ++++++++----- .../aggregate/AggregateOpDescSpec.scala | 49 +++++++++++++++++++ .../hashJoin/HashJoinOpDescSpec.scala | 43 ++++++++++++++++ 4 files changed, 128 insertions(+), 18 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDesc.scala index 789c0debe0b..7acf4b3233d 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDesc.scala @@ -189,19 +189,28 @@ class AggregateOpDesc extends LogicalOp with StandaloneCodeGenerator { | total = int(series.sum()) | return ((total + (1 << 31)) % (1 << 32)) - (1 << 31) | + |def _texera_agg_ts_epoch_ms(series): + | # A timestamp reaches the engine as its epoch milliseconds whatever + | # resolution the column carries, and reading the integers out of a + | # microsecond column asks for a different number than a nanosecond + | # one, so cast to milliseconds before reading them. + | return series.dropna().astype("datetime64[ms]").astype("int64") + | |def _texera_agg_ts_sum(series): | # SUM keeps the column's own type, so the engine adds the epoch - | # milliseconds and builds a timestamp from the total. - | kept = series.dropna() - | return pd.Timestamp(int(kept.astype("int64").sum() // 10**6), unit="ms") + | # milliseconds as Java longs, which wrap, and builds a timestamp + | # from the total. Python integers carry the sum exactly, so the + | # wrap is the only place a total loses anything. + | total = int(_texera_agg_ts_epoch_ms(series).astype(object).sum()) + | return pd.Timestamp(((total + (1 << 63)) % (1 << 64)) - (1 << 63), unit="ms") | |def _texera_agg_ts_mean(series): | # AVERAGE is declared DOUBLE whatever column it reads, so this is | # the mean of the epoch milliseconds and not a timestamp. - | kept = series.dropna() + | kept = _texera_agg_ts_epoch_ms(series) | if len(kept) == 0: | return None - | return float(kept.astype("int64").mean() / 10**6)""".stripMargin + | return float(kept.astype(object).sum()) / len(kept)""".stripMargin if (keys.isEmpty) { val rowEntries = aggs diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDesc.scala index b1c97df48b7..d98aedc2155 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDesc.scala @@ -210,25 +210,34 @@ class HashJoinOpDesc[K] extends LogicalOp with StandaloneCodeGenerator { */ override def generateStandaloneCode(inputSchemas: Map[PortIdentity, Schema]): String = { val block = generateStandaloneCode() - val integerColumns = inputSchemas.values - .flatMap(_.getAttributes) - .filter(a => a.getType == AttributeType.INTEGER || a.getType == AttributeType.LONG) - .map(_.getName) - .toSeq - .distinct + val integral = (a: Attribute) => + a.getType == AttributeType.INTEGER || a.getType == AttributeType.LONG + val integerColumns = ( + inputSchemas.get(operatorInfo.inputPorts.head.id), + inputSchemas.get(operatorInfo.inputPorts.last.id) + ) match { + case (Some(build), Some(probe)) => + val leftNames = build.getAttributes.map(_.getName).toSet + val rightNames = probe.getAttributes.map(_.getName).toSet + // Name the right column as the merge will name it, so that a double on + // the right is not read as the integer on the left it collided with. + val renamedRight = probe.getAttributes.filter(integral).map { attr => + var name = attr.getName + val others = rightNames - attr.getName + while (leftNames.contains(name) || others.contains(name)) name = name + "#@1" + name + } + (build.getAttributes.filter(integral).map(_.getName) ++ renamedRight).distinct + case _ => List.empty + } if (integerColumns.isEmpty || joinType == JoinType.INNER) block else { val namesLit = integerColumns.map(pyStringLiteral).mkString("[", ", ", "]") s"""$block |# A hole costs a pandas integer column its type; the engine writes a |# null and leaves it INTEGER. - |_texera_int_names = $namesLit - |for _texera_int_col in out1df.columns: - | # A renamed right column wears one "#@1" per collision it lost. - | _texera_base = _texera_int_col - | while _texera_base not in _texera_int_names and _texera_base.endswith("#@1"): - | _texera_base = _texera_base[:-3] - | if _texera_base in _texera_int_names: + |for _texera_int_col in $namesLit: + | if _texera_int_col in out1df.columns: | out1df[_texera_int_col] = out1df[_texera_int_col].astype("Int64")""".stripMargin } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDescSpec.scala index 262ac1db2ac..3740c0edbef 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDescSpec.scala @@ -168,6 +168,55 @@ class AggregateOpDescSpec extends AnyFlatSpec with Matchers { } } + // The integers behind a timestamp column mean microseconds or nanoseconds + // depending on the resolution it was read at, and a nanosecond total leaves + // the range of a 64-bit integer after a handful of modern dates. + it should "read a timestamp as epoch milliseconds at either resolution" in { + val python = resolvePython().getOrElse(cancel("No runnable python executable")) + if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + + val input = Schema().add("t", AttributeType.TIMESTAMP) + val desc = descWith( + List.empty, + aggOp(AggregationFunction.SUM, "t", "ts_total"), + aggOp(AggregationFunction.AVERAGE, "t", "ts_avg") + ) + val block = desc.generateStandaloneCode(Map(PortIdentity() -> input)) + + def report(unit: String): String = + s"""in1df = pd.DataFrame({ + | "t": pd.to_datetime(["2024-01-01"] * 6).astype("datetime64[$unit]"), + |}) + |$block + |print(pd.Timestamp(out1df.iloc[0]["ts_total"]).isoformat()) + |print(repr(float(out1df.iloc[0]["ts_avg"])))""".stripMargin + + val driver = + s"""import pandas as pd + |${report("ns")} + |${report("us")} + |""".stripMargin + + val script = Files.createTempFile("aggregate-timestamp-units-", ".py") + script.toFile.deleteOnExit() + Files.write(script, driver.getBytes(StandardCharsets.UTF_8)) + val process = new ProcessBuilder(python, script.toString).redirectErrorStream(true).start() + val out = Source.fromInputStream(process.getInputStream).mkString + process.waitFor(120, TimeUnit.SECONDS) + + withClue(s"python said:\n$out\nscript:\n$driver") { + process.exitValue() shouldBe 0 + // Six times 1704067200000 milliseconds, read back as a timestamp: a date + // no nanosecond count can hold, and the same one at either resolution. + out.trim.linesIterator.toSeq shouldBe Seq( + "2293-12-31T00:00:00", + "1704067200000.0", + "2293-12-31T00:00:00", + "1704067200000.0" + ) + } + } + // A group keyed on a missing value and one keyed on a NaN are two groups to // the engine. The two are only distinct in a nullable dtype, which is what an // Arrow file is read into. diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDescSpec.scala index 6ad1518e287..bae32675d20 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDescSpec.scala @@ -226,6 +226,49 @@ class HashJoinOpDescSpec extends AnyFlatSpec with Matchers { } } + // The two sides can name a column alike and declare it differently, and only + // the left one is the integer: the right one wears its rename out. + it should "leave a right column alone that shares an integer column's name" in { + val python = resolvePython().getOrElse(cancel("No runnable python executable")) + if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + + val left = Schema() + .add(new Attribute("k", AttributeType.INTEGER)) + .add(new Attribute("x", AttributeType.INTEGER)) + val right = Schema() + .add(new Attribute("kk", AttributeType.INTEGER)) + .add(new Attribute("x", AttributeType.DOUBLE)) + + val d = new HashJoinOpDesc[Integer] + d.buildAttributeName = "k" + d.probeAttributeName = "kk" + d.joinType = JoinType.FULL_OUTER + val block = + d.generateStandaloneCode(Map(PortIdentity() -> left, PortIdentity(1) -> right)) + + val driver = + s"""import pandas as pd + | + |in1df = pd.DataFrame({"k": [1, 2], "x": [10, 20]}) + |in2df = pd.DataFrame({"kk": [1, 3], "x": [1.5, 2.5]}) + |$block + |print(",".join(f"{c}:{out1df[c].dtype}" for c in out1df.columns)) + |""".stripMargin + + val script = Files.createTempFile("hashjoin-outer-int-collision-", ".py") + script.toFile.deleteOnExit() + Files.write(script, driver.getBytes(StandardCharsets.UTF_8)) + val process = new ProcessBuilder(python, script.toString).redirectErrorStream(true).start() + val out = Source.fromInputStream(process.getInputStream).mkString + process.waitFor(120, TimeUnit.SECONDS) + withClue(s"python said:\n$out\nscript:\n$driver") { + process.exitValue() shouldBe 0 + // The renamed right column keeps its 1.5, rather than being cast to the + // integer the left column declared. + out.trim shouldBe "k:Int64,x:Int64,x#@1:float64" + } + } + // A key that is missing and a key holding a NaN are two different keys to the // engine, so only the missing one matches a missing right key. The two are // only distinct in a nullable dtype, which is what an Arrow file is read into. From b8e5dbfb0b5eb9ef0747f044cc2dc974ef68bd29 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Sat, 19 Sep 2026 00:28:57 -0700 Subject: [PATCH 11/32] fix(operator): emit a join key once when both sides name it the same The everyday join, the same column name on both inputs, carried a second key column the operator never promises: `k`, `name`, `k#@1`, `score` where the engine emits `k`, `name`, `score`. Moving the right frame's columns aside before the merge, so a chained name collision resolves, also moves the key aside when the two sides name it alike. The drop was still decided by comparing the operator's own two names, which read that case as one shared column and kept the copy. It now asks the name the rename settled on, which is the only place that knows, and covers the three shapes alike: the same name, a free name, and one that met a left column. Co-Authored-By: Claude Opus 5 (1M context) --- .../operator/hashJoin/HashJoinOpDesc.scala | 13 ++-- .../hashJoin/HashJoinOpDescSpec.scala | 69 +++++++++++++++++-- 2 files changed, 73 insertions(+), 9 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDesc.scala index d98aedc2155..6d03ca3415c 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDesc.scala @@ -273,12 +273,15 @@ class HashJoinOpDesc[K] extends LogicalOp with StandaloneCodeGenerator { | right_on=_probe_key, |)""".stripMargin // `_probe_key` is whatever the rename settled on, so this drops the right - // column rather than a left one that happened to share its name. + // column rather than a left one that happened to share its name. Asked of + // the name the rename produced and not of the two the operator was given: + // when both sides name the key alike the rename still moves the right one + // aside, and a test on the operator's own two names read that as one shared + // column and kept the copy, where the engine emits the key once. val tail = - if (buildAttributeName != probeAttributeName) - "out1df = out1df.drop(columns=[_probe_key]).reset_index(drop=True)" - else - "out1df = out1df.reset_index(drop=True)" + s"""if _probe_key != $buildKeyLit: + | out1df = out1df.drop(columns=[_probe_key]) + |out1df = out1df.reset_index(drop=True)""".stripMargin s"$merge\n$tail" } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDescSpec.scala index bae32675d20..e63fb9dbf6b 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDescSpec.scala @@ -94,11 +94,18 @@ class HashJoinOpDescSpec extends AnyFlatSpec with Matchers { d.generateStandaloneCode() should include("drop(columns=[_probe_key])") } - it should "keep the shared key when both sides name it the same" in { + // Both sides naming the key alike does not make it one column: the rename + // moves the right one aside because it meets the left one, so the drop is + // decided by the name the rename produced rather than by the two the operator + // was given. + it should "emit the shared key once when both sides name it the same" in { val d = new HashJoinOpDesc[String] d.buildAttributeName = "k" d.probeAttributeName = "k" - d.generateStandaloneCode() should not include "drop(columns=" + val code = d.generateStandaloneCode() + code should include("""_probe_key = _rename.get("k", "k")""") + code should include("""if _probe_key != "k":""") + code should include("drop(columns=[_probe_key])") } /** The bug this pins: with a left `key` payload and a right `key` join column, @@ -153,6 +160,58 @@ class HashJoinOpDescSpec extends AnyFlatSpec with Matchers { } } + /** The bug this pins: both sides naming the key alike reads as one shared + * column, but the rename moves the right one aside before the merge, so the + * frame carried a second `k#@1` the operator never promises. This is the + * everyday join, the same name on both sides. + */ + it should "promise and emit the key once when both sides name it the same" in { + val python = resolvePython().getOrElse( + cancel("No runnable python executable (udf.conf python.path, python3, python, py)") + ) + if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + + val left = Schema() + .add(new Attribute("k", AttributeType.INTEGER)) + .add(new Attribute("name", AttributeType.STRING)) + val right = Schema() + .add(new Attribute("k", AttributeType.INTEGER)) + .add(new Attribute("score", AttributeType.INTEGER)) + + val d = new HashJoinOpDesc[Integer] + d.buildAttributeName = "k" + d.probeAttributeName = "k" + val promised = d + .getExternalOutputSchemas(Map(PortIdentity() -> left, PortIdentity(1) -> right))( + d.operatorInfo.outputPorts.head.id + ) + .getAttributeNames + + val driver = + s"""import pandas as pd + | + |in1df = pd.DataFrame({"k": [1], "name": ["alice"]}) + |in2df = pd.DataFrame({"k": [1], "score": [95]}) + |${d.generateStandaloneCode()} + |print(",".join(map(str, out1df.columns))) + |print(",".join(map(str, out1df.iloc[0].tolist()))) + |""".stripMargin + + val script = Files.createTempFile("hashjoin-shared-key-", ".py") + script.toFile.deleteOnExit() + Files.write(script, driver.getBytes(StandardCharsets.UTF_8)) + val process = new ProcessBuilder(python, script.toString).redirectErrorStream(true).start() + val out = Source.fromInputStream(process.getInputStream).mkString + process.waitFor(120, TimeUnit.SECONDS) + withClue(s"python said:\n$out\nscript:\n$driver") { + process.exitValue() shouldBe 0 + val lines = out.trim.linesIterator.toSeq + promised shouldBe List("k", "name", "score") + lines.head.split(",").toList shouldBe promised + lines(1) shouldBe "1,alice,95" + } + } + // A right column meeting an already-suffixed one lands on "#@1#@1". pandas' // `suffixes` refused the duplicate it made and ended the run. it should "rename a right column that collides twice, the way the engine does" in { @@ -181,8 +240,10 @@ class HashJoinOpDescSpec extends AnyFlatSpec with Matchers { withClue(s"python said:\n$out\nscript:\n$driver") { process.exitValue() shouldBe 0 // Left names first and untouched, then the right ones in their own order: - // "k" and "x" each lost a collision, and the right's own "x#@1" was free. - out.trim shouldBe "k,x,k#@1,x#@1#@1,x#@1" + // "x" lost a collision and the right's own "x#@1" was free. The right "k" + // lost one too, and being the join key it is then dropped, the engine + // emitting the key once. + out.trim shouldBe "k,x,x#@1#@1,x#@1" } } From 7ccb215f500fbff2b940f3cb78dfc1909cf8d752 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Sat, 19 Sep 2026 01:07:20 -0700 Subject: [PATCH 12/32] fix(operator): widen an outer join's integer columns before the merge An unmatched row leaves a hole, and a hole costs a pandas integer column its type: int64 becomes float64. float64 cannot hold an integer past 2^53, so the cast that put INTEGER back was reading a value already rounded. A column holding 9007199254740993 came out of a full outer join as 9007199254740992. Each input's declared integer columns are now cast to the nullable integer dtype on that input's own frame, before the merge digs the hole. The cast runs before the rename, so a right column is typed by the right schema rather than by the left column it collided with, and the merged frame is never cast back. Co-Authored-By: Claude Opus 5 (1M context) --- .../operator/hashJoin/HashJoinOpDesc.scala | 79 ++++++++++--------- .../hashJoin/HashJoinOpDescSpec.scala | 62 ++++++++++++++- 2 files changed, 104 insertions(+), 37 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDesc.scala index 6d03ca3415c..38213bf972e 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDesc.scala @@ -209,40 +209,27 @@ class HashJoinOpDesc[K] extends LogicalOp with StandaloneCodeGenerator { * without a schema the widening stands. */ override def generateStandaloneCode(inputSchemas: Map[PortIdentity, Schema]): String = { - val block = generateStandaloneCode() val integral = (a: Attribute) => a.getType == AttributeType.INTEGER || a.getType == AttributeType.LONG - val integerColumns = ( - inputSchemas.get(operatorInfo.inputPorts.head.id), - inputSchemas.get(operatorInfo.inputPorts.last.id) - ) match { - case (Some(build), Some(probe)) => - val leftNames = build.getAttributes.map(_.getName).toSet - val rightNames = probe.getAttributes.map(_.getName).toSet - // Name the right column as the merge will name it, so that a double on - // the right is not read as the integer on the left it collided with. - val renamedRight = probe.getAttributes.filter(integral).map { attr => - var name = attr.getName - val others = rightNames - attr.getName - while (leftNames.contains(name) || others.contains(name)) name = name + "#@1" - name - } - (build.getAttributes.filter(integral).map(_.getName) ++ renamedRight).distinct - case _ => List.empty - } - if (integerColumns.isEmpty || joinType == JoinType.INNER) block - else { - val namesLit = integerColumns.map(pyStringLiteral).mkString("[", ", ", "]") - s"""$block - |# A hole costs a pandas integer column its type; the engine writes a - |# null and leaves it INTEGER. - |for _texera_int_col in $namesLit: - | if _texera_int_col in out1df.columns: - | out1df[_texera_int_col] = out1df[_texera_int_col].astype("Int64")""".stripMargin - } + val declaredIntegers = (port: PortIdentity) => + inputSchemas.get(port).map(_.getAttributes.filter(integral).map(_.getName)).getOrElse(List()) + // Each side keeps its own list, under the names that side's frame carries, + // so a column the merge renames is still the column its own schema typed. + if (joinType == JoinType.INNER) generateStandaloneCode() + else + generateStandaloneCode( + declaredIntegers(operatorInfo.inputPorts.head.id), + declaredIntegers(operatorInfo.inputPorts.last.id) + ) } - override def generateStandaloneCode(): String = { + override def generateStandaloneCode(): String = + generateStandaloneCode(List(), List()) + + private def generateStandaloneCode( + leftIntegerColumns: List[String], + rightIntegerColumns: List[String] + ): String = { val buildKeyLit = objectMapper.writeValueAsString(buildAttributeName) val probeKeyLit = objectMapper.writeValueAsString(probeAttributeName) val how = joinType match { @@ -251,6 +238,25 @@ class HashJoinOpDesc[K] extends LogicalOp with StandaloneCodeGenerator { case JoinType.RIGHT_OUTER => "right" case JoinType.FULL_OUTER => "outer" } + // An unmatched row leaves a hole, and a hole costs a pandas integer column + // its type: int64 becomes float64, which rounds every value past 2^53 + // before anything can put the type back. The engine writes a null and + // leaves the column INTEGER, so widen to the integer dtype that holds a + // hole before the merge digs one. + val widen = leftIntegerColumns.nonEmpty || rightIntegerColumns.nonEmpty + val namesLit = (names: List[String]) => names.map(pyStringLiteral).mkString("[", ", ", "]") + val widening = + if (!widen) "" + else + s"""_left_ints = {_c: "Int64" for _c in ${namesLit(leftIntegerColumns)} if _c in in1df.columns} + |_right_ints = {_c: "Int64" for _c in ${namesLit(rightIntegerColumns)} if _c in in2df.columns} + |""".stripMargin + val leftFrame = if (widen) "in1df.astype(_left_ints)" else "in1df" + // Cast before the rename, because the names above are the ones the right + // input declared. + val rightFrame = + if (widen) "in2df.astype(_right_ints).rename(columns=_rename)" + else "in2df.rename(columns=_rename)" // HashJoinProbeOpExec's rename, written out: append "#@1" until the name is // free. pandas' `suffixes` appends once and then refuses the duplicate it // just made. @@ -266,12 +272,13 @@ class HashJoinOpDesc[K] extends LogicalOp with StandaloneCodeGenerator { | _new = _new + "#@1" | _rename[_col] = _new |_probe_key = _rename.get($probeKeyLit, $probeKeyLit) - |out1df = in1df.merge( - | in2df.rename(columns=_rename), - | how=${pyStringLiteral(how)}, - | left_on=$buildKeyLit, - | right_on=_probe_key, - |)""".stripMargin + |""".stripMargin + widening + + s"""out1df = $leftFrame.merge( + | $rightFrame, + | how=${pyStringLiteral(how)}, + | left_on=$buildKeyLit, + | right_on=_probe_key, + |)""".stripMargin // `_probe_key` is whatever the rename settled on, so this drops the right // column rather than a left one that happened to share its name. Asked of // the name the rename produced and not of the two the operator was given: diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDescSpec.scala index e63fb9dbf6b..78230e3530f 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDescSpec.scala @@ -287,6 +287,50 @@ class HashJoinOpDescSpec extends AnyFlatSpec with Matchers { } } + /** The bug this pins: the merge widened an unmatched integer column to + * float64, and the cast that put the type back read a value already rounded. + * 9007199254740993 is the first integer float64 cannot hold. + */ + it should "keep an integer past 2^53 exact across an outer join" in { + val python = resolvePython().getOrElse(cancel("No runnable python executable")) + if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + + val left = Schema() + .add(new Attribute("k", AttributeType.LONG)) + .add(new Attribute("id", AttributeType.LONG)) + val right = Schema() + .add(new Attribute("kk", AttributeType.LONG)) + .add(new Attribute("y", AttributeType.STRING)) + + val d = new HashJoinOpDesc[java.lang.Long] + d.buildAttributeName = "k" + d.probeAttributeName = "kk" + d.joinType = JoinType.FULL_OUTER + val block = + d.generateStandaloneCode(Map(PortIdentity() -> left, PortIdentity(1) -> right)) + + val driver = + s"""import pandas as pd + | + |in1df = pd.DataFrame({"k": [1], "id": [9007199254740993]}) + |in2df = pd.DataFrame({"kk": [2], "y": ["b"]}) + |$block + |print(out1df["id"].iloc[0]) + |""".stripMargin + + val script = Files.createTempFile("hashjoin-outer-big-int-", ".py") + script.toFile.deleteOnExit() + Files.write(script, driver.getBytes(StandardCharsets.UTF_8)) + val process = new ProcessBuilder(python, script.toString).redirectErrorStream(true).start() + val out = Source.fromInputStream(process.getInputStream).mkString + process.waitFor(120, TimeUnit.SECONDS) + withClue(s"python said:\n$out\nscript:\n$driver") { + process.exitValue() shouldBe 0 + // Not the 9007199254740992 a round trip through float64 leaves behind. + out.trim shouldBe "9007199254740993" + } + } + // The two sides can name a column alike and declare it differently, and only // the left one is the integer: the right one wears its rename out. it should "leave a right column alone that shares an integer column's name" in { @@ -373,7 +417,23 @@ class HashJoinOpDescSpec extends AnyFlatSpec with Matchers { d.buildAttributeName = "k" d.probeAttributeName = "kk" d.joinType = JoinType.FULL_OUTER - d.generateStandaloneCode() should not include "astype(\"Int64\")" + d.generateStandaloneCode() should not include "astype(" + } + + // The cast has to happen before the merge, because the merge is what digs the + // hole that costs the column its type. + it should "widen the inputs rather than the merged frame" in { + val left = Schema().add(new Attribute("k", AttributeType.INTEGER)) + val right = Schema().add(new Attribute("kk", AttributeType.INTEGER)) + val d = new HashJoinOpDesc[Integer] + d.buildAttributeName = "k" + d.probeAttributeName = "kk" + d.joinType = JoinType.FULL_OUTER + val block = d.generateStandaloneCode(Map(PortIdentity() -> left, PortIdentity(1) -> right)) + block should include("in1df.astype(_left_ints).merge(") + block should include("in2df.astype(_right_ints).rename(columns=_rename)") + // The frame the merge produced is never cast back. + block should not include "out1df[_texera_int_col]" } "HashJoinOpDesc" should "round-trip its config fields through the polymorphic base" in { From de40276f106a66ee1972ceb73f9d3b8ec8bdc7c5 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Mon, 21 Sep 2026 17:15:17 -0700 Subject: [PATCH 13/32] fix(operator): sort on keys of its own so every input column survives The sort built its null and NaN tiers as columns on the input frame, named "_texera_null_" and "_texera_nan_" plus the key. An input already carrying one of those names lost it twice over: the tier overwrote the payload, and the drop that cleared the tiers afterwards took the column with it. Sorting a frame of "x", "_texera_null_x" and "_texera_nan_x" by "x" returned "x" alone. The tiers now go in a frame of their own, named by position, and the input is reordered by the index that frame sorts to. Nothing is added to the input and nothing is dropped from it, so no name it carries can collide. The ordering is unchanged: a null still goes last whichever way the key points, and a NaN still follows the key in a dtype that tells the two apart. Co-Authored-By: Claude Opus 5 (1M context) --- .../operator/sort/StableMergeSortOpDesc.scala | 34 +++++++++------- .../sort/StableMergeSortOpDescSpec.scala | 40 +++++++++++++++++++ 2 files changed, 60 insertions(+), 14 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpDesc.scala index e498b2df8ab..b48b5e0ee09 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpDesc.scala @@ -94,24 +94,30 @@ class StableMergeSortOpDesc extends LogicalOp with StandaloneCodeGenerator { // compares above every number, so it goes last ascending and first // descending. A column read into a numpy dtype has one slot for both, and // there both land in the null tier, which is where they were before. - s"""_texera_sorted = in1df.copy() + // + // The tiers are built in a frame of their own and named by position, so + // the input keeps every column it arrived with: a helper named after the + // key would overwrite an input column that already answers to that name, + // and dropping the helper afterwards would take the payload with it. + s"""_texera_sorted = in1df.reset_index(drop=True) + |_texera_keys = pd.DataFrame(index=_texera_sorted.index) |_texera_by = [] |_texera_asc = [] - |_texera_helpers = [] - |for _texera_col, _texera_a in zip($cols, $ascending): - | _texera_null = "_texera_null_" + _texera_col - | _texera_nan = "_texera_nan_" + _texera_col - | _texera_sorted[_texera_null] = _texera_sorted[_texera_col].isna() - | _texera_sorted[_texera_nan] = ( + |for _texera_i, (_texera_col, _texera_a) in enumerate(zip($cols, $ascending)): + | _texera_null = "null_" + str(_texera_i) + | _texera_nan = "nan_" + str(_texera_i) + | _texera_val = "val_" + str(_texera_i) + | _texera_keys[_texera_null] = _texera_sorted[_texera_col].isna() + | _texera_keys[_texera_nan] = ( | _texera_sorted[_texera_col] != _texera_sorted[_texera_col] | ).fillna(False) - | _texera_helpers += [_texera_null, _texera_nan] - | _texera_by += [_texera_null, _texera_nan, _texera_col] + | _texera_keys[_texera_val] = _texera_sorted[_texera_col] + | _texera_by += [_texera_null, _texera_nan, _texera_val] | _texera_asc += [True, _texera_a, _texera_a] - |out1df = ( - | _texera_sorted.sort_values(by=_texera_by, ascending=_texera_asc, kind="mergesort") - | .drop(columns=_texera_helpers) - | .reset_index(drop=True) - |)""".stripMargin + |out1df = _texera_sorted.loc[ + | _texera_keys.sort_values( + | by=_texera_by, ascending=_texera_asc, kind="mergesort" + | ).index + |].reset_index(drop=True)""".stripMargin } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpDescSpec.scala index e2dbcdac192..b5fa9a92885 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpDescSpec.scala @@ -132,6 +132,46 @@ class StableMergeSortOpDescSpec extends AnyFlatSpec with Matchers { } } + // The sort tiers are named after nothing the input can be carrying, so a + // column that happens to answer to a helper's old name keeps its values. + it should "keep an input column named after a sort helper" in { + val python = resolvePython().getOrElse(cancel("No runnable python executable")) + if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + + val unit = new SortCriteriaUnit + unit.attributeName = "x" + unit.sortPreference = SortPreference.ASC + val desc = new StableMergeSortOpDesc + desc.keys = ListBuffer(unit) + + val driver = + s"""import pandas as pd + | + |in1df = pd.DataFrame({ + | "x": [2, 1], + | "_texera_null_x": ["a", "b"], + | "_texera_nan_x": ["c", "d"], + |}) + |${desc.generateStandaloneCode()} + |print(list(out1df.columns)) + |print(list(out1df["_texera_null_x"]), list(out1df["_texera_nan_x"])) + |""".stripMargin + + val script = Files.createTempFile("sort-helper-name-", ".py") + script.toFile.deleteOnExit() + Files.write(script, driver.getBytes(StandardCharsets.UTF_8)) + val process = new ProcessBuilder(python, script.toString).redirectErrorStream(true).start() + val out = Source.fromInputStream(process.getInputStream).mkString + process.waitFor(120, TimeUnit.SECONDS) + + withClue(s"python said:\n$out\nscript:\n$driver") { + process.exitValue() shouldBe 0 + val lines = out.trim.linesIterator.toSeq + lines.head shouldBe "['x', '_texera_null_x', '_texera_nan_x']" + lines(1) shouldBe "['b', 'a'] ['d', 'c']" + } + } + private def resolvePython(): Option[String] = { def fromConfig: Option[String] = Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption From a5f3576af9fa436748dd1742a4e7370d25b14f9e Mon Sep 17 00:00:00 2001 From: kary zheng Date: Mon, 21 Sep 2026 17:15:26 -0700 Subject: [PATCH 14/32] fix(operator): set the probe key aside before renaming the payload The probe key is dropped after the merge, but it was still counted when the payload columns looked for a free name, so it pushed one of them a suffix further along than the engine does. Joining a left "k", "x" with a right "x#@1", "x" on "x#@1" emitted "x#@1#@1" where JoinUtils and the operator's own schema propagation both emit "x#@1", and a downstream projection asking for the promised name failed. A right input that has already been through a join is where those names come from, so a chain of two joins is enough to reach it. JoinUtils filters the probe key out before it renames anything, and the rename now does the same. The key still rides along for the merge to join on, and is given a name nothing else claims before it is dropped again. Checked against the engine's own loop over every left and right column set that can be built from "k", "x", "x#@1", "x#@1#@1" and "pk". Co-Authored-By: Claude Opus 5 (1M context) --- .../operator/hashJoin/HashJoinOpDesc.scala | 19 ++++-- .../hashJoin/HashJoinOpDescSpec.scala | 64 +++++++++++++++++-- 2 files changed, 74 insertions(+), 9 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDesc.scala index 38213bf972e..a8668df3b8c 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDesc.scala @@ -260,18 +260,29 @@ class HashJoinOpDesc[K] extends LogicalOp with StandaloneCodeGenerator { // HashJoinProbeOpExec's rename, written out: append "#@1" until the name is // free. pandas' `suffixes` appends once and then refuses the duplicate it // just made. + // + // JoinUtils sets the probe key aside before it renames anything, so a + // payload column never has to step around the key that is about to leave. + // The key itself still rides along, because the merge joins on it, and it + // only has to land where nothing else already sits. val merge = s"""_left_cols = set(in1df.columns) |_right_cols = list(in2df.columns) - |_right_set = set(_right_cols) + |_payload = [_c for _c in _right_cols if _c != $probeKeyLit] + |_payload_set = set(_payload) |_rename = {} - |for _col in _right_cols: + |for _col in _payload: | _new = _col - | _others = _right_set - {_col} + | _others = _payload_set - {_col} | while _new in _left_cols or _new in _others: | _new = _new + "#@1" | _rename[_col] = _new - |_probe_key = _rename.get($probeKeyLit, $probeKeyLit) + |_probe_key = $probeKeyLit + |_taken = _left_cols | set(_rename.values()) + |while _probe_key in _taken: + | _probe_key = _probe_key + "#@1" + |if _probe_key != $probeKeyLit: + | _rename[$probeKeyLit] = _probe_key |""".stripMargin + widening + s"""out1df = $leftFrame.merge( | $rightFrame, diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDescSpec.scala index 78230e3530f..da96f9add0e 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDescSpec.scala @@ -87,11 +87,13 @@ class HashJoinOpDescSpec extends AnyFlatSpec with Matchers { val d = new HashJoinOpDesc[String] d.buildAttributeName = "id" d.probeAttributeName = "key" - // "key" is renamed only when the left frame has a column of that name, and - // the frame is not known until the script runs, so the name the rename + // "key" is moved aside only when something already sits where it would + // land, and the frames are not known until the script runs, so the name it // settled on is what gets dropped. - d.generateStandaloneCode() should include("""_probe_key = _rename.get("key", "key")""") - d.generateStandaloneCode() should include("drop(columns=[_probe_key])") + val code = d.generateStandaloneCode() + code should include("_probe_key = \"key\"") + code should include("while _probe_key in _taken:") + code should include("drop(columns=[_probe_key])") } // Both sides naming the key alike does not make it one column: the rename @@ -103,7 +105,8 @@ class HashJoinOpDescSpec extends AnyFlatSpec with Matchers { d.buildAttributeName = "k" d.probeAttributeName = "k" val code = d.generateStandaloneCode() - code should include("""_probe_key = _rename.get("k", "k")""") + code should include("_probe_key = \"k\"") + code should include("while _probe_key in _taken:") code should include("""if _probe_key != "k":""") code should include("drop(columns=[_probe_key])") } @@ -247,6 +250,57 @@ class HashJoinOpDescSpec extends AnyFlatSpec with Matchers { } } + /** A right input that already went through a join carries a "#@1" name, and + * that name can be the key of the next one. The key leaves, so it is not a + * name the payload has to step around: the engine sets it aside before it + * renames anything, and a script that counted it pushed the payload one + * suffix too far, past the name the downstream operators were promised. + */ + it should "not let the discarded probe key push a payload column aside" in { + val python = resolvePython().getOrElse(cancel("No runnable python executable")) + if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + + val left = Schema() + .add(new Attribute("k", AttributeType.INTEGER)) + .add(new Attribute("x", AttributeType.STRING)) + val right = Schema() + .add(new Attribute("x#@1", AttributeType.INTEGER)) + .add(new Attribute("x", AttributeType.STRING)) + + val d = new HashJoinOpDesc[Integer] + d.buildAttributeName = "k" + d.probeAttributeName = "x#@1" + val promised = d + .getExternalOutputSchemas(Map(PortIdentity() -> left, PortIdentity(1) -> right))( + d.operatorInfo.outputPorts.head.id + ) + .getAttributeNames + + val driver = + s"""import pandas as pd + | + |in1df = pd.DataFrame({"k": [1], "x": ["left"]}) + |in2df = pd.DataFrame({"x#@1": [1], "x": ["right"]}) + |${d.generateStandaloneCode()} + |print(",".join(map(str, out1df.columns))) + |print(",".join(map(str, out1df.iloc[0].tolist()))) + |""".stripMargin + + val script = Files.createTempFile("hashjoin-probe-key-collision-", ".py") + script.toFile.deleteOnExit() + Files.write(script, driver.getBytes(StandardCharsets.UTF_8)) + val process = new ProcessBuilder(python, script.toString).redirectErrorStream(true).start() + val out = Source.fromInputStream(process.getInputStream).mkString + process.waitFor(120, TimeUnit.SECONDS) + withClue(s"python said:\n$out\nscript:\n$driver") { + process.exitValue() shouldBe 0 + val lines = out.trim.linesIterator.toSeq + promised shouldBe List("k", "x", "x#@1") + lines.head.split(",").toList shouldBe promised + lines(1) shouldBe "1,left,right" + } + } + // A hole costs a pandas integer column its type; the engine keeps INTEGER. it should "keep a declared integer column integral across an outer join" in { val python = resolvePython().getOrElse(cancel("No runnable python executable")) From 516a0661117b33e6886f51626fe3daeefec4df13 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Mon, 21 Sep 2026 17:15:37 -0700 Subject: [PATCH 15/32] fix(operator): add timestamps in the zone the script runs in A timestamp column holds a wall clock, and the engine reads one through `Timestamp.getTime`, which answers for the instant that wall clock names in the JVM's default zone. The generated helper read the column as if the zone were UTC, so every epoch millisecond it added was a whole offset out. With the JVM in America/Mexico_City, summing midnight on 2024-01-01 and 2024-01-02 gives 2078-01-01 06:00:00 where the script said midnight, and the average differed by 21600000. UTC agreed, which is why this went unseen. The column is now localized before its integers are read and the total is carried back through the same zone, the way `new Timestamp(long)` renders one. The two daylight-saving flags are java.time's own readings: the later of a repeated hour, and a skipped one shifted past the gap. Both were checked against Java 17 in America/New_York. The two timestamp tests that were already here read as UTC without saying so, so they now name the zone to the child process, and a third runs the arithmetic in a zone that is not UTC. Co-Authored-By: Claude Opus 5 (1M context) --- .../operator/aggregate/AggregateOpDesc.scala | 32 ++++++++- .../aggregate/AggregateOpDescSpec.scala | 72 +++++++++++++++---- 2 files changed, 88 insertions(+), 16 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDesc.scala index 7acf4b3233d..d5a78cf99b0 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDesc.scala @@ -189,12 +189,35 @@ class AggregateOpDesc extends LogicalOp with StandaloneCodeGenerator { | total = int(series.sum()) | return ((total + (1 << 31)) % (1 << 32)) - (1 << 31) | + |def _texera_agg_ts_zone(): + | # The engine reads and writes a timestamp in the JVM's default zone, + | # so the arithmetic below has to name the same one. gettz() and not + | # the current offset: the zone carries its daylight rules, and each + | # instant needs the offset in force when it happened. + | from dateutil.tz import gettz + | + | return gettz() + | |def _texera_agg_ts_epoch_ms(series): | # A timestamp reaches the engine as its epoch milliseconds whatever | # resolution the column carries, and reading the integers out of a | # microsecond column asks for a different number than a nanosecond | # one, so cast to milliseconds before reading them. - | return series.dropna().astype("datetime64[ms]").astype("int64") + | # + | # A column holds a wall clock, and `Timestamp.getTime` answers for + | # the instant that wall clock names locally, so localize before + | # reading the integers out: left as UTC every one of them is a whole + | # offset away. The two flags are java.time's own reading of an hour + | # daylight saving repeats (the later one) or skips (shifted past the + | # gap). + | return ( + | series.dropna() + | .astype("datetime64[ms]") + | .dt.tz_localize( + | _texera_agg_ts_zone(), ambiguous=False, nonexistent=pd.Timedelta("1h") + | ) + | .astype("int64") + | ) | |def _texera_agg_ts_sum(series): | # SUM keeps the column's own type, so the engine adds the epoch @@ -202,7 +225,12 @@ class AggregateOpDesc extends LogicalOp with StandaloneCodeGenerator { | # from the total. Python integers carry the sum exactly, so the | # wrap is the only place a total loses anything. | total = int(_texera_agg_ts_epoch_ms(series).astype(object).sum()) - | return pd.Timestamp(((total + (1 << 63)) % (1 << 64)) - (1 << 63), unit="ms") + | total = ((total + (1 << 63)) % (1 << 64)) - (1 << 63) + | return ( + | pd.Timestamp(total, unit="ms", tz="UTC") + | .tz_convert(_texera_agg_ts_zone()) + | .tz_localize(None) + | ) | |def _texera_agg_ts_mean(series): | # AVERAGE is declared DOUBLE whatever column it reads, so this is diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDescSpec.scala index 3740c0edbef..6a025bd0675 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDescSpec.scala @@ -150,15 +150,12 @@ class AggregateOpDescSpec extends AnyFlatSpec with Matchers { |print(row["flags"]) |""".stripMargin - val script = Files.createTempFile("aggregate-engine-answers-", ".py") - script.toFile.deleteOnExit() - Files.write(script, driver.getBytes(StandardCharsets.UTF_8)) - val process = new ProcessBuilder(python, script.toString).redirectErrorStream(true).start() - val out = Source.fromInputStream(process.getInputStream).mkString - process.waitFor(120, TimeUnit.SECONDS) + // Timestamp arithmetic goes through the zone the script runs in, the way + // the engine's goes through the JVM's, so the zone is named here and the + // numbers below can be read. + val out = runPython(python, driver, "aggregate-engine-answers-", "UTC") withClue(s"python said:\n$out\nscript:\n$driver") { - process.exitValue() shouldBe 0 val lines = out.trim.linesIterator.toSeq lines.head shouldBe "-2147483648" // 1577836800000 + 1577923200000 milliseconds, read back as a timestamp. @@ -197,15 +194,9 @@ class AggregateOpDescSpec extends AnyFlatSpec with Matchers { |${report("us")} |""".stripMargin - val script = Files.createTempFile("aggregate-timestamp-units-", ".py") - script.toFile.deleteOnExit() - Files.write(script, driver.getBytes(StandardCharsets.UTF_8)) - val process = new ProcessBuilder(python, script.toString).redirectErrorStream(true).start() - val out = Source.fromInputStream(process.getInputStream).mkString - process.waitFor(120, TimeUnit.SECONDS) + val out = runPython(python, driver, "aggregate-timestamp-units-", "UTC") withClue(s"python said:\n$out\nscript:\n$driver") { - process.exitValue() shouldBe 0 // Six times 1704067200000 milliseconds, read back as a timestamp: a date // no nanosecond count can hold, and the same one at either resolution. out.trim.linesIterator.toSeq shouldBe Seq( @@ -217,6 +208,59 @@ class AggregateOpDescSpec extends AnyFlatSpec with Matchers { } } + /** `Timestamp.getTime` answers for the instant a wall clock names in the + * JVM's default zone, and `new Timestamp(long)` renders one back there, so a + * script that read the column as UTC was a whole offset out on every value + * it added. Checked against Java 17 in the same zone: the sum below is + * 2078-01-01 06:00:00 and the average 1704132000000. + */ + it should "add timestamps in the zone the script runs in" in { + val python = resolvePython().getOrElse(cancel("No runnable python executable")) + if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + + val input = Schema().add("t", AttributeType.TIMESTAMP) + val desc = descWith( + List.empty, + aggOp(AggregationFunction.SUM, "t", "ts_total"), + aggOp(AggregationFunction.AVERAGE, "t", "ts_avg") + ) + val block = desc.generateStandaloneCode(Map(PortIdentity() -> input)) + + val driver = + s"""import pandas as pd + |in1df = pd.DataFrame({ + | "t": pd.to_datetime(["2024-01-01 00:00:00", "2024-01-02 00:00:00"]), + |}) + |$block + |print(pd.Timestamp(out1df.iloc[0]["ts_total"]).isoformat()) + |print(repr(float(out1df.iloc[0]["ts_avg"]))) + |""".stripMargin + + // A zone six hours behind UTC and with no daylight saving of its own, so + // one offset covers both dates. + val out = runPython(python, driver, "aggregate-timestamp-zone-", "America/Mexico_City") + + withClue(s"python said:\n$out\nscript:\n$driver") { + out.trim.linesIterator.toSeq shouldBe Seq("2078-01-01T06:00:00", "1704132000000.0") + } + } + + /** Runs the generated block with the zone named to the child, because a + * timestamp's arithmetic reads the one in force where it runs. + */ + private def runPython(python: String, driver: String, prefix: String, zone: String): String = { + val script = Files.createTempFile(prefix, ".py") + script.toFile.deleteOnExit() + Files.write(script, driver.getBytes(StandardCharsets.UTF_8)) + val builder = new ProcessBuilder(python, script.toString).redirectErrorStream(true) + builder.environment().put("TZ", zone) + val process = builder.start() + val out = Source.fromInputStream(process.getInputStream).mkString + process.waitFor(120, TimeUnit.SECONDS) + withClue(s"python said:\n$out\nscript:\n$driver")(process.exitValue() shouldBe 0) + out + } + // A group keyed on a missing value and one keyed on a NaN are two groups to // the engine. The two are only distinct in a nullable dtype, which is what an // Arrow file is read into. From 88ff2a94e9a9dc96fe5a191bbdc8ba2edcb3bc83 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Fri, 25 Sep 2026 23:59:17 -0700 Subject: [PATCH 16/32] fix(operator): name a join's right columns by one rule in the schema and the rows HashJoin declared a right column's new name by checking only the left names, while the executor also steers clear of the other right names, so a right `x` next to a right `x#@1` was declared as a second `x#@1` and one value was lost when the row was matched to the schema by name. An unmatched right row in a right or full outer join kept its raw names and landed in the left column. Both now go through one renaming rule. The export spec cases that the shared verify tables now run on both paths are removed from the HashJoin and Aggregate specs. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../operator/hashJoin/HashJoinOpDesc.scala | 20 +- .../hashJoin/HashJoinProbeOpExec.scala | 50 ++- .../aggregate/AggregateOpDescSpec.scala | 51 --- .../hashJoin/HashJoinOpDescSpec.scala | 365 +----------------- .../hashJoin/HashJoinProbeOpExecSpec.scala | 80 ++++ 5 files changed, 126 insertions(+), 440 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDesc.scala index a8668df3b8c..097601dbe6e 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDesc.scala @@ -155,19 +155,19 @@ class HashJoinOpDesc[K] extends LogicalOp with StandaloneCodeGenerator { buildSchema.getAttributes.filterNot(_.getName == HASH_JOIN_INTERNAL_KEY_NAME) val leftAttributeNames = leftAttributes.map(_.getName).toSet - // Filter and rename attributes from the probe schema to avoid conflicts + // Filter and rename attributes from the probe schema to avoid conflicts, + // by the rule the executor names its fields with. Checking the left + // side alone declared two columns under one name when the right side + // already carried the suffixed one, and the row, matched to the schema + // by name, lost the value the executor had put under the other name. + val rightNames = probeSchema.getAttributeNames.filterNot(_ == probeAttributeName) val rightAttributes = probeSchema.getAttributes .filterNot(_.getName == probeAttributeName) .map { attr => - var newName = attr.getName - while (leftAttributeNames.contains(newName)) { - val suffixIndex = """#@(\d+)$""".r - .findFirstMatchIn(newName) - .map(_.group(1).toInt + 1) - .getOrElse(1) - newName = s"${attr.getName}#@$suffixIndex" - } - new Attribute(newName, attr.getType) + new Attribute( + JoinUtils.renamed(attr.getName, leftAttributeNames.toSeq, rightNames), + attr.getType + ) } // Combine left and right attributes into a new schema diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinProbeOpExec.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinProbeOpExec.scala index a27ae1d5e24..4b94b404984 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinProbeOpExec.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinProbeOpExec.scala @@ -28,6 +28,20 @@ import scala.collection.mutable import scala.collection.mutable.ListBuffer object JoinUtils { + + /** The name a right column takes in the joined row: "#@1" appended until it is + * free of every left name and of every other right name. The schema the + * operator declares, a matched row and an unmatched right row all name their + * right columns by this one rule, since the row is matched to the schema by + * name and a column the two name apart loses its value. + */ + def renamed(name: String, leftNames: Seq[String], rightNames: Seq[String]): String = { + val others = rightNames.filterNot(_ == name) + var newName = name + while (leftNames.contains(newName) || others.contains(newName)) newName = s"$newName#@1" + newName + } + def joinTuples( leftTuple: Tuple, rightTuple: Tuple, @@ -45,15 +59,7 @@ object JoinUtils { // Create a Map from rightTuple's fields, renaming conflicts val rightTupleFields = rightAttributeNames .map { name => - var newName = name - while ( - leftAttributeNames.contains(newName) || rightAttributeNames - .filter(attrName => name != attrName) - .contains(newName) - ) { - newName = s"$newName#@1" - } - newName -> rightTuple.getField[Any](name) + renamed(name, leftAttributeNames, rightAttributeNames) -> rightTuple.getField[Any](name) } TupleLike(leftTupleFields ++ rightTupleFields) @@ -68,6 +74,10 @@ class HashJoinProbeOpExec[K]( objectMapper.readValue(descString, classOf[HashJoinOpDesc[K]]) var buildTableHashMap: mutable.HashMap[K, (ListBuffer[Tuple], Boolean)] = _ + // The build side's column names, learned from its first row. An unmatched + // right row has no build row of its own to name them. + private var leftAttributeNames: List[String] = List.empty + override def open(): Unit = { buildTableHashMap = new mutable.HashMap[K, (mutable.ListBuffer[Tuple], Boolean)]() } @@ -80,10 +90,10 @@ class HashJoinProbeOpExec[K]( if (port == 0) { // Load build hash map val key = tuple.getField[K](HASH_JOIN_INTERNAL_KEY_NAME) + val names = tuple.getSchema.getAttributeNames.filterNot(n => n == HASH_JOIN_INTERNAL_KEY_NAME) + if (leftAttributeNames.isEmpty) leftAttributeNames = names buildTableHashMap.getOrElseUpdate(key, (new ListBuffer[Tuple](), false))._1 += tuple - .getPartialTuple( - tuple.getSchema.getAttributeNames.filterNot(n => n == HASH_JOIN_INTERNAL_KEY_NAME) - ) + .getPartialTuple(names) Iterator.empty } else { // Probe phase @@ -142,11 +152,21 @@ class HashJoinProbeOpExec[K]( } } - private def performRightAntiJoin(tuple: Tuple): Iterator[TupleLike] = + // The right columns are named as a matched row names them, so a column the + // left side also names goes under its suffixed name and not into the left + // column. The probe key keeps its own name: where both sides name the key + // alike, the unmatched row's key is what fills the left key column. + private def performRightAntiJoin(tuple: Tuple): Iterator[TupleLike] = { + val rightNames = tuple.getSchema.getAttributeNames.filterNot(_ == desc.probeAttributeName) Iterator( TupleLike( - tuple.getSchema.getAttributeNames - .map(attributeName => attributeName -> tuple.getField(attributeName)): _* + tuple.getSchema.getAttributeNames.map { name => + val named = + if (name == desc.probeAttributeName) name + else JoinUtils.renamed(name, leftAttributeNames, rightNames) + named -> tuple.getField[Any](name) + }: _* ) ) + } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDescSpec.scala index 6a025bd0675..cd7cbd74e5b 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDescSpec.scala @@ -114,57 +114,6 @@ class AggregateOpDescSpec extends AnyFlatSpec with Matchers { } } - // Run rather than asserted as text: the arithmetic is the point. - it should "answer SUM, AVERAGE and CONCAT the way the engine answers them" in { - val python = resolvePython().getOrElse(cancel("No runnable python executable")) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") - - val input = Schema() - .add("i", AttributeType.INTEGER) - .add("t", AttributeType.TIMESTAMP) - .add("b", AttributeType.BOOLEAN) - val desc = descWith( - List.empty, - aggOp(AggregationFunction.SUM, "i", "int_total"), - aggOp(AggregationFunction.SUM, "t", "ts_total"), - aggOp(AggregationFunction.AVERAGE, "t", "ts_avg"), - aggOp(AggregationFunction.CONCAT, "b", "flags") - ) - val block = desc.generateStandaloneCode(Map(PortIdentity() -> input)) - - val driver = - s"""import pandas as pd - |in1df = pd.DataFrame({ - | "i": pd.Series([2147483647, 1], dtype="int64"), - | "t": pd.to_datetime(["2020-01-01", "2020-01-02"]), - | "b": pd.Series([True, False], dtype=bool), - |}) - |$block - |row = out1df.iloc[0] - |# The engine adds INTEGER as a Java int, which wraps; sums timestamps - |# into a timestamp; declares AVERAGE a DOUBLE whatever it read; and - |# folds CONCAT with Java's toString, which lowercases a boolean. - |print(int(row["int_total"])) - |print(pd.Timestamp(row["ts_total"]).isoformat()) - |print(repr(float(row["ts_avg"]))) - |print(row["flags"]) - |""".stripMargin - - // Timestamp arithmetic goes through the zone the script runs in, the way - // the engine's goes through the JVM's, so the zone is named here and the - // numbers below can be read. - val out = runPython(python, driver, "aggregate-engine-answers-", "UTC") - - withClue(s"python said:\n$out\nscript:\n$driver") { - val lines = out.trim.linesIterator.toSeq - lines.head shouldBe "-2147483648" - // 1577836800000 + 1577923200000 milliseconds, read back as a timestamp. - lines(1) shouldBe "2070-01-01T00:00:00" - lines(2) shouldBe "1577880000000.0" - lines(3) shouldBe "true,false" - } - } - // The integers behind a timestamp column mean microseconds or nanoseconds // depending on the resolution it was read at, and a nanosecond total leaves // the range of a 64-bit integer after a handful of modern dates. diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDescSpec.scala index da96f9add0e..5485507c013 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDescSpec.scala @@ -82,356 +82,10 @@ class HashJoinOpDescSpec extends AnyFlatSpec with Matchers { out(d.operatorInfo.outputPorts.head.id).getAttributeNames shouldBe List("a", "k", "k#@1") } - "HashJoinOpDesc.generateStandaloneCode" should - "drop the probe key under the name the merge left it with" in { - val d = new HashJoinOpDesc[String] - d.buildAttributeName = "id" - d.probeAttributeName = "key" - // "key" is moved aside only when something already sits where it would - // land, and the frames are not known until the script runs, so the name it - // settled on is what gets dropped. - val code = d.generateStandaloneCode() - code should include("_probe_key = \"key\"") - code should include("while _probe_key in _taken:") - code should include("drop(columns=[_probe_key])") - } - - // Both sides naming the key alike does not make it one column: the rename - // moves the right one aside because it meets the left one, so the drop is - // decided by the name the rename produced rather than by the two the operator - // was given. - it should "emit the shared key once when both sides name it the same" in { - val d = new HashJoinOpDesc[String] - d.buildAttributeName = "k" - d.probeAttributeName = "k" - val code = d.generateStandaloneCode() - code should include("_probe_key = \"k\"") - code should include("while _probe_key in _taken:") - code should include("""if _probe_key != "k":""") - code should include("drop(columns=[_probe_key])") - } - - /** The bug this pins: with a left `key` payload and a right `key` join column, - * dropping the bare `key` took the payload and kept the right key. Run the - * generated pandas and hold its columns against the schema the operator - * itself promises for the same two inputs. - */ - it should "return the columns getExternalOutputSchemas promises" in { - val python = resolvePython().getOrElse( - cancel("No runnable python executable (udf.conf python.path, python3, python, py)") - ) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") - - val left = Schema() - .add(new Attribute("id", AttributeType.INTEGER)) - .add(new Attribute("key", AttributeType.STRING)) - val right = Schema() - .add(new Attribute("key", AttributeType.INTEGER)) - .add(new Attribute("value", AttributeType.INTEGER)) - - val d = new HashJoinOpDesc[Integer] - d.buildAttributeName = "id" - d.probeAttributeName = "key" - val promised = d - .getExternalOutputSchemas(Map(PortIdentity() -> left, PortIdentity(1) -> right))( - d.operatorInfo.outputPorts.head.id - ) - .getAttributeNames - - val driver = - s"""import pandas as pd - | - |in1df = pd.DataFrame({"id": [1], "key": ["payload"]}) - |in2df = pd.DataFrame({"key": [1], "value": [9]}) - |${d.generateStandaloneCode()} - |print(",".join(map(str, out1df.columns))) - |print(",".join(map(str, out1df.iloc[0].tolist()))) - |""".stripMargin - - val script = Files.createTempFile("hashjoin-columns-", ".py") - script.toFile.deleteOnExit() - Files.write(script, driver.getBytes(StandardCharsets.UTF_8)) - val process = new ProcessBuilder(python, script.toString).redirectErrorStream(true).start() - val out = Source.fromInputStream(process.getInputStream).mkString - process.waitFor(120, TimeUnit.SECONDS) - withClue(s"python said:\n$out\nscript:\n$driver") { - process.exitValue() shouldBe 0 - val lines = out.trim.linesIterator.toSeq - lines.head.split(",").toList shouldBe promised - // The payload the drop used to take, not the right key it used to keep. - lines(1) shouldBe "1,payload,9" - } - } - - /** The bug this pins: both sides naming the key alike reads as one shared - * column, but the rename moves the right one aside before the merge, so the - * frame carried a second `k#@1` the operator never promises. This is the - * everyday join, the same name on both sides. - */ - it should "promise and emit the key once when both sides name it the same" in { - val python = resolvePython().getOrElse( - cancel("No runnable python executable (udf.conf python.path, python3, python, py)") - ) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") - - val left = Schema() - .add(new Attribute("k", AttributeType.INTEGER)) - .add(new Attribute("name", AttributeType.STRING)) - val right = Schema() - .add(new Attribute("k", AttributeType.INTEGER)) - .add(new Attribute("score", AttributeType.INTEGER)) - - val d = new HashJoinOpDesc[Integer] - d.buildAttributeName = "k" - d.probeAttributeName = "k" - val promised = d - .getExternalOutputSchemas(Map(PortIdentity() -> left, PortIdentity(1) -> right))( - d.operatorInfo.outputPorts.head.id - ) - .getAttributeNames - - val driver = - s"""import pandas as pd - | - |in1df = pd.DataFrame({"k": [1], "name": ["alice"]}) - |in2df = pd.DataFrame({"k": [1], "score": [95]}) - |${d.generateStandaloneCode()} - |print(",".join(map(str, out1df.columns))) - |print(",".join(map(str, out1df.iloc[0].tolist()))) - |""".stripMargin - - val script = Files.createTempFile("hashjoin-shared-key-", ".py") - script.toFile.deleteOnExit() - Files.write(script, driver.getBytes(StandardCharsets.UTF_8)) - val process = new ProcessBuilder(python, script.toString).redirectErrorStream(true).start() - val out = Source.fromInputStream(process.getInputStream).mkString - process.waitFor(120, TimeUnit.SECONDS) - withClue(s"python said:\n$out\nscript:\n$driver") { - process.exitValue() shouldBe 0 - val lines = out.trim.linesIterator.toSeq - promised shouldBe List("k", "name", "score") - lines.head.split(",").toList shouldBe promised - lines(1) shouldBe "1,alice,95" - } - } - - // A right column meeting an already-suffixed one lands on "#@1#@1". pandas' - // `suffixes` refused the duplicate it made and ended the run. - it should "rename a right column that collides twice, the way the engine does" in { - val python = resolvePython().getOrElse(cancel("No runnable python executable")) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") - - val d = new HashJoinOpDesc[Integer] - d.buildAttributeName = "k" - d.probeAttributeName = "k" - - val driver = - s"""import pandas as pd - | - |in1df = pd.DataFrame({"k": [1], "x": [10]}) - |in2df = pd.DataFrame({"k": [1], "x": [20], "x#@1": [30]}) - |${d.generateStandaloneCode()} - |print(",".join(map(str, out1df.columns))) - |""".stripMargin - - val script = Files.createTempFile("hashjoin-chained-rename-", ".py") - script.toFile.deleteOnExit() - Files.write(script, driver.getBytes(StandardCharsets.UTF_8)) - val process = new ProcessBuilder(python, script.toString).redirectErrorStream(true).start() - val out = Source.fromInputStream(process.getInputStream).mkString - process.waitFor(120, TimeUnit.SECONDS) - withClue(s"python said:\n$out\nscript:\n$driver") { - process.exitValue() shouldBe 0 - // Left names first and untouched, then the right ones in their own order: - // "x" lost a collision and the right's own "x#@1" was free. The right "k" - // lost one too, and being the join key it is then dropped, the engine - // emitting the key once. - out.trim shouldBe "k,x,x#@1#@1,x#@1" - } - } - - /** A right input that already went through a join carries a "#@1" name, and - * that name can be the key of the next one. The key leaves, so it is not a - * name the payload has to step around: the engine sets it aside before it - * renames anything, and a script that counted it pushed the payload one - * suffix too far, past the name the downstream operators were promised. - */ - it should "not let the discarded probe key push a payload column aside" in { - val python = resolvePython().getOrElse(cancel("No runnable python executable")) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") - - val left = Schema() - .add(new Attribute("k", AttributeType.INTEGER)) - .add(new Attribute("x", AttributeType.STRING)) - val right = Schema() - .add(new Attribute("x#@1", AttributeType.INTEGER)) - .add(new Attribute("x", AttributeType.STRING)) - - val d = new HashJoinOpDesc[Integer] - d.buildAttributeName = "k" - d.probeAttributeName = "x#@1" - val promised = d - .getExternalOutputSchemas(Map(PortIdentity() -> left, PortIdentity(1) -> right))( - d.operatorInfo.outputPorts.head.id - ) - .getAttributeNames - - val driver = - s"""import pandas as pd - | - |in1df = pd.DataFrame({"k": [1], "x": ["left"]}) - |in2df = pd.DataFrame({"x#@1": [1], "x": ["right"]}) - |${d.generateStandaloneCode()} - |print(",".join(map(str, out1df.columns))) - |print(",".join(map(str, out1df.iloc[0].tolist()))) - |""".stripMargin - - val script = Files.createTempFile("hashjoin-probe-key-collision-", ".py") - script.toFile.deleteOnExit() - Files.write(script, driver.getBytes(StandardCharsets.UTF_8)) - val process = new ProcessBuilder(python, script.toString).redirectErrorStream(true).start() - val out = Source.fromInputStream(process.getInputStream).mkString - process.waitFor(120, TimeUnit.SECONDS) - withClue(s"python said:\n$out\nscript:\n$driver") { - process.exitValue() shouldBe 0 - val lines = out.trim.linesIterator.toSeq - promised shouldBe List("k", "x", "x#@1") - lines.head.split(",").toList shouldBe promised - lines(1) shouldBe "1,left,right" - } - } - - // A hole costs a pandas integer column its type; the engine keeps INTEGER. - it should "keep a declared integer column integral across an outer join" in { - val python = resolvePython().getOrElse(cancel("No runnable python executable")) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") - - val left = Schema() - .add(new Attribute("k", AttributeType.INTEGER)) - .add(new Attribute("x", AttributeType.INTEGER)) - val right = Schema() - .add(new Attribute("kk", AttributeType.INTEGER)) - .add(new Attribute("y", AttributeType.INTEGER)) - - val d = new HashJoinOpDesc[Integer] - d.buildAttributeName = "k" - d.probeAttributeName = "kk" - d.joinType = JoinType.FULL_OUTER - val block = - d.generateStandaloneCode(Map(PortIdentity() -> left, PortIdentity(1) -> right)) - - val driver = - s"""import pandas as pd - | - |in1df = pd.DataFrame({"k": [1, 2], "x": [10, 20]}) - |in2df = pd.DataFrame({"kk": [1, 3], "y": [30, 40]}) - |$block - |print(",".join(f"{c}:{out1df[c].dtype}" for c in out1df.columns)) - |""".stripMargin - - val script = Files.createTempFile("hashjoin-outer-int-", ".py") - script.toFile.deleteOnExit() - Files.write(script, driver.getBytes(StandardCharsets.UTF_8)) - val process = new ProcessBuilder(python, script.toString).redirectErrorStream(true).start() - val out = Source.fromInputStream(process.getInputStream).mkString - process.waitFor(120, TimeUnit.SECONDS) - withClue(s"python said:\n$out\nscript:\n$driver") { - process.exitValue() shouldBe 0 - out.trim shouldBe "k:Int64,x:Int64,y:Int64" - } - } - - /** The bug this pins: the merge widened an unmatched integer column to - * float64, and the cast that put the type back read a value already rounded. - * 9007199254740993 is the first integer float64 cannot hold. - */ - it should "keep an integer past 2^53 exact across an outer join" in { - val python = resolvePython().getOrElse(cancel("No runnable python executable")) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") - - val left = Schema() - .add(new Attribute("k", AttributeType.LONG)) - .add(new Attribute("id", AttributeType.LONG)) - val right = Schema() - .add(new Attribute("kk", AttributeType.LONG)) - .add(new Attribute("y", AttributeType.STRING)) - - val d = new HashJoinOpDesc[java.lang.Long] - d.buildAttributeName = "k" - d.probeAttributeName = "kk" - d.joinType = JoinType.FULL_OUTER - val block = - d.generateStandaloneCode(Map(PortIdentity() -> left, PortIdentity(1) -> right)) - - val driver = - s"""import pandas as pd - | - |in1df = pd.DataFrame({"k": [1], "id": [9007199254740993]}) - |in2df = pd.DataFrame({"kk": [2], "y": ["b"]}) - |$block - |print(out1df["id"].iloc[0]) - |""".stripMargin - - val script = Files.createTempFile("hashjoin-outer-big-int-", ".py") - script.toFile.deleteOnExit() - Files.write(script, driver.getBytes(StandardCharsets.UTF_8)) - val process = new ProcessBuilder(python, script.toString).redirectErrorStream(true).start() - val out = Source.fromInputStream(process.getInputStream).mkString - process.waitFor(120, TimeUnit.SECONDS) - withClue(s"python said:\n$out\nscript:\n$driver") { - process.exitValue() shouldBe 0 - // Not the 9007199254740992 a round trip through float64 leaves behind. - out.trim shouldBe "9007199254740993" - } - } - - // The two sides can name a column alike and declare it differently, and only - // the left one is the integer: the right one wears its rename out. - it should "leave a right column alone that shares an integer column's name" in { - val python = resolvePython().getOrElse(cancel("No runnable python executable")) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") - - val left = Schema() - .add(new Attribute("k", AttributeType.INTEGER)) - .add(new Attribute("x", AttributeType.INTEGER)) - val right = Schema() - .add(new Attribute("kk", AttributeType.INTEGER)) - .add(new Attribute("x", AttributeType.DOUBLE)) - - val d = new HashJoinOpDesc[Integer] - d.buildAttributeName = "k" - d.probeAttributeName = "kk" - d.joinType = JoinType.FULL_OUTER - val block = - d.generateStandaloneCode(Map(PortIdentity() -> left, PortIdentity(1) -> right)) - - val driver = - s"""import pandas as pd - | - |in1df = pd.DataFrame({"k": [1, 2], "x": [10, 20]}) - |in2df = pd.DataFrame({"kk": [1, 3], "x": [1.5, 2.5]}) - |$block - |print(",".join(f"{c}:{out1df[c].dtype}" for c in out1df.columns)) - |""".stripMargin - - val script = Files.createTempFile("hashjoin-outer-int-collision-", ".py") - script.toFile.deleteOnExit() - Files.write(script, driver.getBytes(StandardCharsets.UTF_8)) - val process = new ProcessBuilder(python, script.toString).redirectErrorStream(true).start() - val out = Source.fromInputStream(process.getInputStream).mkString - process.waitFor(120, TimeUnit.SECONDS) - withClue(s"python said:\n$out\nscript:\n$driver") { - process.exitValue() shouldBe 0 - // The renamed right column keeps its 1.5, rather than being cast to the - // integer the left column declared. - out.trim shouldBe "k:Int64,x:Int64,x#@1:float64" - } - } - // A key that is missing and a key holding a NaN are two different keys to the // engine, so only the missing one matches a missing right key. The two are // only distinct in a nullable dtype, which is what an Arrow file is read into. - it should "match a missing key but not a NaN one" in { + "HashJoinOpDesc.generateStandaloneCode" should "match a missing key but not a NaN one" in { val python = resolvePython().getOrElse(cancel("No runnable python executable")) if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") @@ -473,23 +127,6 @@ class HashJoinOpDescSpec extends AnyFlatSpec with Matchers { d.joinType = JoinType.FULL_OUTER d.generateStandaloneCode() should not include "astype(" } - - // The cast has to happen before the merge, because the merge is what digs the - // hole that costs the column its type. - it should "widen the inputs rather than the merged frame" in { - val left = Schema().add(new Attribute("k", AttributeType.INTEGER)) - val right = Schema().add(new Attribute("kk", AttributeType.INTEGER)) - val d = new HashJoinOpDesc[Integer] - d.buildAttributeName = "k" - d.probeAttributeName = "kk" - d.joinType = JoinType.FULL_OUTER - val block = d.generateStandaloneCode(Map(PortIdentity() -> left, PortIdentity(1) -> right)) - block should include("in1df.astype(_left_ints).merge(") - block should include("in2df.astype(_right_ints).rename(columns=_rename)") - // The frame the merge produced is never cast back. - block should not include "out1df[_texera_int_col]" - } - "HashJoinOpDesc" should "round-trip its config fields through the polymorphic base" in { val d = new HashJoinOpDesc[String] d.buildAttributeName = "lk" diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinProbeOpExecSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinProbeOpExecSpec.scala index 2a98fd214e9..79150fee4ad 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinProbeOpExecSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinProbeOpExecSpec.scala @@ -20,6 +20,8 @@ package org.apache.texera.amber.operator.hashJoin import org.apache.texera.amber.core.tuple._ +import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} +import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.operator.hashJoin.HashJoinOpDesc.HASH_JOIN_INTERNAL_KEY_NAME import org.apache.texera.amber.util.JSONUtils.objectMapper import org.scalatest.flatspec.AnyFlatSpec @@ -274,4 +276,82 @@ class HashJoinProbeOpExecSpec extends AnyFlatSpec { probeExec.close() assert(probeExec.buildTableHashMap.isEmpty) } + + /** The rows a join over `left` and `right` on `k` emits, matched to the schema + * the operator declares by name, as the engine matches them. + */ + private def joinedRows( + joinType: JoinType, + left: Schema, + leftRows: Seq[Array[Any]], + right: Schema, + rightRows: Seq[Array[Any]] + ): (List[String], List[List[Any]]) = { + val desc = new HashJoinOpDesc[String]() + desc.buildAttributeName = "k" + desc.probeAttributeName = "k" + desc.joinType = joinType + val plan = desc.getPhysicalPlan(WorkflowIdentity(1L), ExecutionIdentity(1L)) + val build = plan.operators.find(_.id.layerName == "build").get + val probe = plan.operators.find(_.id.layerName == "probe").get + val buildOut = + build.propagateSchema.func(Map(PortIdentity() -> left))(PortIdentity(0, internal = true)) + val declared = probe.propagateSchema + .func(Map(PortIdentity(0, internal = true) -> buildOut, PortIdentity(1) -> right))( + PortIdentity() + ) + + val descString = objectMapper.writeValueAsString(desc) + val buildExec = new HashJoinBuildOpExec[String](descString) + buildExec.open() + leftRows.foreach(r => buildExec.processTuple(Tuple.builder(left).addSequentially(r).build(), 0)) + val probeExec = new HashJoinProbeOpExec[String](descString) + probeExec.open() + buildExec + .onFinish(0) + .foreach(t => + probeExec.processTuple(t.asInstanceOf[SchemaEnforceable].enforceSchema(buildOut), 0) + ) + val emitted = rightRows.flatMap(r => + probeExec.processTuple(Tuple.builder(right).addSequentially(r).build(), 1) + ) ++ probeExec.onFinish(1) + ( + declared.getAttributeNames, + emitted.map(_.asInstanceOf[SchemaEnforceable].enforceSchema(declared).getFields.toList).toList + ) + } + + // A right column meeting a left name takes "#@1", and the right side can + // already hold that name, as the output of an earlier join does. The schema + // declared two columns under it while the executor named the row "#@1#@1", + // so matching the row to the schema lost the right x and doubled the other. + it should "name a right column colliding twice alike in the schema and the row" in { + val (names, rows) = joinedRows( + JoinType.INNER, + Schema().add("k", AttributeType.STRING).add("x", AttributeType.STRING), + Seq(Array[Any]("1", "left")), + Schema() + .add("k", AttributeType.STRING) + .add("x", AttributeType.STRING) + .add("x#@1", AttributeType.STRING), + Seq(Array[Any]("1", "right x", "right x#@1")) + ) + assert(names == List("k", "x", "x#@1#@1", "x#@1")) + assert(rows == List(List("1", "left", "right x", "right x#@1"))) + } + + // An unmatched right row kept its columns' own names, so one the left side + // also names landed in the left column, and failed there on a type the left + // one did not declare. Its key still fills the left key column. + it should "name an unmatched right row's columns as a joined row names them" in { + val (names, rows) = joinedRows( + JoinType.FULL_OUTER, + Schema().add("k", AttributeType.STRING).add("x", AttributeType.INTEGER), + Seq(Array[Any]("1", 10)), + Schema().add("k", AttributeType.STRING).add("x", AttributeType.DOUBLE), + Seq(Array[Any]("2", 2.5)) + ) + assert(names == List("k", "x", "x#@1")) + assert(rows.toSet == Set(List("2", null, 2.5), List("1", 10, null))) + } } From d80f561bb3f5a04280ddbf0e34db754255d3873b Mon Sep 17 00:00:00 2001 From: kary zheng Date: Sat, 26 Sep 2026 13:21:19 -0700 Subject: [PATCH 17/32] fix(operator): put an unmatched right row's key where the engine puts it When both sides name the join key alike, the engine's unmatched right row fills the left key column with its own key, where the exported right or full outer join left it empty. The engine in turn put an unmatched right key into a left payload column that happened to share its name; it now leaves the key out there. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../operator/hashJoin/HashJoinOpDesc.scala | 21 +++++++++++-- .../hashJoin/HashJoinProbeOpExec.scala | 19 +++++++----- .../hashJoin/HashJoinProbeOpExecSpec.scala | 30 +++++++++++++++---- 3 files changed, 54 insertions(+), 16 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDesc.scala index 097601dbe6e..d139ec65079 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDesc.scala @@ -296,10 +296,25 @@ class HashJoinOpDesc[K] extends LogicalOp with StandaloneCodeGenerator { // when both sides name the key alike the rename still moves the right one // aside, and a test on the operator's own two names read that as one shared // column and kept the copy, where the engine emits the key once. + // + // Where both sides name the key alike, the engine's unmatched right row + // fills the left key column with its own key. The merge left that column + // empty and put the key in the renamed right one, which is about to go. + // A matched row holds the same value in both, so filling the holes changes + // only the rows the right side alone brought. + val rightRowsKept = joinType == JoinType.RIGHT_OUTER || joinType == JoinType.FULL_OUTER + val fillKey = + if (rightRowsKept && buildAttributeName == probeAttributeName) + s"""out1df[$buildKeyLit] = out1df[$buildKeyLit].where( + | out1df[$buildKeyLit].notna(), out1df[_probe_key] + |) + |""".stripMargin + else "" val tail = - s"""if _probe_key != $buildKeyLit: - | out1df = out1df.drop(columns=[_probe_key]) - |out1df = out1df.reset_index(drop=True)""".stripMargin + fillKey + + s"""if _probe_key != $buildKeyLit: + | out1df = out1df.drop(columns=[_probe_key]) + |out1df = out1df.reset_index(drop=True)""".stripMargin s"$merge\n$tail" } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinProbeOpExec.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinProbeOpExec.scala index 4b94b404984..e6783d3bf8c 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinProbeOpExec.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinProbeOpExec.scala @@ -154,17 +154,20 @@ class HashJoinProbeOpExec[K]( // The right columns are named as a matched row names them, so a column the // left side also names goes under its suffixed name and not into the left - // column. The probe key keeps its own name: where both sides name the key - // alike, the unmatched row's key is what fills the left key column. + // column. The probe key fills the left key column where both sides name the + // key alike, and is left out where a left payload shares its name, which it + // would otherwise fill. private def performRightAntiJoin(tuple: Tuple): Iterator[TupleLike] = { - val rightNames = tuple.getSchema.getAttributeNames.filterNot(_ == desc.probeAttributeName) + val keyName = desc.probeAttributeName + val rightNames = tuple.getSchema.getAttributeNames.filterNot(_ == keyName) + val key = + if (keyName == desc.buildAttributeName || !leftAttributeNames.contains(keyName)) + Seq(keyName -> tuple.getField[Any](keyName)) + else Seq.empty Iterator( TupleLike( - tuple.getSchema.getAttributeNames.map { name => - val named = - if (name == desc.probeAttributeName) name - else JoinUtils.renamed(name, leftAttributeNames, rightNames) - named -> tuple.getField[Any](name) + key ++ rightNames.map { name => + JoinUtils.renamed(name, leftAttributeNames, rightNames) -> tuple.getField[Any](name) }: _* ) ) diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinProbeOpExecSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinProbeOpExecSpec.scala index 79150fee4ad..660aba86acd 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinProbeOpExecSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinProbeOpExecSpec.scala @@ -277,19 +277,21 @@ class HashJoinProbeOpExecSpec extends AnyFlatSpec { assert(probeExec.buildTableHashMap.isEmpty) } - /** The rows a join over `left` and `right` on `k` emits, matched to the schema - * the operator declares by name, as the engine matches them. + /** The rows a join over `left` and `right` emits, matched to the schema the + * operator declares by name, as the engine matches them. */ private def joinedRows( joinType: JoinType, left: Schema, leftRows: Seq[Array[Any]], right: Schema, - rightRows: Seq[Array[Any]] + rightRows: Seq[Array[Any]], + buildKey: String = "k", + probeKey: String = "k" ): (List[String], List[List[Any]]) = { val desc = new HashJoinOpDesc[String]() - desc.buildAttributeName = "k" - desc.probeAttributeName = "k" + desc.buildAttributeName = buildKey + desc.probeAttributeName = probeKey desc.joinType = joinType val plan = desc.getPhysicalPlan(WorkflowIdentity(1L), ExecutionIdentity(1L)) val build = plan.operators.find(_.id.layerName == "build").get @@ -354,4 +356,22 @@ class HashJoinProbeOpExecSpec extends AnyFlatSpec { assert(names == List("k", "x", "x#@1")) assert(rows.toSet == Set(List("2", null, 2.5), List("1", 10, null))) } + + // The key an unmatched right row carries fills the left key column only when + // the two keys share a name. Under any other name it belonged to no declared + // column, and where a left payload happened to carry that name the key landed + // in the payload. + it should "not let an unmatched right key fill a left payload that shares its name" in { + val (names, rows) = joinedRows( + JoinType.FULL_OUTER, + Schema().add("id", AttributeType.STRING).add("key", AttributeType.STRING), + Seq(Array[Any]("1", "payload")), + Schema().add("key", AttributeType.STRING).add("value", AttributeType.INTEGER), + Seq(Array[Any]("2", 9)), + buildKey = "id", + probeKey = "key" + ) + assert(names == List("id", "key", "value")) + assert(rows.toSet == Set(List(null, null, 9), List("1", "payload", null))) + } } From 928116317981af641ca3546ac9119dff7d43de7e Mon Sep 17 00:00:00 2001 From: kary zheng Date: Sat, 26 Sep 2026 20:09:32 -0700 Subject: [PATCH 18/32] fix(operator): concatenate and sample as the engine does, say which route If takes, and pin where a null and a NaN land Co-Authored-By: Claude Opus 5.5 (1M context) --- .../operator/aggregate/AggregateOpDesc.scala | 13 ++-- .../operator/hashJoin/HashJoinOpDesc.scala | 7 +- .../amber/operator/ifStatement/IfOpDesc.scala | 15 +++- .../ReservoirSamplingOpDesc.scala | 13 ++-- .../sortPartitions/SortPartitionsOpDesc.scala | 6 +- .../aggregate/AggregateOpDescSpec.scala | 10 ++- .../difference/DifferenceOpDescSpec.scala | 6 +- .../hashJoin/HashJoinOpDescSpec.scala | 6 +- .../operator/ifStatement/IfOpDescSpec.scala | 10 ++- .../intersect/IntersectOpDescSpec.scala | 71 +++++++++++++++++++ .../intervalJoin/IntervalJoinOpDescSpec.scala | 6 +- .../ReservoirSamplingOpDescSpec.scala | 14 ++-- .../sort/StableMergeSortOpDescSpec.scala | 8 ++- .../SortPartitionsOpDescSpec.scala | 70 ++++++++++++++++++ .../SymmetricDifferenceOpDescSpec.scala | 71 +++++++++++++++++++ 15 files changed, 293 insertions(+), 33 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDesc.scala index d5a78cf99b0..4cd2b8cb499 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDesc.scala @@ -29,7 +29,7 @@ import org.apache.texera.amber.core.virtualidentity.{ WorkflowIdentity } import org.apache.texera.amber.core.workflow._ -import org.apache.texera.amber.operator.{LogicalOp, StandaloneCodeGenerator} +import org.apache.texera.amber.operator.{LogicalOp, StandaloneCodeGenerator, StandaloneHelpers} import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeNameList import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral @@ -159,6 +159,8 @@ class AggregateOpDesc extends LogicalOp with StandaloneCodeGenerator { override def generateStandaloneCode(): String = build(_ => None) + override def standaloneHelpers(): Seq[String] = Seq(StandaloneHelpers.AttributeCasts) + private def build(declaredType: String => Option[AttributeType]): String = { val keys = Option(groupByKeys).getOrElse(List()) val aggs = Option(aggregations).getOrElse(List()) @@ -172,15 +174,10 @@ class AggregateOpDesc extends LogicalOp with StandaloneCodeGenerator { | # comma: "", "a", "" concatenates to "a," and not ",a,". A null is | # read as the empty string, which is what makes the two the same | # here. This is concatAgg's fold, written out. + | # Each value is written as Java's toString writes it. | partial = "" | for v in series: - | if pd.isna(v): - | text = "" - | elif isinstance(v, bool) or (hasattr(v, "dtype") and v.dtype == bool): - | # Java's toString spells a boolean in lower case. - | text = "true" if v else "false" - | else: - | text = str(v) + | text = "" if pd.isna(v) else _texera_java_text(v) | partial = text if partial == "" else partial + "," + text | return partial | diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDesc.scala index d139ec65079..74f0e67f975 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDesc.scala @@ -201,10 +201,9 @@ class HashJoinOpDesc[K] extends LogicalOp with StandaloneCodeGenerator { outputPorts = List(OutputPort()) ) - // Equi-join: drop the probe key (kept only when its name differs from the - // build key), suffix colliding right columns "#@1" — matches JoinUtils. Known - // Texera divergences: row order, null keys (NaN != NaN in merge), outer - // anti-row column placement. + // Equi-join: the probe key is dropped and a colliding right column takes + // "#@1", as JoinUtils names it. Rows come out in the merge's order rather + // than the engine's. /** Only the declared type can say which columns an outer join widened, so * without a schema the widening stands. */ diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/ifStatement/IfOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/ifStatement/IfOpDesc.scala index 7ce5fa07941..296e70e6928 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/ifStatement/IfOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/ifStatement/IfOpDesc.scala @@ -78,10 +78,23 @@ class IfOpDesc extends LogicalOp with StandaloneCodeGenerator { // Condition port. A script has no State channel, so the condition is read from // a global named after it, true when nothing set it, which is the port the // engine starts on. The Condition input carries no rows to read either way. + // + // Taking True silently would report the True route for a workflow whose + // condition says otherwise, so an unset switch says which route it took and + // how to take the other. + override def standaloneImports(): Seq[String] = Seq("import sys") + override def generateStandaloneCode(): String = { val globalName = "_texera_if_" + Option(conditionName).getOrElse("") val globalLit = pyStringLiteral(globalName) - s"""_texera_if_cond = bool(globals().get($globalLit, True)) + s"""if $globalLit not in globals(): + | print( + | "If: the script cannot receive the condition the workflow sets upstream, " + | "so it takes the True branch. To take the False branch, set " + | + $globalLit + " = False before this step.", + | file=sys.stderr, + | ) + |_texera_if_cond = bool(globals().get($globalLit, True)) |if _texera_if_cond: | out2df = in2df.copy() | out1df = in2df.iloc[0:0].copy() diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/reservoirsampling/ReservoirSamplingOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/reservoirsampling/ReservoirSamplingOpDesc.scala index efa43d253a6..a20112f84b6 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/reservoirsampling/ReservoirSamplingOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/reservoirsampling/ReservoirSamplingOpDesc.scala @@ -70,17 +70,22 @@ class ReservoirSamplingOpDesc extends LogicalOp with StandaloneCodeGenerator { // Its generator is seeded with the worker count, so the rows agree with a // single-worker run and not with a wider one. See RandomKSamplingOpDesc for // why no seed closes that gap. + // + // The reservoir holds row positions, and the rows are taken from the input + // by them, so every column keeps its dtype. A frame rebuilt from the rows' + // values infers each one again, and a timestamp past pandas' nanosecond + // range came back as an object column. override def generateStandaloneCode(): String = { s"""_texera_rs_rng = _TexeraJavaRandom(1) |_texera_rs_k = $k |_texera_rs_reservoir = [] - |for _texera_rs_n, _texera_rs_row in enumerate(in1df.itertuples(index=False, name=None)): + |for _texera_rs_n in range(len(in1df)): | if _texera_rs_n < _texera_rs_k: - | _texera_rs_reservoir.append(_texera_rs_row) + | _texera_rs_reservoir.append(_texera_rs_n) | else: | _texera_rs_i = _texera_rs_rng.next_int(_texera_rs_n) | if _texera_rs_i < _texera_rs_k: - | _texera_rs_reservoir[_texera_rs_i] = _texera_rs_row - |out1df = pd.DataFrame(_texera_rs_reservoir, columns=list(in1df.columns)).reset_index(drop=True)""".stripMargin + | _texera_rs_reservoir[_texera_rs_i] = _texera_rs_n + |out1df = in1df.iloc[_texera_rs_reservoir].reset_index(drop=True)""".stripMargin } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sortPartitions/SortPartitionsOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sortPartitions/SortPartitionsOpDesc.scala index b50f269a7cf..861e0e6fe69 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sortPartitions/SortPartitionsOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sortPartitions/SortPartitionsOpDesc.scala @@ -93,8 +93,10 @@ class SortPartitionsOpDesc extends LogicalOp with StandaloneCodeGenerator { // stable, ascending, nulls first. The domain bounds are partitioning hints // that the sort itself never reads. // - // NaN is where the two part. The engine orders it after positive infinity; - // pandas treats it as missing and puts it where the nulls go, which is first. + // The engine orders a NaN after positive infinity. In a nullable column, + // which is what an Arrow file is read into, pandas keeps a NaN apart from a + // null and sorts it last too. A numpy float column has one slot for both, so + // there a NaN goes first with the nulls. // // A string column parts more narrowly: the engine reads UTF-16 code units and // pandas reads code points, which agree below U+FFFF and can differ above it. diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDescSpec.scala index cd7cbd74e5b..e046c1d569f 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDescSpec.scala @@ -24,6 +24,8 @@ import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, Workflow import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorMetadataGenerator} import com.typesafe.config.ConfigFactory +import org.apache.texera.amber.operator.tags.IntegrationTest +import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -36,6 +38,8 @@ import scala.util.Try class AggregateOpDescSpec extends AnyFlatSpec with Matchers { + private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) + private val workflowId = WorkflowIdentity(1L) private val executionId = ExecutionIdentity(1L) @@ -117,7 +121,7 @@ class AggregateOpDescSpec extends AnyFlatSpec with Matchers { // The integers behind a timestamp column mean microseconds or nanoseconds // depending on the resolution it was read at, and a nanosecond total leaves // the range of a 64-bit integer after a handful of modern dates. - it should "read a timestamp as epoch milliseconds at either resolution" in { + it should "read a timestamp as epoch milliseconds at either resolution" taggedAs NeedsPythonPackages in { val python = resolvePython().getOrElse(cancel("No runnable python executable")) if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") @@ -163,7 +167,7 @@ class AggregateOpDescSpec extends AnyFlatSpec with Matchers { * it added. Checked against Java 17 in the same zone: the sum below is * 2078-01-01 06:00:00 and the average 1704132000000. */ - it should "add timestamps in the zone the script runs in" in { + it should "add timestamps in the zone the script runs in" taggedAs NeedsPythonPackages in { val python = resolvePython().getOrElse(cancel("No runnable python executable")) if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") @@ -213,7 +217,7 @@ class AggregateOpDescSpec extends AnyFlatSpec with Matchers { // A group keyed on a missing value and one keyed on a NaN are two groups to // the engine. The two are only distinct in a nullable dtype, which is what an // Arrow file is read into. - it should "group a missing key apart from a NaN one" in { + it should "group a missing key apart from a NaN one" taggedAs NeedsPythonPackages in { val python = resolvePython().getOrElse(cancel("No runnable python executable")) if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/difference/DifferenceOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/difference/DifferenceOpDescSpec.scala index 98824ba6a1e..ef7af3113c6 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/difference/DifferenceOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/difference/DifferenceOpDescSpec.scala @@ -30,6 +30,8 @@ import org.apache.texera.amber.core.workflow.{ UnknownPartition } import org.apache.texera.amber.operator.metadata.OperatorGroupConstants +import org.apache.texera.amber.operator.tags.IntegrationTest +import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -41,6 +43,8 @@ import scala.util.Try class DifferenceOpDescSpec extends AnyFlatSpec with Matchers { + private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) + private val workflowId = WorkflowIdentity(1L) private val executionId = ExecutionIdentity(1L) @@ -153,7 +157,7 @@ class DifferenceOpDescSpec extends AnyFlatSpec with Matchers { // NaN row. The two are only distinct in a nullable dtype, which is what an // Arrow file is read into. "DifferenceOpDesc.generateStandaloneCode" should - "keep the NaN row that a null row does not cancel" in { + "keep the NaN row that a null row does not cancel" taggedAs NeedsPythonPackages in { val python = resolvePython().getOrElse(cancel("No runnable python executable")) if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDescSpec.scala index 5485507c013..1efef3e96f1 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDescSpec.scala @@ -24,7 +24,9 @@ import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema} import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.OperatorGroupConstants +import org.apache.texera.amber.operator.tags.IntegrationTest import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -36,6 +38,8 @@ import scala.util.Try class HashJoinOpDescSpec extends AnyFlatSpec with Matchers { + private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) + private def leftRight(): (Schema, Schema) = ( Schema() @@ -85,7 +89,7 @@ class HashJoinOpDescSpec extends AnyFlatSpec with Matchers { // A key that is missing and a key holding a NaN are two different keys to the // engine, so only the missing one matches a missing right key. The two are // only distinct in a nullable dtype, which is what an Arrow file is read into. - "HashJoinOpDesc.generateStandaloneCode" should "match a missing key but not a NaN one" in { + "HashJoinOpDesc.generateStandaloneCode" should "match a missing key but not a NaN one" taggedAs NeedsPythonPackages in { val python = resolvePython().getOrElse(cancel("No runnable python executable")) if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/ifStatement/IfOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/ifStatement/IfOpDescSpec.scala index 0709ed450e5..03e879caf06 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/ifStatement/IfOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/ifStatement/IfOpDescSpec.scala @@ -25,6 +25,8 @@ import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.operator.metadata.OperatorGroupConstants +import org.apache.texera.amber.operator.tags.IntegrationTest +import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -36,6 +38,8 @@ import scala.util.Try class IfOpDescSpec extends AnyFlatSpec with Matchers { + private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) + private val workflowId = WorkflowIdentity(1L) private val executionId = ExecutionIdentity(1L) @@ -93,7 +97,7 @@ class IfOpDescSpec extends AnyFlatSpec with Matchers { // The engine picks the route from a State message, which the verification // harness has no channel for, so False is only reachable here. - it should "send the rows to True by default and to False when the switch is off" in { + it should "send the rows to True by default and to False when the switch is off" taggedAs NeedsPythonPackages in { val python = resolvePythonExecutable().getOrElse(cancel("No runnable python executable")) if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") @@ -103,6 +107,7 @@ class IfOpDescSpec extends AnyFlatSpec with Matchers { val driver = s"""import pandas as pd + |import sys | |for _switch in (None, True, False): | in2df = pd.DataFrame({"id": [1, 2, 3]}) @@ -129,6 +134,9 @@ class IfOpDescSpec extends AnyFlatSpec with Matchers { lines should contain("None [] [1, 2, 3]") lines should contain("True [] [1, 2, 3]") lines should contain("False [1, 2, 3] []") + // Only the unset switch takes True without being told to, so it alone + // says so. + lines.count(_.contains("_texera_if_ready = False before this step")) shouldBe 1 } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/intersect/IntersectOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/intersect/IntersectOpDescSpec.scala index d9be82e0cdb..3280eff83be 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/intersect/IntersectOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/intersect/IntersectOpDescSpec.scala @@ -19,15 +19,26 @@ package org.apache.texera.amber.operator.intersect +import com.typesafe.config.ConfigFactory import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.{HashPartition, SinglePartition, UnknownPartition} import org.apache.texera.amber.operator.metadata.OperatorGroupConstants +import org.apache.texera.amber.operator.tags.IntegrationTest +import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.util.concurrent.TimeUnit +import scala.io.Source +import scala.util.Try + class IntersectOpDescSpec extends AnyFlatSpec with Matchers { + private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) + private val workflowId = WorkflowIdentity(1L) private val executionId = ExecutionIdentity(1L) @@ -79,4 +90,64 @@ class IntersectOpDescSpec extends AnyFlatSpec with Matchers { fail(s"expected OpExecWithClassName, got $other") } } + + // A missing value and a stored NaN are two values to the engine's HashSet, so + // a left holding both against a right holding only the missing one keeps the + // missing row alone. They are only distinct in a nullable dtype, which is what + // an Arrow file is read into. + "IntersectOpDesc.generateStandaloneCode" should + "keep the null row a null row matches and not the NaN row" taggedAs NeedsPythonPackages in { + val python = resolvePython().getOrElse(cancel("No runnable python executable")) + if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + + val driver = + s"""import pandas as pd, numpy as np + | + |def col(vals, mask): + | return pd.arrays.FloatingArray(np.array(vals), np.array(mask)) + | + |in1df = pd.DataFrame({"v": col([0.0, np.nan], [True, False])}) + |in2df = pd.DataFrame({"v": col([0.0], [True])}) + |${(new IntersectOpDesc).generateStandaloneCode()} + |print(len(out1df), [("NULL" if x is pd.NA else repr(float(x))) for x in out1df["v"]]) + |""".stripMargin + + val script = Files.createTempFile("intersect-null-nan-", ".py") + script.toFile.deleteOnExit() + Files.write(script, driver.getBytes(StandardCharsets.UTF_8)) + val process = new ProcessBuilder(python, script.toString).redirectErrorStream(true).start() + val out = Source.fromInputStream(process.getInputStream).mkString + process.waitFor(120, TimeUnit.SECONDS) + + withClue(s"python said:\n$out\nscript:\n$driver") { + process.exitValue() shouldBe 0 + out.trim shouldBe "1 ['NULL']" + } + } + + private def resolvePython(): Option[String] = { + def fromConfig: Option[String] = + Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption + .orElse(Try(ConfigFactory.load()).toOption) + .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) + .map(_.trim) + .filter(_.nonEmpty) + + def runnable(exe: String): Boolean = + Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption + .exists { p => + if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false } + else p.exitValue() == 0 + } + + (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(runnable) + } + + private def canImportPandas(python: String): Boolean = + Try( + new ProcessBuilder(python, "-c", "import pandas, numpy").redirectErrorStream(true).start() + ).toOption.exists { p => + if (!p.waitFor(60, TimeUnit.SECONDS)) { p.destroyForcibly(); false } + else p.exitValue() == 0 + } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDescSpec.scala index cba15368c5e..6237a3cf0f3 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDescSpec.scala @@ -26,7 +26,9 @@ import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, Workflow import org.apache.texera.amber.core.workflow.{HashPartition, PortIdentity} import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.OperatorGroupConstants +import org.apache.texera.amber.operator.tags.IntegrationTest import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -38,6 +40,8 @@ import scala.util.Try class IntervalJoinOpDescSpec extends AnyFlatSpec with Matchers { + private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) + private val workflowId = WorkflowIdentity(1L) private val executionId = ExecutionIdentity(1L) @@ -125,7 +129,7 @@ class IntervalJoinOpDescSpec extends AnyFlatSpec with Matchers { * overwritten and then dropped. Run the generated pandas and hold its * columns against the schema the operator promises for the same two inputs. */ - it should "keep an input column that carries a temporary's name" in { + it should "keep an input column that carries a temporary's name" taggedAs NeedsPythonPackages in { val python = resolvePython().getOrElse( cancel("No runnable python executable (udf.conf python.path, python3, python, py)") ) diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/reservoirsampling/ReservoirSamplingOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/reservoirsampling/ReservoirSamplingOpDescSpec.scala index 52b43300ff1..0e806034f64 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/reservoirsampling/ReservoirSamplingOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/reservoirsampling/ReservoirSamplingOpDescSpec.scala @@ -22,9 +22,11 @@ package org.apache.texera.amber.operator.reservoirsampling import com.typesafe.config.ConfigFactory import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} +import org.apache.texera.amber.operator.tags.IntegrationTest import org.apache.texera.amber.operator.{LogicalOp, SamplingHelpers} import org.apache.texera.amber.operator.metadata.OperatorGroupConstants import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -36,6 +38,8 @@ import scala.util.Try class ReservoirSamplingOpDescSpec extends AnyFlatSpec with Matchers { + private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) + private val workflowId = WorkflowIdentity(1L) private val executionId = ExecutionIdentity(1L) @@ -77,21 +81,21 @@ class ReservoirSamplingOpDescSpec extends AnyFlatSpec with Matchers { """_texera_rs_rng = _TexeraJavaRandom(1) |_texera_rs_k = 3 |_texera_rs_reservoir = [] - |for _texera_rs_n, _texera_rs_row in enumerate(in1df.itertuples(index=False, name=None)): + |for _texera_rs_n in range(len(in1df)): | if _texera_rs_n < _texera_rs_k: - | _texera_rs_reservoir.append(_texera_rs_row) + | _texera_rs_reservoir.append(_texera_rs_n) | else: | _texera_rs_i = _texera_rs_rng.next_int(_texera_rs_n) | if _texera_rs_i < _texera_rs_k: - | _texera_rs_reservoir[_texera_rs_i] = _texera_rs_row - |out1df = pd.DataFrame(_texera_rs_reservoir, columns=list(in1df.columns)).reset_index(drop=True)""".stripMargin + | _texera_rs_reservoir[_texera_rs_i] = _texera_rs_n + |out1df = in1df.iloc[_texera_rs_reservoir].reset_index(drop=True)""".stripMargin } // A reservoir of zero ends the engine's run on the first row: the executor // skips the fill branch and hands nextInt a bound of zero, which Java refuses. // The script has to refuse it there too. Answering with an empty table would // report a result the run never produced. - it should "fail on the first row when the reservoir holds nothing" in { + it should "fail on the first row when the reservoir holds nothing" taggedAs NeedsPythonPackages in { val python = resolvePython().getOrElse(cancel("No runnable python executable")) if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpDescSpec.scala index b5fa9a92885..c4be492e2bd 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpDescSpec.scala @@ -24,7 +24,9 @@ import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.OperatorGroupConstants +import org.apache.texera.amber.operator.tags.IntegrationTest import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -37,6 +39,8 @@ import scala.util.Try class StableMergeSortOpDescSpec extends AnyFlatSpec with Matchers { + private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) + private val workflowId = WorkflowIdentity(1L) private val executionId = ExecutionIdentity(1L) @@ -80,7 +84,7 @@ class StableMergeSortOpDescSpec extends AnyFlatSpec with Matchers { // number, so it goes last ascending and first descending. Only a column read // into a nullable dtype can hold both, which is where this is run. "StableMergeSortOpDesc.generateStandaloneCode" should - "put a null last both ways and a NaN where the key points" in { + "put a null last both ways and a NaN where the key points" taggedAs NeedsPythonPackages in { val python = resolvePython().getOrElse(cancel("No runnable python executable")) if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") @@ -134,7 +138,7 @@ class StableMergeSortOpDescSpec extends AnyFlatSpec with Matchers { // The sort tiers are named after nothing the input can be carrying, so a // column that happens to answer to a helper's old name keeps its values. - it should "keep an input column named after a sort helper" in { + it should "keep an input column named after a sort helper" taggedAs NeedsPythonPackages in { val python = resolvePython().getOrElse(cancel("No runnable python executable")) if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sortPartitions/SortPartitionsOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sortPartitions/SortPartitionsOpDescSpec.scala index d9ae20b14a0..9d6a2bfdc5b 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sortPartitions/SortPartitionsOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sortPartitions/SortPartitionsOpDescSpec.scala @@ -19,17 +19,28 @@ package org.apache.texera.amber.operator.sortPartitions +import com.typesafe.config.ConfigFactory import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.RangePartition import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.OperatorGroupConstants +import org.apache.texera.amber.operator.tags.IntegrationTest import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.util.concurrent.TimeUnit +import scala.io.Source +import scala.util.Try + class SortPartitionsOpDescSpec extends AnyFlatSpec with Matchers { + private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) + private val workflowId = WorkflowIdentity(1L) private val executionId = ExecutionIdentity(1L) @@ -77,4 +88,63 @@ class SortPartitionsOpDescSpec extends AnyFlatSpec with Matchers { sp.domainMin shouldBe 1L sp.domainMax shouldBe 99L } + + // The engine puts a null first and a NaN after positive infinity. Only a + // column read into a nullable dtype can hold both, which is what an Arrow + // file is read into, so that is where this is run. + "SortPartitionsOpDesc.generateStandaloneCode" should + "put a null first and a NaN after infinity" taggedAs NeedsPythonPackages in { + val python = resolvePython().getOrElse(cancel("No runnable python executable")) + if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + + val driver = + s"""import pandas as pd, numpy as np + | + |# 3.0, a null, a stored NaN, 1.0, +inf + |in1df = pd.DataFrame({"v": pd.arrays.FloatingArray( + | np.array([3.0, 0.0, np.nan, 1.0, np.inf]), + | np.array([False, True, False, False, False]), + |)}) + |${newDesc("v", 0L, 10L).generateStandaloneCode()} + |print([("NULL" if x is pd.NA else repr(float(x))) for x in out1df["v"]]) + |""".stripMargin + + val script = Files.createTempFile("sort-partitions-nan-", ".py") + script.toFile.deleteOnExit() + Files.write(script, driver.getBytes(StandardCharsets.UTF_8)) + val process = new ProcessBuilder(python, script.toString).redirectErrorStream(true).start() + val out = Source.fromInputStream(process.getInputStream).mkString + process.waitFor(120, TimeUnit.SECONDS) + + withClue(s"python said:\n$out\nscript:\n$driver") { + process.exitValue() shouldBe 0 + out.trim shouldBe "['NULL', '1.0', '3.0', 'inf', 'nan']" + } + } + + private def resolvePython(): Option[String] = { + def fromConfig: Option[String] = + Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption + .orElse(Try(ConfigFactory.load()).toOption) + .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) + .map(_.trim) + .filter(_.nonEmpty) + + def runnable(exe: String): Boolean = + Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption + .exists { p => + if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false } + else p.exitValue() == 0 + } + + (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(runnable) + } + + private def canImportPandas(python: String): Boolean = + Try( + new ProcessBuilder(python, "-c", "import pandas, numpy").redirectErrorStream(true).start() + ).toOption.exists { p => + if (!p.waitFor(60, TimeUnit.SECONDS)) { p.destroyForcibly(); false } + else p.exitValue() == 0 + } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/symmetricDifference/SymmetricDifferenceOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/symmetricDifference/SymmetricDifferenceOpDescSpec.scala index e6e2995877e..866e08c6ae1 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/symmetricDifference/SymmetricDifferenceOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/symmetricDifference/SymmetricDifferenceOpDescSpec.scala @@ -19,18 +19,29 @@ package org.apache.texera.amber.operator.symmetricDifference +import com.typesafe.config.ConfigFactory import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.{HashPartition, PortIdentity} import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.OperatorGroupConstants +import org.apache.texera.amber.operator.tags.IntegrationTest import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.util.concurrent.TimeUnit +import scala.io.Source +import scala.util.Try + class SymmetricDifferenceOpDescSpec extends AnyFlatSpec with Matchers { + private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) + private val workflowId = WorkflowIdentity(1L) private val executionId = ExecutionIdentity(1L) @@ -86,4 +97,64 @@ class SymmetricDifferenceOpDescSpec extends AnyFlatSpec with Matchers { val json = objectMapper.writeValueAsString(new SymmetricDifferenceOpDesc) objectMapper.readValue(json, classOf[LogicalOp]) shouldBe a[SymmetricDifferenceOpDesc] } + + // A missing value and a stored NaN are two values to the engine's HashSet, so + // a left holding both against a right holding only the missing one leaves the + // NaN row alone in exactly one input. They are only distinct in a nullable + // dtype, which is what an Arrow file is read into. + "SymmetricDifferenceOpDesc.generateStandaloneCode" should + "keep the NaN row that a null row does not cancel" taggedAs NeedsPythonPackages in { + val python = resolvePython().getOrElse(cancel("No runnable python executable")) + if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + + val driver = + s"""import pandas as pd, numpy as np + | + |def col(vals, mask): + | return pd.arrays.FloatingArray(np.array(vals), np.array(mask)) + | + |in1df = pd.DataFrame({"v": col([0.0, np.nan], [True, False])}) + |in2df = pd.DataFrame({"v": col([0.0], [True])}) + |${(new SymmetricDifferenceOpDesc).generateStandaloneCode()} + |print(len(out1df), [("NULL" if x is pd.NA else repr(float(x))) for x in out1df["v"]]) + |""".stripMargin + + val script = Files.createTempFile("symmetric-difference-null-nan-", ".py") + script.toFile.deleteOnExit() + Files.write(script, driver.getBytes(StandardCharsets.UTF_8)) + val process = new ProcessBuilder(python, script.toString).redirectErrorStream(true).start() + val out = Source.fromInputStream(process.getInputStream).mkString + process.waitFor(120, TimeUnit.SECONDS) + + withClue(s"python said:\n$out\nscript:\n$driver") { + process.exitValue() shouldBe 0 + out.trim shouldBe "1 ['nan']" + } + } + + private def resolvePython(): Option[String] = { + def fromConfig: Option[String] = + Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption + .orElse(Try(ConfigFactory.load()).toOption) + .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) + .map(_.trim) + .filter(_.nonEmpty) + + def runnable(exe: String): Boolean = + Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption + .exists { p => + if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false } + else p.exitValue() == 0 + } + + (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(runnable) + } + + private def canImportPandas(python: String): Boolean = + Try( + new ProcessBuilder(python, "-c", "import pandas, numpy").redirectErrorStream(true).start() + ).toOption.exists { p => + if (!p.waitFor(60, TimeUnit.SECONDS)) { p.destroyForcibly(); false } + else p.exitValue() == 0 + } } From e831318abafd4c8ae6121c3da961127b9ee67596 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Sat, 26 Sep 2026 20:31:24 -0700 Subject: [PATCH 19/32] refactor(workflow-operator): leave Aggregate, the sort family, the samplers and control flow to the two PRs that follow Co-Authored-By: Claude Opus 5.5 (1M context) --- .../texera/amber/operator/LogicalOp.scala | 12 -- .../amber/operator/SamplingHelpers.scala | 68 ------ .../operator/aggregate/AggregateOpDesc.scala | 200 +----------------- .../amber/operator/dummy/DummyOpDesc.scala | 17 +- .../amber/operator/ifStatement/IfOpDesc.scala | 34 +-- .../RandomKSamplingOpDesc.scala | 22 +- .../ReservoirSamplingOpDesc.scala | 32 +-- .../amber/operator/sleep/SleepOpDesc.scala | 10 +- .../operator/sort/SortCriteriaUnit.scala | 6 +- .../amber/operator/sort/SortOpDesc.scala | 20 +- .../operator/sort/StableMergeSortOpDesc.scala | 55 +---- .../sortPartitions/SortPartitionsOpDesc.scala | 25 +-- .../amber/operator/split/SplitOpDesc.scala | 25 +-- .../amber/operator/SamplingHelpersSpec.scala | 117 ---------- .../aggregate/AggregateOpDescSpec.scala | 172 --------------- .../operator/dummy/DummyOpDescSpec.scala | 5 - .../operator/ifStatement/IfOpDescSpec.scala | 89 -------- .../RandomKSamplingOpDescSpec.scala | 24 --- .../ReservoirSamplingOpDescSpec.scala | 89 +------- .../operator/sleep/SleepOpExecSpec.scala | 17 -- .../sort/StableMergeSortOpDescSpec.scala | 134 ------------ .../SortPartitionsOpDescSpec.scala | 70 ------ .../operator/split/SplitOpDescSpec.scala | 38 ---- 23 files changed, 23 insertions(+), 1258 deletions(-) delete mode 100644 common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/SamplingHelpers.scala delete mode 100644 common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/SamplingHelpersSpec.scala diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/LogicalOp.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/LogicalOp.scala index 5e7be51d003..efa46144180 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/LogicalOp.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/LogicalOp.scala @@ -459,18 +459,6 @@ abstract class LogicalOp extends PortDescriptor with Serializable { def operatorInfo: OperatorInfo - /** - * Whether the row ORDER of this operator's output is part of its contract. - * Defaults to false: the engine runs operators across parallel workers, so - * for almost every operator the output row order is an implementation-defined - * interleaving. Only operators whose very purpose is to establish an order - * override this, which here is the sort family: Sort, Stable Merge Sort and - * Sort Partitions. Anything comparing two runs of an operator reads it to - * decide whether the rows have to arrive in the same order or only be the - * same rows. - */ - def orderSensitive: Boolean = false - private def getOperatorVersion: String = { val path = "amber/src/main/scala/" val operatorPath = path + this.getClass.getPackage.getName.replace(".", "/") diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/SamplingHelpers.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/SamplingHelpers.scala deleted file mode 100644 index 1fa9210898b..00000000000 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/SamplingHelpers.scala +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.texera.amber.operator - -/** The Python a sampler's standalone code needs, emitted once per script via - * [[StandaloneCodeGenerator.standaloneHelpers]]. - */ -object SamplingHelpers { - - /** - * A Python transcription of `java.util.Random`, for operators whose executor - * draws from one. - * - * A sampler decides per row whether to keep it, so which rows survive is - * fixed by the exact sequence the generator produces. Seeding Python's - * `random` or numpy's with the engine's seed selects a different set, and - * the script would then report a different sample than the workflow it came - * from. Only the same generator gives the same rows. - */ - val JavaRandom: String = - """# java.util.Random, transcribed so sampling matches the engine. - |class _TexeraJavaRandom: - | _MASK = (1 << 48) - 1 - | _MULTIPLIER = 0x5DEECE66D - | _ADDEND = 0xB - | - | def __init__(self, seed): - | self._seed = (seed ^ self._MULTIPLIER) & self._MASK - | - | def _next(self, bits): - | self._seed = (self._seed * self._MULTIPLIER + self._ADDEND) & self._MASK - | value = self._seed >> (48 - bits) - | return value - (1 << 32) if value >= (1 << 31) else value - | - | def next_double(self): - | return ((self._next(26) << 27) + self._next(27)) * (2.0 ** -53) - | - | def next_int(self, bound): - | if bound <= 0: - | raise ValueError("bound must be positive") - | if bound & (-bound) == bound: - | return (bound * self._next(31)) >> 31 - | while True: - | bits = self._next(31) - | value = bits % bound - | # Java rejects a draw by letting this sum overflow an int. - | # Python would carry it and never reject. - | probe = bits - value + (bound - 1) - | if ((probe + (1 << 31)) % (1 << 32)) - (1 << 31) >= 0: - | return value""".stripMargin -} diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDesc.scala index 4cd2b8cb499..1bace652c8b 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDesc.scala @@ -22,24 +22,21 @@ package org.apache.texera.amber.operator.aggregate import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle import org.apache.texera.amber.core.executor.OpExecWithClassName -import org.apache.texera.amber.core.tuple.{AttributeType, Schema} +import org.apache.texera.amber.core.tuple.Schema import org.apache.texera.amber.core.virtualidentity.{ ExecutionIdentity, PhysicalOpIdentity, WorkflowIdentity } import org.apache.texera.amber.core.workflow._ -import org.apache.texera.amber.operator.{LogicalOp, StandaloneCodeGenerator, StandaloneHelpers} +import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeNameList import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} -import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral import org.apache.texera.amber.util.JSONUtils.objectMapper import javax.validation.constraints.{NotNull, Size} -import scala.util.Try - -class AggregateOpDesc extends LogicalOp with StandaloneCodeGenerator { +class AggregateOpDesc extends LogicalOp { @JsonProperty(value = "aggregations", required = true) @JsonPropertyDescription("multiple aggregation functions") @NotNull(message = "aggregation cannot be null") @@ -141,195 +138,4 @@ class AggregateOpDesc extends LogicalOp with StandaloneCodeGenerator { inputPorts = List(InputPort()), outputPorts = List(OutputPort()) ) - - /** The engine aggregates in two phases across partitions; one process needs - * only the one groupby, or a single-row reduction when no key is grouped on. - * - * Must run before `getPhysicalPlan`, which rewrites `aggregations` in place: - * it turns COUNT into SUM for the final phase, and this reads them as - * written. - * - * SUM and AVERAGE follow the column's DECLARED type: a holed INTEGER column - * arrives as a float, and an INTEGER and a LONG arrive alike. - */ - override def generateStandaloneCode(inputSchemas: Map[PortIdentity, Schema]): String = { - val schema = inputSchemas.get(operatorInfo.inputPorts.head.id) - build(name => schema.flatMap(s => Try(s.getAttribute(name).getType).toOption)) - } - - override def generateStandaloneCode(): String = build(_ => None) - - override def standaloneHelpers(): Seq[String] = Seq(StandaloneHelpers.AttributeCasts) - - private def build(declaredType: String => Option[AttributeType]): String = { - val keys = Option(groupByKeys).getOrElse(List()) - val aggs = Option(aggregations).getOrElse(List()) - - // Identical helper definition each call — keeps the standalone module - // self-contained without relying on a shared prelude. - val concatHelper = - """def _texera_agg_concat(series): - | # The accumulator starts empty and only earns a separator once it - | # holds something, so a leading empty value adds neither text nor - | # comma: "", "a", "" concatenates to "a," and not ",a,". A null is - | # read as the empty string, which is what makes the two the same - | # here. This is concatAgg's fold, written out. - | # Each value is written as Java's toString writes it. - | partial = "" - | for v in series: - | text = "" if pd.isna(v) else _texera_java_text(v) - | partial = text if partial == "" else partial + "," + text - | return partial - | - |def _texera_agg_int_sum(series): - | # The engine adds an INTEGER column as Java ints, which wrap. - | total = int(series.sum()) - | return ((total + (1 << 31)) % (1 << 32)) - (1 << 31) - | - |def _texera_agg_ts_zone(): - | # The engine reads and writes a timestamp in the JVM's default zone, - | # so the arithmetic below has to name the same one. gettz() and not - | # the current offset: the zone carries its daylight rules, and each - | # instant needs the offset in force when it happened. - | from dateutil.tz import gettz - | - | return gettz() - | - |def _texera_agg_ts_epoch_ms(series): - | # A timestamp reaches the engine as its epoch milliseconds whatever - | # resolution the column carries, and reading the integers out of a - | # microsecond column asks for a different number than a nanosecond - | # one, so cast to milliseconds before reading them. - | # - | # A column holds a wall clock, and `Timestamp.getTime` answers for - | # the instant that wall clock names locally, so localize before - | # reading the integers out: left as UTC every one of them is a whole - | # offset away. The two flags are java.time's own reading of an hour - | # daylight saving repeats (the later one) or skips (shifted past the - | # gap). - | return ( - | series.dropna() - | .astype("datetime64[ms]") - | .dt.tz_localize( - | _texera_agg_ts_zone(), ambiguous=False, nonexistent=pd.Timedelta("1h") - | ) - | .astype("int64") - | ) - | - |def _texera_agg_ts_sum(series): - | # SUM keeps the column's own type, so the engine adds the epoch - | # milliseconds as Java longs, which wrap, and builds a timestamp - | # from the total. Python integers carry the sum exactly, so the - | # wrap is the only place a total loses anything. - | total = int(_texera_agg_ts_epoch_ms(series).astype(object).sum()) - | total = ((total + (1 << 63)) % (1 << 64)) - (1 << 63) - | return ( - | pd.Timestamp(total, unit="ms", tz="UTC") - | .tz_convert(_texera_agg_ts_zone()) - | .tz_localize(None) - | ) - | - |def _texera_agg_ts_mean(series): - | # AVERAGE is declared DOUBLE whatever column it reads, so this is - | # the mean of the epoch milliseconds and not a timestamp. - | kept = _texera_agg_ts_epoch_ms(series) - | if len(kept) == 0: - | return None - | return float(kept.astype(object).sum()) / len(kept)""".stripMargin - - if (keys.isEmpty) { - val rowEntries = aggs - .map(agg => - s" ${pyStringLiteral(agg.resultAttribute)}: ${aggExprScalar(agg, declaredType)}," - ) - .mkString("\n") - s"""$concatHelper - |out1df = pd.DataFrame([{ - |$rowEntries - |}])""".stripMargin - } else { - val keysLit = keys.map(pyStringLiteral).mkString("[", ", ", "]") - val aggLines = aggs.zipWithIndex - .map { - case (agg, i) => - s"_texera_agg_s$i = ${aggExprGroupby(agg, "_texera_agg_groups", declaredType)}" - } - .mkString("\n") - val mergeLines = aggs.indices - .map(i => - s"""out1df = out1df.merge(_texera_agg_s$i.reset_index(), on=$keysLit, how="left")""" - ) - .mkString("\n") - s"""$concatHelper - |_texera_agg_groups = in1df.groupby($keysLit, dropna=False, sort=False) - |out1df = in1df[$keysLit].drop_duplicates().reset_index(drop=True) - |$aggLines - |$mergeLines""".stripMargin - } - } - - private def aggExprScalar( - agg: AggregationOperation, - declaredType: String => Option[AttributeType] - ): String = { - val attrLit = - if (agg.attribute == null || agg.attribute.isEmpty) "None" - else pyStringLiteral(agg.attribute) - val declared = Option(agg.attribute).filter(_.nonEmpty).flatMap(declaredType) - agg.aggFunction match { - case AggregationFunction.SUM => - declared match { - case Some(AttributeType.INTEGER) => s"_texera_agg_int_sum(in1df[$attrLit])" - case Some(AttributeType.TIMESTAMP) => s"_texera_agg_ts_sum(in1df[$attrLit])" - case _ => s"in1df[$attrLit].sum()" - } - case AggregationFunction.AVERAGE => - declared match { - case Some(AttributeType.TIMESTAMP) => s"_texera_agg_ts_mean(in1df[$attrLit])" - case _ => s"in1df[$attrLit].mean()" - } - case AggregationFunction.MIN => s"in1df[$attrLit].min()" - case AggregationFunction.MAX => s"in1df[$attrLit].max()" - case AggregationFunction.COUNT => - if (agg.attribute == null || agg.attribute.isEmpty) "int(len(in1df))" - else s"int(in1df[$attrLit].count())" - case AggregationFunction.CONCAT => s"_texera_agg_concat(in1df[$attrLit])" - } - } - - private def aggExprGroupby( - agg: AggregationOperation, - groups: String, - declaredType: String => Option[AttributeType] - ): String = { - val attrLit = - if (agg.attribute == null || agg.attribute.isEmpty) "None" - else pyStringLiteral(agg.attribute) - val resultLit = pyStringLiteral(agg.resultAttribute) - val declared = Option(agg.attribute).filter(_.nonEmpty).flatMap(declaredType) - agg.aggFunction match { - case AggregationFunction.SUM => - declared match { - case Some(AttributeType.INTEGER) => - s"$groups[$attrLit].apply(_texera_agg_int_sum).rename($resultLit)" - case Some(AttributeType.TIMESTAMP) => - s"$groups[$attrLit].apply(_texera_agg_ts_sum).rename($resultLit)" - case _ => s"$groups[$attrLit].sum().rename($resultLit)" - } - case AggregationFunction.AVERAGE => - declared match { - case Some(AttributeType.TIMESTAMP) => - s"$groups[$attrLit].apply(_texera_agg_ts_mean).rename($resultLit)" - case _ => s"$groups[$attrLit].mean().rename($resultLit)" - } - case AggregationFunction.MIN => s"$groups[$attrLit].min().rename($resultLit)" - case AggregationFunction.MAX => s"$groups[$attrLit].max().rename($resultLit)" - case AggregationFunction.COUNT => - if (agg.attribute == null || agg.attribute.isEmpty) - s"$groups.size().rename($resultLit)" - else s"$groups[$attrLit].count().rename($resultLit)" - case AggregationFunction.CONCAT => - s"$groups[$attrLit].apply(_texera_agg_concat).rename($resultLit)" - } - } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/dummy/DummyOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/dummy/DummyOpDesc.scala index bee942cd370..4a2c4c48d54 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/dummy/DummyOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/dummy/DummyOpDesc.scala @@ -23,14 +23,9 @@ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle import org.apache.texera.amber.core.workflow.{InputPort, OutputPort, PortIdentity} import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} -import org.apache.texera.amber.operator.{ - LogicalOp, - PortDescription, - PortDescriptor, - StandaloneCodeGenerator -} +import org.apache.texera.amber.operator.{LogicalOp, PortDescription, PortDescriptor} -class DummyOpDesc extends LogicalOp with PortDescriptor with StandaloneCodeGenerator { +class DummyOpDesc extends LogicalOp with PortDescriptor { @JsonProperty @JsonSchemaTitle("Description") @@ -71,12 +66,4 @@ class DummyOpDesc extends LogicalOp with PortDescriptor with StandaloneCodeGener allowPortCustomization = true ) } - - override def generateStandaloneCode(): String = { - // Placeholder operator: pass the first input through to the output. - // Multi-port configurations don't fully translate under the current - // single-output placeholder scheme; downstream of extra ports would - // alias the same variable. - "out1df = in1df" - } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/ifStatement/IfOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/ifStatement/IfOpDesc.scala index 296e70e6928..f811ed34296 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/ifStatement/IfOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/ifStatement/IfOpDesc.scala @@ -24,12 +24,11 @@ import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow._ -import org.apache.texera.amber.operator.{LogicalOp, StandaloneCodeGenerator} +import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} -import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral import org.apache.texera.amber.util.JSONUtils.objectMapper -class IfOpDesc extends LogicalOp with StandaloneCodeGenerator { +class IfOpDesc extends LogicalOp { @JsonProperty(required = true) @JsonSchemaTitle("Condition State") @JsonPropertyDescription("name of the state variable to evaluate") @@ -73,33 +72,4 @@ class IfOpDesc extends LogicalOp with StandaloneCodeGenerator { ), outputPorts = List(OutputPort(PortIdentity(), "False"), OutputPort(PortIdentity(1), "True")) ) - - // The engine flips the active output port when a State message arrives on the - // Condition port. A script has no State channel, so the condition is read from - // a global named after it, true when nothing set it, which is the port the - // engine starts on. The Condition input carries no rows to read either way. - // - // Taking True silently would report the True route for a workflow whose - // condition says otherwise, so an unset switch says which route it took and - // how to take the other. - override def standaloneImports(): Seq[String] = Seq("import sys") - - override def generateStandaloneCode(): String = { - val globalName = "_texera_if_" + Option(conditionName).getOrElse("") - val globalLit = pyStringLiteral(globalName) - s"""if $globalLit not in globals(): - | print( - | "If: the script cannot receive the condition the workflow sets upstream, " - | "so it takes the True branch. To take the False branch, set " - | + $globalLit + " = False before this step.", - | file=sys.stderr, - | ) - |_texera_if_cond = bool(globals().get($globalLit, True)) - |if _texera_if_cond: - | out2df = in2df.copy() - | out1df = in2df.iloc[0:0].copy() - |else: - | out1df = in2df.copy() - | out2df = in2df.iloc[0:0].copy()""".stripMargin - } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/randomksampling/RandomKSamplingOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/randomksampling/RandomKSamplingOpDesc.scala index 188d3e38ef4..31cafcbc76b 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/randomksampling/RandomKSamplingOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/randomksampling/RandomKSamplingOpDesc.scala @@ -23,12 +23,11 @@ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.{InputPort, OutputPort, PhysicalOp} -import org.apache.texera.amber.operator.{SamplingHelpers, StandaloneCodeGenerator} import org.apache.texera.amber.operator.filter.FilterOpDesc import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.util.JSONUtils.objectMapper -class RandomKSamplingOpDesc extends FilterOpDesc with StandaloneCodeGenerator { +class RandomKSamplingOpDesc extends FilterOpDesc { @JsonProperty(value = "random k sample percentage", required = true) @JsonPropertyDescription("random k sampling with given percentage") @@ -61,23 +60,4 @@ class RandomKSamplingOpDesc extends FilterOpDesc with StandaloneCodeGenerator { outputPorts = List(OutputPort()), supportReconfiguration = true ) - - override def standaloneHelpers(): Seq[String] = Seq(SamplingHelpers.JavaRandom) - - // A per-row Bernoulli filter drawn from the same generator, so a rerun keeps - // the exact rows rather than merely the same proportion of them. - // - // The engine seeds it with the worker count, which makes the surviving rows a - // property of the deployment. One process states the one seed it can and so - // agrees with a single-worker run; a wider one splits the input and samples - // each share from the start of the sequence, which no seed here reproduces. - override def generateStandaloneCode(): String = { - val p = percentage / 100.0 - s"""_texera_rks_rng = _TexeraJavaRandom(1) - |_texera_rks_mask = pd.Series( - | [$p >= _texera_rks_rng.next_double() for _ in range(len(in1df))], - | index=in1df.index, dtype=bool - |) - |out1df = in1df[_texera_rks_mask].reset_index(drop=True)""".stripMargin - } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/reservoirsampling/ReservoirSamplingOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/reservoirsampling/ReservoirSamplingOpDesc.scala index a20112f84b6..eda2aa299fe 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/reservoirsampling/ReservoirSamplingOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/reservoirsampling/ReservoirSamplingOpDesc.scala @@ -23,11 +23,11 @@ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.{InputPort, OutputPort, PhysicalOp} -import org.apache.texera.amber.operator.{LogicalOp, SamplingHelpers, StandaloneCodeGenerator} +import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.util.JSONUtils.objectMapper -class ReservoirSamplingOpDesc extends LogicalOp with StandaloneCodeGenerator { +class ReservoirSamplingOpDesc extends LogicalOp { @JsonProperty(value = "number of item sampled in reservoir sampling", required = true) @JsonPropertyDescription("reservoir sampling with k items being kept randomly") @@ -60,32 +60,4 @@ class ReservoirSamplingOpDesc extends LogicalOp with StandaloneCodeGenerator { outputPorts = List(OutputPort()) ) } - - override def standaloneHelpers(): Seq[String] = Seq(SamplingHelpers.JavaRandom) - - // The executor runs Algorithm R (Vitter): fill the reservoir with the first k - // tuples, then for tuple m+1 (m >= k) draw i = rand.nextInt(m), uniform in - // [0, m), and replace reservoir[i] iff i < k. - // - // Its generator is seeded with the worker count, so the rows agree with a - // single-worker run and not with a wider one. See RandomKSamplingOpDesc for - // why no seed closes that gap. - // - // The reservoir holds row positions, and the rows are taken from the input - // by them, so every column keeps its dtype. A frame rebuilt from the rows' - // values infers each one again, and a timestamp past pandas' nanosecond - // range came back as an object column. - override def generateStandaloneCode(): String = { - s"""_texera_rs_rng = _TexeraJavaRandom(1) - |_texera_rs_k = $k - |_texera_rs_reservoir = [] - |for _texera_rs_n in range(len(in1df)): - | if _texera_rs_n < _texera_rs_k: - | _texera_rs_reservoir.append(_texera_rs_n) - | else: - | _texera_rs_i = _texera_rs_rng.next_int(_texera_rs_n) - | if _texera_rs_i < _texera_rs_k: - | _texera_rs_reservoir[_texera_rs_i] = _texera_rs_n - |out1df = in1df.iloc[_texera_rs_reservoir].reset_index(drop=True)""".stripMargin - } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sleep/SleepOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sleep/SleepOpDesc.scala index 1986eb52d83..3eee3cd33b7 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sleep/SleepOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sleep/SleepOpDesc.scala @@ -24,11 +24,11 @@ import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.{InputPort, OutputPort, PhysicalOp} -import org.apache.texera.amber.operator.{LogicalOp, StandaloneCodeGenerator} +import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.util.JSONUtils.objectMapper -class SleepOpDesc extends LogicalOp with StandaloneCodeGenerator { +class SleepOpDesc extends LogicalOp { @JsonProperty(required = true) @JsonSchemaTitle("Sleep Time (seconds)") @@ -63,10 +63,4 @@ class SleepOpDesc extends LogicalOp with StandaloneCodeGenerator { inputPorts = List(InputPort()), outputPorts = List(OutputPort()) ) - - override def generateStandaloneCode(): String = { - // JVM op sleeps between each tuple; pandas operates in batch, not row-by-row, - // so per-row sleep has no equivalent. Translate as a passthrough. - "out1df = in1df" - } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sort/SortCriteriaUnit.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sort/SortCriteriaUnit.scala index 9b34fe44b1a..f0e851110b8 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sort/SortCriteriaUnit.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sort/SortCriteriaUnit.scala @@ -21,17 +21,13 @@ package org.apache.texera.amber.operator.sort import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString -import org.apache.texera.amber.operator.metadata.annotations.{AutofillAttributeName, SampleColumn} +import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName class SortCriteriaUnit { - // Verification picks a column with TIE GROUPS. Sorting on a unique key never - // puts two equal keys next to each other, so the stability this operator is - // named for goes unchecked. @JsonProperty(value = "attribute", required = true) @JsonPropertyDescription("Attribute name to sort by") @AutofillAttributeName - @SampleColumn("name") var attributeName: EncodableString = _ @JsonProperty(value = "sortPreference", required = true) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sort/SortOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sort/SortOpDesc.scala index 662fd37377d..db2ae21e7f6 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sort/SortOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sort/SortOpDesc.scala @@ -21,18 +21,11 @@ package org.apache.texera.amber.operator.sort import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import org.apache.texera.amber.core.tuple.Schema -import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ - PythonTemplateBuilderStringContext, - pyStringLiteral -} +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext import org.apache.texera.amber.core.workflow.{InputPort, OutputPort, PortIdentity} -import org.apache.texera.amber.operator.{PythonOperatorDescriptor, StandaloneCodeGenerator} +import org.apache.texera.amber.operator.PythonOperatorDescriptor import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} -class SortOpDesc extends PythonOperatorDescriptor with StandaloneCodeGenerator { - - // Sorting is this operator's contract: output row order is meaningful. - override def orderSensitive: Boolean = true - +class SortOpDesc extends PythonOperatorDescriptor { @JsonProperty(required = true) @JsonPropertyDescription("column to perform sorting on") var attributes: List[SortCriteriaUnit] = List.empty @@ -85,11 +78,4 @@ class SortOpDesc extends PythonOperatorDescriptor with StandaloneCodeGenerator { outputPorts = List(OutputPort(blocking = true)) ) - override def generateStandaloneCode(): String = { - val cols = attributes.map(c => pyStringLiteral(c.attributeName)).mkString("[", ", ", "]") - val ascending = attributes - .map(c => if (c.sortPreference == SortPreference.ASC) "True" else "False") - .mkString("[", ", ", "]") - s"out1df = in1df.sort_values(by=$cols, ascending=$ascending)" - } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpDesc.scala index b48b5e0ee09..318089da5e3 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpDesc.scala @@ -24,9 +24,8 @@ import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.{InputPort, OutputPort, PhysicalOp} -import org.apache.texera.amber.operator.{LogicalOp, StandaloneCodeGenerator} +import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} -import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral import org.apache.texera.amber.util.JSONUtils.objectMapper import scala.collection.mutable.ListBuffer @@ -37,10 +36,7 @@ import scala.collection.mutable.ListBuffer * the lexicographic order and per-key direction (ASC/DESC). */ //TODO(#3922): disallowing sorting on binary type -class StableMergeSortOpDesc extends LogicalOp with StandaloneCodeGenerator { - - // Sorting is this operator's contract: output row order is meaningful. - override def orderSensitive: Boolean = true +class StableMergeSortOpDesc extends LogicalOp { @JsonProperty(value = "keys", required = true) @JsonSchemaTitle("Sort Keys") @@ -73,51 +69,4 @@ class StableMergeSortOpDesc extends LogicalOp with StandaloneCodeGenerator { List(InputPort()), List(OutputPort(blocking = true)) ) - - // The engine runs an incremental stable merge sort with nulls last whichever - // way a key points. pandas' mergesort is stable too, so the ordering below is - // the same one. - // - // A string column parts more narrowly: the engine reads UTF-16 code units and - // pandas reads code points, which agree below U+FFFF and can differ above it. - override def generateStandaloneCode(): String = { - val criteria = Option(keys).getOrElse(ListBuffer.empty) - if (criteria.isEmpty) return "out1df = in1df.copy()" - val cols = criteria - .map(c => pyStringLiteral(c.attributeName)) - .mkString("[", ", ", "]") - val ascending = criteria - .map(c => if (c.sortPreference == SortPreference.ASC) "True" else "False") - .mkString("[", ", ", "]") - // Sort each key in three tiers, because the engine treats a null and a NaN - // differently: a null goes last whichever way the key points, while a NaN - // compares above every number, so it goes last ascending and first - // descending. A column read into a numpy dtype has one slot for both, and - // there both land in the null tier, which is where they were before. - // - // The tiers are built in a frame of their own and named by position, so - // the input keeps every column it arrived with: a helper named after the - // key would overwrite an input column that already answers to that name, - // and dropping the helper afterwards would take the payload with it. - s"""_texera_sorted = in1df.reset_index(drop=True) - |_texera_keys = pd.DataFrame(index=_texera_sorted.index) - |_texera_by = [] - |_texera_asc = [] - |for _texera_i, (_texera_col, _texera_a) in enumerate(zip($cols, $ascending)): - | _texera_null = "null_" + str(_texera_i) - | _texera_nan = "nan_" + str(_texera_i) - | _texera_val = "val_" + str(_texera_i) - | _texera_keys[_texera_null] = _texera_sorted[_texera_col].isna() - | _texera_keys[_texera_nan] = ( - | _texera_sorted[_texera_col] != _texera_sorted[_texera_col] - | ).fillna(False) - | _texera_keys[_texera_val] = _texera_sorted[_texera_col] - | _texera_by += [_texera_null, _texera_nan, _texera_val] - | _texera_asc += [True, _texera_a, _texera_a] - |out1df = _texera_sorted.loc[ - | _texera_keys.sort_values( - | by=_texera_by, ascending=_texera_asc, kind="mergesort" - | ).index - |].reset_index(drop=True)""".stripMargin - } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sortPartitions/SortPartitionsOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sortPartitions/SortPartitionsOpDesc.scala index 861e0e6fe69..9290d549f2e 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sortPartitions/SortPartitionsOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sortPartitions/SortPartitionsOpDesc.scala @@ -24,10 +24,9 @@ import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchema import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.{InputPort, OutputPort, PhysicalOp, RangePartition} -import org.apache.texera.amber.operator.{LogicalOp, StandaloneCodeGenerator} +import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} -import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral import org.apache.texera.amber.util.JSONUtils.objectMapper @JsonSchemaInject(json = """ @@ -39,10 +38,7 @@ import org.apache.texera.amber.util.JSONUtils.objectMapper } } """) -class SortPartitionsOpDesc extends LogicalOp with StandaloneCodeGenerator { - - // Sorting is this operator's contract: output row order is meaningful. - override def orderSensitive: Boolean = true +class SortPartitionsOpDesc extends LogicalOp { @JsonProperty(required = true) @JsonSchemaTitle("Attribute") @@ -88,21 +84,4 @@ class SortPartitionsOpDesc extends LogicalOp with StandaloneCodeGenerator { inputPorts = List(InputPort()), outputPorts = List(OutputPort(blocking = true)) ) - - // Row order is this operator's answer, so the sort matches the engine's: - // stable, ascending, nulls first. The domain bounds are partitioning hints - // that the sort itself never reads. - // - // The engine orders a NaN after positive infinity. In a nullable column, - // which is what an Arrow file is read into, pandas keeps a NaN apart from a - // null and sorts it last too. A numpy float column has one slot for both, so - // there a NaN goes first with the nulls. - // - // A string column parts more narrowly: the engine reads UTF-16 code units and - // pandas reads code points, which agree below U+FFFF and can differ above it. - override def generateStandaloneCode(): String = { - val col = pyStringLiteral(Option(sortAttributeName).getOrElse("")) - s"""out1df = in1df.sort_values(by=$col, ascending=True, kind="mergesort", na_position="first").reset_index(drop=True)""" - } - } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/split/SplitOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/split/SplitOpDesc.scala index aa3c377da59..c3c34df948c 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/split/SplitOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/split/SplitOpDesc.scala @@ -29,12 +29,12 @@ import com.kjetland.jackson.jsonSchema.annotations.{ import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow._ -import org.apache.texera.amber.operator.{LogicalOp, SamplingHelpers, StandaloneCodeGenerator} +import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.annotations.HideAnnotation import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.util.JSONUtils.objectMapper -class SplitOpDesc extends LogicalOp with StandaloneCodeGenerator { +class SplitOpDesc extends LogicalOp { @JsonSchemaTitle("Split Percentage") @JsonProperty(defaultValue = "80") @@ -99,25 +99,4 @@ class SplitOpDesc extends LogicalOp with StandaloneCodeGenerator { ) } - override def standaloneHelpers(): Seq[String] = Seq(SamplingHelpers.JavaRandom) - - override def generateStandaloneCode(): String = { - // The executor sends each tuple to the upper port iff nextInt(100) < k, - // drawing from one generator per run. Reproducing that draw keeps the same - // rows on the same side; the mask drives both ports, so `out1df` takes the - // upper k% and `out2df` the remainder and the two partition the input. - // - // With "Auto-Generate Seed" the executor seeds from the clock, so that run - // is not reproducible by anything, itself included — the script then seeds - // from its own clock, matching the intent rather than a particular run. - val seedExpr = if (random) "int(_texera_time.time() * 1000)" else seed.toString - val timeImport = if (random) "import time as _texera_time\n" else "" - s"""${timeImport}_texera_split_rng = _TexeraJavaRandom($seedExpr) - |_texera_split_mask = pd.Series( - | [_texera_split_rng.next_int(100) < $k for _ in range(len(in1df))], - | index=in1df.index, dtype=bool - |) - |out1df = in1df[_texera_split_mask].reset_index(drop=True) - |out2df = in1df[~_texera_split_mask].reset_index(drop=True)""".stripMargin - } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/SamplingHelpersSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/SamplingHelpersSpec.scala deleted file mode 100644 index 8bf4f8f6f22..00000000000 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/SamplingHelpersSpec.scala +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.texera.amber.operator - -import com.typesafe.config.ConfigFactory -import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers - -import java.nio.charset.StandardCharsets -import java.nio.file.Files -import java.util.concurrent.TimeUnit -import scala.io.Source -import scala.util.Try - -/** The transcribed java.util.Random, checked against what Java would answer. - * - * Only the generator is exercised here. What each sampling operator does with - * the numbers is its own spec's business. - */ -class SamplingHelpersSpec extends AnyFlatSpec with Matchers { - - /** Run `body` after the helper class, and return what it printed. */ - private def runWithHelper(body: String): (Int, String) = { - val python = resolvePython().getOrElse(cancel("No runnable python executable")) - val script = Files.createTempFile("sampling-helpers-", ".py") - script.toFile.deleteOnExit() - Files.write( - script, - (SamplingHelpers.JavaRandom + "\n\n" + body).getBytes(StandardCharsets.UTF_8) - ) - val process = new ProcessBuilder(python, script.toString).redirectErrorStream(true).start() - val out = Source.fromInputStream(process.getInputStream).mkString - process.waitFor(60, TimeUnit.SECONDS) - (process.exitValue(), out) - } - - // A bound of 2**30 + 1 puts about half the draws in the rejection zone, so the - // first one with this seed is already a draw Java throws away. The five - // numbers are what java.util.Random answers. - "SamplingHelpers.JavaRandom" should "reject the draws Java's overflow check rejects" in { - val (exit, out) = runWithHelper( - """r = _TexeraJavaRandom(1) - |print([r.next_int((1 << 30) + 1) for _ in range(5)]) - |""".stripMargin - ) - withClue(s"python said:\n$out") { - exit shouldBe 0 - out.trim shouldBe "[215764588, 880641847, 874970313, 446064254, 77814904]" - } - } - - // Reservoir Sampling reaches this check with a reservoir of zero, and the - // engine ends the run there. - it should "refuse a bound that is not positive, the way Java does" in { - val (exit, out) = runWithHelper( - """r = _TexeraJavaRandom(1) - |try: - | r.next_int(0) - | print("no error") - |except ValueError as e: - | print("ValueError:", e) - |""".stripMargin - ) - withClue(s"python said:\n$out") { - exit shouldBe 0 - out.trim shouldBe "ValueError: bound must be positive" - } - } - - // The power-of-two bound takes the branch that never rejects. - it should "take the power-of-two shortcut without rejecting" in { - val (exit, out) = runWithHelper( - """r = _TexeraJavaRandom(42) - |print(all(0 <= r.next_int(256) < 256 for _ in range(1000))) - |""".stripMargin - ) - withClue(s"python said:\n$out") { - exit shouldBe 0 - out.trim shouldBe "True" - } - } - - private def resolvePython(): Option[String] = { - def fromConfig: Option[String] = - Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption - .orElse(Try(ConfigFactory.load()).toOption) - .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) - .map(_.trim) - .filter(_.nonEmpty) - - def runnable(exe: String): Boolean = - Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption - .exists { p => - if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } - - (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(runnable) - } -} diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDescSpec.scala index e046c1d569f..0f0e372ad66 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDescSpec.scala @@ -23,23 +23,13 @@ import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorMetadataGenerator} -import com.typesafe.config.ConfigFactory -import org.apache.texera.amber.operator.tags.IntegrationTest -import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers -import java.nio.charset.StandardCharsets -import java.nio.file.Files -import java.util.concurrent.TimeUnit -import scala.io.Source import scala.jdk.CollectionConverters._ -import scala.util.Try class AggregateOpDescSpec extends AnyFlatSpec with Matchers { - private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) - private val workflowId = WorkflowIdentity(1L) private val executionId = ExecutionIdentity(1L) @@ -118,142 +108,6 @@ class AggregateOpDescSpec extends AnyFlatSpec with Matchers { } } - // The integers behind a timestamp column mean microseconds or nanoseconds - // depending on the resolution it was read at, and a nanosecond total leaves - // the range of a 64-bit integer after a handful of modern dates. - it should "read a timestamp as epoch milliseconds at either resolution" taggedAs NeedsPythonPackages in { - val python = resolvePython().getOrElse(cancel("No runnable python executable")) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") - - val input = Schema().add("t", AttributeType.TIMESTAMP) - val desc = descWith( - List.empty, - aggOp(AggregationFunction.SUM, "t", "ts_total"), - aggOp(AggregationFunction.AVERAGE, "t", "ts_avg") - ) - val block = desc.generateStandaloneCode(Map(PortIdentity() -> input)) - - def report(unit: String): String = - s"""in1df = pd.DataFrame({ - | "t": pd.to_datetime(["2024-01-01"] * 6).astype("datetime64[$unit]"), - |}) - |$block - |print(pd.Timestamp(out1df.iloc[0]["ts_total"]).isoformat()) - |print(repr(float(out1df.iloc[0]["ts_avg"])))""".stripMargin - - val driver = - s"""import pandas as pd - |${report("ns")} - |${report("us")} - |""".stripMargin - - val out = runPython(python, driver, "aggregate-timestamp-units-", "UTC") - - withClue(s"python said:\n$out\nscript:\n$driver") { - // Six times 1704067200000 milliseconds, read back as a timestamp: a date - // no nanosecond count can hold, and the same one at either resolution. - out.trim.linesIterator.toSeq shouldBe Seq( - "2293-12-31T00:00:00", - "1704067200000.0", - "2293-12-31T00:00:00", - "1704067200000.0" - ) - } - } - - /** `Timestamp.getTime` answers for the instant a wall clock names in the - * JVM's default zone, and `new Timestamp(long)` renders one back there, so a - * script that read the column as UTC was a whole offset out on every value - * it added. Checked against Java 17 in the same zone: the sum below is - * 2078-01-01 06:00:00 and the average 1704132000000. - */ - it should "add timestamps in the zone the script runs in" taggedAs NeedsPythonPackages in { - val python = resolvePython().getOrElse(cancel("No runnable python executable")) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") - - val input = Schema().add("t", AttributeType.TIMESTAMP) - val desc = descWith( - List.empty, - aggOp(AggregationFunction.SUM, "t", "ts_total"), - aggOp(AggregationFunction.AVERAGE, "t", "ts_avg") - ) - val block = desc.generateStandaloneCode(Map(PortIdentity() -> input)) - - val driver = - s"""import pandas as pd - |in1df = pd.DataFrame({ - | "t": pd.to_datetime(["2024-01-01 00:00:00", "2024-01-02 00:00:00"]), - |}) - |$block - |print(pd.Timestamp(out1df.iloc[0]["ts_total"]).isoformat()) - |print(repr(float(out1df.iloc[0]["ts_avg"]))) - |""".stripMargin - - // A zone six hours behind UTC and with no daylight saving of its own, so - // one offset covers both dates. - val out = runPython(python, driver, "aggregate-timestamp-zone-", "America/Mexico_City") - - withClue(s"python said:\n$out\nscript:\n$driver") { - out.trim.linesIterator.toSeq shouldBe Seq("2078-01-01T06:00:00", "1704132000000.0") - } - } - - /** Runs the generated block with the zone named to the child, because a - * timestamp's arithmetic reads the one in force where it runs. - */ - private def runPython(python: String, driver: String, prefix: String, zone: String): String = { - val script = Files.createTempFile(prefix, ".py") - script.toFile.deleteOnExit() - Files.write(script, driver.getBytes(StandardCharsets.UTF_8)) - val builder = new ProcessBuilder(python, script.toString).redirectErrorStream(true) - builder.environment().put("TZ", zone) - val process = builder.start() - val out = Source.fromInputStream(process.getInputStream).mkString - process.waitFor(120, TimeUnit.SECONDS) - withClue(s"python said:\n$out\nscript:\n$driver")(process.exitValue() shouldBe 0) - out - } - - // A group keyed on a missing value and one keyed on a NaN are two groups to - // the engine. The two are only distinct in a nullable dtype, which is what an - // Arrow file is read into. - it should "group a missing key apart from a NaN one" taggedAs NeedsPythonPackages in { - val python = resolvePython().getOrElse(cancel("No runnable python executable")) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") - - val desc = descWith(List("k"), aggOp(AggregationFunction.SUM, "v", "total")) - - val driver = - s"""import pandas as pd, numpy as np - | - |in1df = pd.DataFrame({ - | "k": pd.arrays.FloatingArray(np.array([0.0, np.nan]), np.array([True, False])), - | "v": [1, 2], - |}) - |${desc.generateStandaloneCode()} - |print(len(out1df), sorted(int(x) for x in out1df["total"])) - |""".stripMargin - - val script = Files.createTempFile("aggregate-null-nan-", ".py") - script.toFile.deleteOnExit() - Files.write(script, driver.getBytes(StandardCharsets.UTF_8)) - val process = new ProcessBuilder(python, script.toString).redirectErrorStream(true).start() - val out = Source.fromInputStream(process.getInputStream).mkString - process.waitFor(120, TimeUnit.SECONDS) - withClue(s"python said:\n$out\nscript:\n$driver") { - process.exitValue() shouldBe 0 - // Two groups of one row each, not one group totalling 3. - out.trim shouldBe "2 [1, 2]" - } - } - - // Without a schema the type-led branches cannot be chosen. - it should "fall back to pandas' own answers when no schema is given" in { - val desc = descWith(List.empty, aggOp(AggregationFunction.SUM, "i", "int_total")) - desc.generateStandaloneCode() should include("in1df[\"i\"].sum()") - desc.generateStandaloneCode() should not include "_texera_agg_int_sum(in1df" - } - "AggregateOpDesc JSON schema" should "make the attribute optional only for count and required for every other function" in { val aggDef = OperatorMetadataGenerator @@ -277,30 +131,4 @@ class AggregateOpDescSpec extends AnyFlatSpec with Matchers { val elseRequired = rule.get("else").get("required").elements().asScala.map(_.asText()).toList elseRequired should contain("attribute") } - - private def resolvePython(): Option[String] = { - def fromConfig: Option[String] = - Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption - .orElse(Try(ConfigFactory.load()).toOption) - .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) - .map(_.trim) - .filter(_.nonEmpty) - - def runnable(exe: String): Boolean = - Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption - .exists { p => - if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } - - (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(runnable) - } - - private def canImportPandas(python: String): Boolean = - Try( - new ProcessBuilder(python, "-c", "import pandas, numpy").redirectErrorStream(true).start() - ).toOption.exists { p => - if (!p.waitFor(60, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/dummy/DummyOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/dummy/DummyOpDescSpec.scala index 4ee8cfe7fce..6f23261b271 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/dummy/DummyOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/dummy/DummyOpDescSpec.scala @@ -71,11 +71,6 @@ class DummyOpDescSpec extends AnyFlatSpec with Matchers { (new DummyOpDesc).dummyOperator shouldBe "" } - "DummyOpDesc.generateStandaloneCode" should - "pass the first input through to the output" in { - (new DummyOpDesc).generateStandaloneCode() shouldBe "out1df = in1df" - } - "DummyOpDesc.getPhysicalOp" should "be the unimplemented LogicalOp stub (throws NotImplementedError)" in { intercept[NotImplementedError] { diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/ifStatement/IfOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/ifStatement/IfOpDescSpec.scala index 03e879caf06..49cd325d572 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/ifStatement/IfOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/ifStatement/IfOpDescSpec.scala @@ -19,27 +19,16 @@ package org.apache.texera.amber.operator.ifStatement -import com.typesafe.config.ConfigFactory import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.operator.tags.IntegrationTest -import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers -import java.nio.charset.StandardCharsets -import java.nio.file.Files -import java.util.concurrent.TimeUnit -import scala.io.Source -import scala.util.Try - class IfOpDescSpec extends AnyFlatSpec with Matchers { - private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) - private val workflowId = WorkflowIdentity(1L) private val executionId = ExecutionIdentity(1L) @@ -87,82 +76,4 @@ class IfOpDescSpec extends AnyFlatSpec with Matchers { ) out shouldBe Map(PortIdentity() -> dataSchema, PortIdentity(1) -> dataSchema) } - - "IfOpDesc.generateStandaloneCode" should - "name the switch after the condition so two Ifs in one script do not share it" in { - val op = new IfOpDesc - op.conditionName = "ready" - op.generateStandaloneCode() should include("\"_texera_if_ready\"") - } - - // The engine picks the route from a State message, which the verification - // harness has no channel for, so False is only reachable here. - it should "send the rows to True by default and to False when the switch is off" taggedAs NeedsPythonPackages in { - val python = resolvePythonExecutable().getOrElse(cancel("No runnable python executable")) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") - - val op = new IfOpDesc - op.conditionName = "ready" - val body = op.generateStandaloneCode().linesIterator.map(" " + _).mkString("\n") - - val driver = - s"""import pandas as pd - |import sys - | - |for _switch in (None, True, False): - | in2df = pd.DataFrame({"id": [1, 2, 3]}) - | if _switch is None: - | globals().pop("_texera_if_ready", None) - | else: - | globals()["_texera_if_ready"] = _switch - |$body - | print(_switch, list(out1df["id"]), list(out2df["id"])) - |""".stripMargin - - val script = Files.createTempFile("if-standalone-", ".py") - script.toFile.deleteOnExit() - Files.write(script, driver.getBytes(StandardCharsets.UTF_8)) - val process = new ProcessBuilder(python, script.toString).redirectErrorStream(true).start() - val out = Source.fromInputStream(process.getInputStream).mkString - process.waitFor(120, TimeUnit.SECONDS) - - withClue(s"python said:\n$out\nscript:\n$driver") { - process.exitValue() shouldBe 0 - val lines = out.trim.linesIterator.toSeq - // out1df is the False port, out2df the True port. Each route has to be - // exclusive: the rows leave by one and the other comes back empty. - lines should contain("None [] [1, 2, 3]") - lines should contain("True [] [1, 2, 3]") - lines should contain("False [1, 2, 3] []") - // Only the unset switch takes True without being told to, so it alone - // says so. - lines.count(_.contains("_texera_if_ready = False before this step")) shouldBe 1 - } - } - - private def resolvePythonExecutable(): Option[String] = { - def fromConfig: Option[String] = - Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption - .orElse(Try(ConfigFactory.load()).toOption) - .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) - .map(_.trim) - .filter(_.nonEmpty) - - def isRunnable(exe: String): Boolean = - Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption - .exists { p => - if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } - - (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(isRunnable) - } - - private def canImportPandas(python: String): Boolean = - Try( - new ProcessBuilder(python, "-c", "import pandas").redirectErrorStream(true).start() - ).toOption.exists { p => - if (!p.waitFor(60, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/randomksampling/RandomKSamplingOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/randomksampling/RandomKSamplingOpDescSpec.scala index 7d6f3781abb..94d450a9559 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/randomksampling/RandomKSamplingOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/randomksampling/RandomKSamplingOpDescSpec.scala @@ -59,30 +59,6 @@ class RandomKSamplingOpDescSpec extends AnyFlatSpec with Matchers { restored.asInstanceOf[RandomKSamplingOpDesc].percentage shouldBe 25 } - "RandomKSamplingOpDesc.generateStandaloneCode" should - "draw a per-row Bernoulli mask from a seeded RNG" in { - val d = new RandomKSamplingOpDesc - d.percentage = 25 - d.generateStandaloneCode() shouldBe - """_texera_rks_rng = _TexeraJavaRandom(1) - |_texera_rks_mask = pd.Series( - | [0.25 >= _texera_rks_rng.next_double() for _ in range(len(in1df))], - | index=in1df.index, dtype=bool - |) - |out1df = in1df[_texera_rks_mask].reset_index(drop=True)""".stripMargin - } - - // percentage is a whole number on the descriptor but a probability in the - // generated code; an integer division here would keep no rows at all. - it should "convert the percentage to a probability" in { - Seq(0 -> "0.0", 30 -> "0.3", 100 -> "1.0").foreach { - case (pct, prob) => - val d = new RandomKSamplingOpDesc - d.percentage = pct - d.generateStandaloneCode() should include(s"[$prob >= _texera_rks_rng.next_double()") - } - } - "RandomKSamplingOpDesc.getPhysicalOp" should "wire the RandomKSamplingOpExec class name and carry ports" in { val d = new RandomKSamplingOpDesc diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/reservoirsampling/ReservoirSamplingOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/reservoirsampling/ReservoirSamplingOpDescSpec.scala index 0e806034f64..0c903d6ce88 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/reservoirsampling/ReservoirSamplingOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/reservoirsampling/ReservoirSamplingOpDescSpec.scala @@ -19,27 +19,16 @@ package org.apache.texera.amber.operator.reservoirsampling -import com.typesafe.config.ConfigFactory import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} -import org.apache.texera.amber.operator.tags.IntegrationTest -import org.apache.texera.amber.operator.{LogicalOp, SamplingHelpers} +import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.OperatorGroupConstants import org.apache.texera.amber.util.JSONUtils.objectMapper -import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers -import java.nio.charset.StandardCharsets -import java.nio.file.Files -import java.util.concurrent.TimeUnit -import scala.io.Source -import scala.util.Try - class ReservoirSamplingOpDescSpec extends AnyFlatSpec with Matchers { - private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) - private val workflowId = WorkflowIdentity(1L) private val executionId = ExecutionIdentity(1L) @@ -70,56 +59,6 @@ class ReservoirSamplingOpDescSpec extends AnyFlatSpec with Matchers { restored.asInstanceOf[ReservoirSamplingOpDesc].k shouldBe 100 } - // Algorithm R: fill the reservoir with the first k rows, then replace a - // uniformly-drawn slot for each later row. Pin the whole snippet — the - // generated Python is indentation-sensitive. - "ReservoirSamplingOpDesc.generateStandaloneCode" should - "emit a seeded Algorithm R reservoir over the input rows" in { - val d = new ReservoirSamplingOpDesc - d.k = 3 - d.generateStandaloneCode() shouldBe - """_texera_rs_rng = _TexeraJavaRandom(1) - |_texera_rs_k = 3 - |_texera_rs_reservoir = [] - |for _texera_rs_n in range(len(in1df)): - | if _texera_rs_n < _texera_rs_k: - | _texera_rs_reservoir.append(_texera_rs_n) - | else: - | _texera_rs_i = _texera_rs_rng.next_int(_texera_rs_n) - | if _texera_rs_i < _texera_rs_k: - | _texera_rs_reservoir[_texera_rs_i] = _texera_rs_n - |out1df = in1df.iloc[_texera_rs_reservoir].reset_index(drop=True)""".stripMargin - } - - // A reservoir of zero ends the engine's run on the first row: the executor - // skips the fill branch and hands nextInt a bound of zero, which Java refuses. - // The script has to refuse it there too. Answering with an empty table would - // report a result the run never produced. - it should "fail on the first row when the reservoir holds nothing" taggedAs NeedsPythonPackages in { - val python = resolvePython().getOrElse(cancel("No runnable python executable")) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") - - val d = new ReservoirSamplingOpDesc - d.k = 0 - val script = Files.createTempFile("reservoir-zero-", ".py") - script.toFile.deleteOnExit() - val driver = - s"""import pandas as pd - |${SamplingHelpers.JavaRandom} - |in1df = pd.DataFrame({"id": [1, 2, 3]}) - |${d.generateStandaloneCode()} - |""".stripMargin - Files.write(script, driver.getBytes(StandardCharsets.UTF_8)) - - val process = new ProcessBuilder(python, script.toString).redirectErrorStream(true).start() - val out = Source.fromInputStream(process.getInputStream).mkString - process.waitFor(60, TimeUnit.SECONDS) - withClue(s"python said:\n$out\nscript:\n$driver") { - process.exitValue() should not be 0 - out should include("bound must be positive") - } - } - "ReservoirSamplingOpDesc.getPhysicalOp" should "wire the ReservoirSamplingOpExec class name and carry ports" in { val d = new ReservoirSamplingOpDesc @@ -134,30 +73,4 @@ class ReservoirSamplingOpDescSpec extends AnyFlatSpec with Matchers { physical.inputPorts.keySet shouldBe d.operatorInfo.inputPorts.map(_.id).toSet physical.outputPorts.keySet shouldBe d.operatorInfo.outputPorts.map(_.id).toSet } - - private def resolvePython(): Option[String] = { - def fromConfig: Option[String] = - Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption - .orElse(Try(ConfigFactory.load()).toOption) - .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) - .map(_.trim) - .filter(_.nonEmpty) - - def runnable(exe: String): Boolean = - Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption - .exists { p => - if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } - - (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(runnable) - } - - private def canImportPandas(python: String): Boolean = - Try( - new ProcessBuilder(python, "-c", "import pandas").redirectErrorStream(true).start() - ).toOption.exists { p => - if (!p.waitFor(60, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sleep/SleepOpExecSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sleep/SleepOpExecSpec.scala index c5280d30994..2aa143f2948 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sleep/SleepOpExecSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sleep/SleepOpExecSpec.scala @@ -54,21 +54,4 @@ class SleepOpExecSpec extends AnyFlatSpec { val emitted = (0 until 5).flatMap(i => exec.processTuple(tuple(i), 0).toList) assert(emitted == (0 until 5).map(tuple).toList) } - - // The one thing the operator is for. Every other case here configures no delay, - // so the seconds-to-millis conversion was the only line nothing asked about: - // dropping it, or losing the factor of a thousand, left them all passing. - // - // One second is the smallest delay the field can ask for, sleepTime being whole - // seconds, and one tuple is enough to observe it. The lower bound is asserted - // rather than a window: a loaded machine may take longer, and a delay that - // overshoots is not the failure this guards against. - it should "delay each tuple by the configured number of seconds" in { - val exec = new SleepOpExec(descString(1)) - val startNanos = System.nanoTime() - val emitted = exec.processTuple(tuple(1), 0).toList - val elapsedMillis = (System.nanoTime() - startNanos) / 1000000 - assert(emitted == List(tuple(1))) - assert(elapsedMillis >= 1000, s"expected at least 1000 ms of sleep, took $elapsedMillis ms") - } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpDescSpec.scala index c4be492e2bd..933549a3b39 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpDescSpec.scala @@ -19,28 +19,16 @@ package org.apache.texera.amber.operator.sort -import com.typesafe.config.ConfigFactory import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.operator.tags.IntegrationTest import org.apache.texera.amber.util.JSONUtils.objectMapper -import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers -import java.nio.charset.StandardCharsets -import java.nio.file.Files -import java.util.concurrent.TimeUnit -import scala.collection.mutable.ListBuffer -import scala.io.Source -import scala.util.Try - class StableMergeSortOpDescSpec extends AnyFlatSpec with Matchers { - private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) - private val workflowId = WorkflowIdentity(1L) private val executionId = ExecutionIdentity(1L) @@ -79,126 +67,4 @@ class StableMergeSortOpDescSpec extends AnyFlatSpec with Matchers { desc.keys.head.attributeName shouldBe "age" desc.keys.head.sortPreference shouldBe SortPreference.DESC } - - // A null goes last whichever way the key points; a NaN compares above every - // number, so it goes last ascending and first descending. Only a column read - // into a nullable dtype can hold both, which is where this is run. - "StableMergeSortOpDesc.generateStandaloneCode" should - "put a null last both ways and a NaN where the key points" taggedAs NeedsPythonPackages in { - val python = resolvePython().getOrElse(cancel("No runnable python executable")) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") - - def blockFor(pref: SortPreference): String = { - val unit = new SortCriteriaUnit - unit.attributeName = "v" - unit.sortPreference = pref - val d = new StableMergeSortOpDesc - d.keys = ListBuffer(unit) - d.generateStandaloneCode().linesIterator.map(" " + _).mkString("\n") - } - - val driver = - s"""import pandas as pd, numpy as np - | - |def frame(): - | # 3.0, a null, a stored NaN, 1.0, +inf - | col = pd.arrays.FloatingArray( - | np.array([3.0, 0.0, np.nan, 1.0, np.inf]), - | np.array([False, True, False, False, False]), - | ) - | return pd.DataFrame({"v": col}) - | - |def show(df): - | return [("NULL" if x is pd.NA else repr(float(x))) for x in df["v"]] - | - |if True: - | in1df = frame() - |${blockFor(SortPreference.ASC)} - | print("asc", show(out1df)) - |if True: - | in1df = frame() - |${blockFor(SortPreference.DESC)} - | print("desc", show(out1df)) - |""".stripMargin - - val script = Files.createTempFile("sort-nan-", ".py") - script.toFile.deleteOnExit() - Files.write(script, driver.getBytes(StandardCharsets.UTF_8)) - val process = new ProcessBuilder(python, script.toString).redirectErrorStream(true).start() - val out = Source.fromInputStream(process.getInputStream).mkString - process.waitFor(120, TimeUnit.SECONDS) - - withClue(s"python said:\n$out\nscript:\n$driver") { - process.exitValue() shouldBe 0 - val lines = out.trim.linesIterator.toSeq - lines.head shouldBe "asc ['1.0', '3.0', 'inf', 'nan', 'NULL']" - lines(1) shouldBe "desc ['nan', 'inf', '3.0', '1.0', 'NULL']" - } - } - - // The sort tiers are named after nothing the input can be carrying, so a - // column that happens to answer to a helper's old name keeps its values. - it should "keep an input column named after a sort helper" taggedAs NeedsPythonPackages in { - val python = resolvePython().getOrElse(cancel("No runnable python executable")) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") - - val unit = new SortCriteriaUnit - unit.attributeName = "x" - unit.sortPreference = SortPreference.ASC - val desc = new StableMergeSortOpDesc - desc.keys = ListBuffer(unit) - - val driver = - s"""import pandas as pd - | - |in1df = pd.DataFrame({ - | "x": [2, 1], - | "_texera_null_x": ["a", "b"], - | "_texera_nan_x": ["c", "d"], - |}) - |${desc.generateStandaloneCode()} - |print(list(out1df.columns)) - |print(list(out1df["_texera_null_x"]), list(out1df["_texera_nan_x"])) - |""".stripMargin - - val script = Files.createTempFile("sort-helper-name-", ".py") - script.toFile.deleteOnExit() - Files.write(script, driver.getBytes(StandardCharsets.UTF_8)) - val process = new ProcessBuilder(python, script.toString).redirectErrorStream(true).start() - val out = Source.fromInputStream(process.getInputStream).mkString - process.waitFor(120, TimeUnit.SECONDS) - - withClue(s"python said:\n$out\nscript:\n$driver") { - process.exitValue() shouldBe 0 - val lines = out.trim.linesIterator.toSeq - lines.head shouldBe "['x', '_texera_null_x', '_texera_nan_x']" - lines(1) shouldBe "['b', 'a'] ['d', 'c']" - } - } - - private def resolvePython(): Option[String] = { - def fromConfig: Option[String] = - Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption - .orElse(Try(ConfigFactory.load()).toOption) - .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) - .map(_.trim) - .filter(_.nonEmpty) - - def runnable(exe: String): Boolean = - Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption - .exists { p => - if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } - - (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(runnable) - } - - private def canImportPandas(python: String): Boolean = - Try( - new ProcessBuilder(python, "-c", "import pandas, numpy").redirectErrorStream(true).start() - ).toOption.exists { p => - if (!p.waitFor(60, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sortPartitions/SortPartitionsOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sortPartitions/SortPartitionsOpDescSpec.scala index 9d6a2bfdc5b..d9ae20b14a0 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sortPartitions/SortPartitionsOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sortPartitions/SortPartitionsOpDescSpec.scala @@ -19,28 +19,17 @@ package org.apache.texera.amber.operator.sortPartitions -import com.typesafe.config.ConfigFactory import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.RangePartition import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.operator.tags.IntegrationTest import org.apache.texera.amber.util.JSONUtils.objectMapper -import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers -import java.nio.charset.StandardCharsets -import java.nio.file.Files -import java.util.concurrent.TimeUnit -import scala.io.Source -import scala.util.Try - class SortPartitionsOpDescSpec extends AnyFlatSpec with Matchers { - private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) - private val workflowId = WorkflowIdentity(1L) private val executionId = ExecutionIdentity(1L) @@ -88,63 +77,4 @@ class SortPartitionsOpDescSpec extends AnyFlatSpec with Matchers { sp.domainMin shouldBe 1L sp.domainMax shouldBe 99L } - - // The engine puts a null first and a NaN after positive infinity. Only a - // column read into a nullable dtype can hold both, which is what an Arrow - // file is read into, so that is where this is run. - "SortPartitionsOpDesc.generateStandaloneCode" should - "put a null first and a NaN after infinity" taggedAs NeedsPythonPackages in { - val python = resolvePython().getOrElse(cancel("No runnable python executable")) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") - - val driver = - s"""import pandas as pd, numpy as np - | - |# 3.0, a null, a stored NaN, 1.0, +inf - |in1df = pd.DataFrame({"v": pd.arrays.FloatingArray( - | np.array([3.0, 0.0, np.nan, 1.0, np.inf]), - | np.array([False, True, False, False, False]), - |)}) - |${newDesc("v", 0L, 10L).generateStandaloneCode()} - |print([("NULL" if x is pd.NA else repr(float(x))) for x in out1df["v"]]) - |""".stripMargin - - val script = Files.createTempFile("sort-partitions-nan-", ".py") - script.toFile.deleteOnExit() - Files.write(script, driver.getBytes(StandardCharsets.UTF_8)) - val process = new ProcessBuilder(python, script.toString).redirectErrorStream(true).start() - val out = Source.fromInputStream(process.getInputStream).mkString - process.waitFor(120, TimeUnit.SECONDS) - - withClue(s"python said:\n$out\nscript:\n$driver") { - process.exitValue() shouldBe 0 - out.trim shouldBe "['NULL', '1.0', '3.0', 'inf', 'nan']" - } - } - - private def resolvePython(): Option[String] = { - def fromConfig: Option[String] = - Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption - .orElse(Try(ConfigFactory.load()).toOption) - .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) - .map(_.trim) - .filter(_.nonEmpty) - - def runnable(exe: String): Boolean = - Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption - .exists { p => - if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } - - (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(runnable) - } - - private def canImportPandas(python: String): Boolean = - Try( - new ProcessBuilder(python, "-c", "import pandas, numpy").redirectErrorStream(true).start() - ).toOption.exists { p => - if (!p.waitFor(60, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/split/SplitOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/split/SplitOpDescSpec.scala index d0a769e9472..d1fecdf7834 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/split/SplitOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/split/SplitOpDescSpec.scala @@ -126,44 +126,6 @@ class SplitOpDescSpec extends AnyFlatSpec with Matchers { } } - // --------------------------------------------------------------------------- - // generateStandaloneCode - // --------------------------------------------------------------------------- - - // The generator is a transcription of java.util.Random, so the script draws the - // rows the executor drew rather than a differently-shaped sample from the same - // distribution. With auto-generate on, the seed comes from the clock, as it does - // on the executor. - "SplitOpDesc.generateStandaloneCode" should - "draw an unseeded mask and partition the input across both outputs" in { - (new SplitOpDesc).generateStandaloneCode() shouldBe - """import time as _texera_time - |_texera_split_rng = _TexeraJavaRandom(int(_texera_time.time() * 1000)) - |_texera_split_mask = pd.Series( - | [_texera_split_rng.next_int(100) < 80 for _ in range(len(in1df))], - | index=in1df.index, dtype=bool - |) - |out1df = in1df[_texera_split_mask].reset_index(drop=True) - |out2df = in1df[~_texera_split_mask].reset_index(drop=True)""".stripMargin - } - - it should "pass the configured seed to the generator when auto-generate is off" in { - val d = new SplitOpDesc - d.random = false - d.seed = 42 - d.generateStandaloneCode() should include("_texera_split_rng = _TexeraJavaRandom(42)") - } - - // k is a percentage on both sides, compared as the executor compares it: - // nextInt(100) < k, not a uniform draw against k/100. - it should "compare the draw against the percentage the executor uses" in { - Seq(0, 30, 80, 100).foreach { k => - val d = new SplitOpDesc - d.k = k - d.generateStandaloneCode() should include(s"_texera_split_rng.next_int(100) < $k") - } - } - // --------------------------------------------------------------------------- // Independent instances // --------------------------------------------------------------------------- From 77c64a3097848403047982f0120671b7cf053ba8 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Mon, 12 Oct 2026 19:54:30 -0700 Subject: [PATCH 20/32] fix(workflow-operator): match set-operator rows by the engine's equality The engine's HashSet[Tuple] compares rows with Scala's ==, under which NaN equals nothing, and hashes them by each double's bits, which tell 0.0 from -0.0. pandas' duplicate checks count both pairs as equal, so Intersect, Difference and Symmetric Difference now select rows by a key that follows the engine. Co-Authored-By: Claude Opus 5.5 --- .../amber/operator/StandaloneHelpers.scala | 43 +++++++++ .../difference/DifferenceOpDesc.scala | 14 +-- .../operator/intersect/IntersectOpDesc.scala | 11 +-- .../SymmetricDifferenceOpDesc.scala | 18 ++-- .../difference/DifferenceOpDescSpec.scala | 27 +++++- .../intersect/IntersectOpDescSpec.scala | 26 +++++- .../SymmetricDifferenceOpDescSpec.scala | 27 +++++- .../amber/testsupport/setop/SetRows.scala | 89 +++++++++++++++++++ 8 files changed, 235 insertions(+), 20 deletions(-) create mode 100644 common/workflow-operator/src/test/scala/org/apache/texera/amber/testsupport/setop/SetRows.scala diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneHelpers.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneHelpers.scala index 1414f6b043c..6e26bbb7b03 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneHelpers.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneHelpers.scala @@ -24,6 +24,49 @@ package org.apache.texera.amber.operator */ object StandaloneHelpers { + /** + * Row equality as the set operators' `HashSet[Tuple]` decides it, for + * Intersect, Difference and Symmetric Difference. + * + * Two rows meet only when both their hashes and `equals` agree. `equals` is + * Scala's `==` on each field, under which NaN equals nothing, not even a + * copy of its own row. The hash is Java's, from each double's bits, so 0.0 + * and -0.0 land apart. pandas' duplicate checks count both pairs as equal. + */ + val SetRowKeys: String = + """def _texera_set_row_keys(df): + | # One key per row, equal exactly where the engine's rows are. A null + | # is read off the column's mask, since a stored NaN is not one. + | import math + | + | def key(v, missing): + | if missing: + | return None + | if isinstance(v, float): + | if math.isnan(v): + | return object() + | return (v, math.copysign(1.0, v)) + | return v + | + | return [ + | tuple(key(v, m) for v, m in zip(values, missing)) + | for values, missing in zip( + | df.itertuples(index=False, name=None), df.isna().to_numpy() + | ) + | ] + | + | + |def _texera_set_select(df, keep): + | # The rows the engine's HashSet holds, the first of each key, that + | # `keep` accepts. + | seen = set() + | picked = [] + | for i, key in enumerate(_texera_set_row_keys(df)): + | if key not in seen and keep(key): + | picked.append(i) + | seen.add(key) + | return df.iloc[picked].reset_index(drop=True)""".stripMargin + /** * A Python transcription of `AttributeTypeUtils`, for operators that cast a * column to a declared type. diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/difference/DifferenceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/difference/DifferenceOpDesc.scala index 887c1aa44e1..b4d2b5f2b22 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/difference/DifferenceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/difference/DifferenceOpDesc.scala @@ -24,7 +24,7 @@ import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow._ import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} -import org.apache.texera.amber.operator.{LogicalOp, StandaloneCodeGenerator} +import org.apache.texera.amber.operator.{LogicalOp, StandaloneCodeGenerator, StandaloneHelpers} class DifferenceOpDesc extends LogicalOp with StandaloneCodeGenerator { override def getPhysicalOp( @@ -61,11 +61,11 @@ class DifferenceOpDesc extends LogicalOp with StandaloneCodeGenerator { outputPorts = List(OutputPort(blocking = true)) ) - // Distinct rows in left (port 0) not in right (port 1). [left, right, right] + - // drop_duplicates(keep=False): right rows appear >=2x and drop, left-only - // survive. concat (not merge) so NaN == NaN matches the JVM HashSet. + // Distinct left (port 0) rows the right (port 1) does not hold, by the + // engine's row equality. override def generateStandaloneCode(): String = - "out1df = pd.concat([in1df.drop_duplicates(), in2df.drop_duplicates(), " + - "in2df.drop_duplicates()], ignore_index=True)" + - ".drop_duplicates(keep=False).reset_index(drop=True)" + """_texera_set_right = set(_texera_set_row_keys(in2df)) + |out1df = _texera_set_select(in1df, lambda key: key not in _texera_set_right)""".stripMargin + + override def standaloneHelpers(): Seq[String] = Seq(StandaloneHelpers.SetRowKeys) } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/intersect/IntersectOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/intersect/IntersectOpDesc.scala index ff2c564398b..092d0a380f8 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/intersect/IntersectOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/intersect/IntersectOpDesc.scala @@ -23,7 +23,7 @@ import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow._ import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} -import org.apache.texera.amber.operator.{LogicalOp, StandaloneCodeGenerator} +import org.apache.texera.amber.operator.{LogicalOp, StandaloneCodeGenerator, StandaloneHelpers} class IntersectOpDesc extends LogicalOp with StandaloneCodeGenerator { override def getPhysicalOp( @@ -52,9 +52,10 @@ class IntersectOpDesc extends LogicalOp with StandaloneCodeGenerator { outputPorts = List(OutputPort(blocking = true)) ) - // Distinct rows in both inputs. concat + duplicated (not pd.merge) so NaN == - // NaN matches the JVM HashSet's null equality. + // Distinct left rows the right also holds, by the engine's row equality. override def generateStandaloneCode(): String = - """_both = pd.concat([in1df.drop_duplicates(), in2df.drop_duplicates()], ignore_index=True) - |out1df = _both[_both.duplicated(keep="first")].reset_index(drop=True)""".stripMargin + """_texera_set_right = set(_texera_set_row_keys(in2df)) + |out1df = _texera_set_select(in1df, lambda key: key in _texera_set_right)""".stripMargin + + override def standaloneHelpers(): Seq[String] = Seq(StandaloneHelpers.SetRowKeys) } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/symmetricDifference/SymmetricDifferenceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/symmetricDifference/SymmetricDifferenceOpDesc.scala index ad073d53628..de4699f6d01 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/symmetricDifference/SymmetricDifferenceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/symmetricDifference/SymmetricDifferenceOpDesc.scala @@ -24,7 +24,7 @@ import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow._ import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} -import org.apache.texera.amber.operator.{LogicalOp, StandaloneCodeGenerator} +import org.apache.texera.amber.operator.{LogicalOp, StandaloneCodeGenerator, StandaloneHelpers} class SymmetricDifferenceOpDesc extends LogicalOp with StandaloneCodeGenerator { override def getPhysicalOp( @@ -61,9 +61,17 @@ class SymmetricDifferenceOpDesc extends LogicalOp with StandaloneCodeGenerator { outputPorts = List(OutputPort(blocking = true)) ) - // Distinct rows in exactly one input. drop_duplicates(keep=False) drops rows - // seen on both sides. concat (not merge) so NaN == NaN matches the JVM HashSet. + // Distinct rows in exactly one input, by the engine's row equality. override def generateStandaloneCode(): String = - "out1df = pd.concat([in1df.drop_duplicates(), in2df.drop_duplicates()], " + - "ignore_index=True).drop_duplicates(keep=False).reset_index(drop=True)" + """_texera_set_left = set(_texera_set_row_keys(in1df)) + |_texera_set_right = set(_texera_set_row_keys(in2df)) + |out1df = pd.concat( + | [ + | _texera_set_select(in1df, lambda key: key not in _texera_set_right), + | _texera_set_select(in2df, lambda key: key not in _texera_set_left), + | ], + | ignore_index=True, + |)""".stripMargin + + override def standaloneHelpers(): Seq[String] = Seq(StandaloneHelpers.SetRowKeys) } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/difference/DifferenceOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/difference/DifferenceOpDescSpec.scala index ef7af3113c6..11f7c875421 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/difference/DifferenceOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/difference/DifferenceOpDescSpec.scala @@ -31,6 +31,7 @@ import org.apache.texera.amber.core.workflow.{ } import org.apache.texera.amber.operator.metadata.OperatorGroupConstants import org.apache.texera.amber.operator.tags.IntegrationTest +import org.apache.texera.amber.testsupport.setop.SetRows import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -169,7 +170,7 @@ class DifferenceOpDescSpec extends AnyFlatSpec with Matchers { | |in1df = pd.DataFrame({"v": col([0.0, np.nan], [True, False])}) |in2df = pd.DataFrame({"v": col([0.0], [True])}) - |${(new DifferenceOpDesc).generateStandaloneCode()} + |${exported()} |print(len(out1df), [("NULL" if x is pd.NA else repr(float(x))) for x in out1df["v"]]) |""".stripMargin @@ -186,6 +187,30 @@ class DifferenceOpDescSpec extends AnyFlatSpec with Matchers { } } + // The engine's HashSet compares rows with Scala's ==, under which NaN equals + // nothing, so both left NaN rows survive, and hashes them by each double's + // bits, so a left 0.0 survives a right -0.0. A finite value and a null on + // both sides are the controls. + it should "match rows by the engine's equality for NaN and signed zero" taggedAs NeedsPythonPackages in { + val python = resolvePython().getOrElse(cancel("No runnable python executable")) + if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + + val exec = new DifferenceOpExec + exec.open() + SetRows.left.foreach(t => exec.processTuple(t, 0)) + SetRows.right.foreach(t => exec.processTuple(t, 1)) + exec.onFinish(0) + val engine = SetRows.rendered(exec.onFinish(1)) + + val out = SetRows.runExport(python, exported(), "difference-nan-zero-") + withClue(s"script:\n${exported()}")(out shouldBe engine) + } + + private def exported(): String = { + val desc = new DifferenceOpDesc + (desc.standaloneHelpers() :+ desc.generateStandaloneCode()).mkString("\n") + } + private def resolvePython(): Option[String] = { def fromConfig: Option[String] = Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/intersect/IntersectOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/intersect/IntersectOpDescSpec.scala index 3280eff83be..05a4ffa0910 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/intersect/IntersectOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/intersect/IntersectOpDescSpec.scala @@ -25,6 +25,7 @@ import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, Workflow import org.apache.texera.amber.core.workflow.{HashPartition, SinglePartition, UnknownPartition} import org.apache.texera.amber.operator.metadata.OperatorGroupConstants import org.apache.texera.amber.operator.tags.IntegrationTest +import org.apache.texera.amber.testsupport.setop.SetRows import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -108,7 +109,7 @@ class IntersectOpDescSpec extends AnyFlatSpec with Matchers { | |in1df = pd.DataFrame({"v": col([0.0, np.nan], [True, False])}) |in2df = pd.DataFrame({"v": col([0.0], [True])}) - |${(new IntersectOpDesc).generateStandaloneCode()} + |${exported()} |print(len(out1df), [("NULL" if x is pd.NA else repr(float(x))) for x in out1df["v"]]) |""".stripMargin @@ -125,6 +126,29 @@ class IntersectOpDescSpec extends AnyFlatSpec with Matchers { } } + // The engine's HashSet compares rows with Scala's ==, under which NaN equals + // nothing, and hashes them by each double's bits, which tell 0.0 from -0.0. + // A finite value and a null on both sides are the controls. + it should "match rows by the engine's equality for NaN and signed zero" taggedAs NeedsPythonPackages in { + val python = resolvePython().getOrElse(cancel("No runnable python executable")) + if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + + val exec = new IntersectOpExec + exec.open() + SetRows.left.foreach(t => exec.processTuple(t, 0)) + SetRows.right.foreach(t => exec.processTuple(t, 1)) + exec.onFinish(0) + val engine = SetRows.rendered(exec.onFinish(1)) + + val out = SetRows.runExport(python, exported(), "intersect-nan-zero-") + withClue(s"script:\n${exported()}")(out shouldBe engine) + } + + private def exported(): String = { + val desc = new IntersectOpDesc + (desc.standaloneHelpers() :+ desc.generateStandaloneCode()).mkString("\n") + } + private def resolvePython(): Option[String] = { def fromConfig: Option[String] = Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/symmetricDifference/SymmetricDifferenceOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/symmetricDifference/SymmetricDifferenceOpDescSpec.scala index 866e08c6ae1..01270b92a40 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/symmetricDifference/SymmetricDifferenceOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/symmetricDifference/SymmetricDifferenceOpDescSpec.scala @@ -27,6 +27,7 @@ import org.apache.texera.amber.core.workflow.{HashPartition, PortIdentity} import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.OperatorGroupConstants import org.apache.texera.amber.operator.tags.IntegrationTest +import org.apache.texera.amber.testsupport.setop.SetRows import org.apache.texera.amber.util.JSONUtils.objectMapper import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec @@ -115,7 +116,7 @@ class SymmetricDifferenceOpDescSpec extends AnyFlatSpec with Matchers { | |in1df = pd.DataFrame({"v": col([0.0, np.nan], [True, False])}) |in2df = pd.DataFrame({"v": col([0.0], [True])}) - |${(new SymmetricDifferenceOpDesc).generateStandaloneCode()} + |${exported()} |print(len(out1df), [("NULL" if x is pd.NA else repr(float(x))) for x in out1df["v"]]) |""".stripMargin @@ -132,6 +133,30 @@ class SymmetricDifferenceOpDescSpec extends AnyFlatSpec with Matchers { } } + // The engine's HashSet compares rows with Scala's ==, under which NaN equals + // nothing, so every NaN row is in exactly one input, and hashes them by each + // double's bits, so 0.0 and -0.0 are each in one. A finite value and a null + // on both sides are the controls. + it should "match rows by the engine's equality for NaN and signed zero" taggedAs NeedsPythonPackages in { + val python = resolvePython().getOrElse(cancel("No runnable python executable")) + if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + + val exec = new SymmetricDifferenceOpExec + exec.open() + SetRows.left.foreach(t => exec.processTuple(t, 0)) + SetRows.right.foreach(t => exec.processTuple(t, 1)) + exec.onFinish(0) + val engine = SetRows.rendered(exec.onFinish(1)) + + val out = SetRows.runExport(python, exported(), "symmetric-difference-nan-zero-") + withClue(s"script:\n${exported()}")(out shouldBe engine) + } + + private def exported(): String = { + val desc = new SymmetricDifferenceOpDesc + (desc.standaloneHelpers() :+ desc.generateStandaloneCode()).mkString("\n") + } + private def resolvePython(): Option[String] = { def fromConfig: Option[String] = Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/testsupport/setop/SetRows.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/testsupport/setop/SetRows.scala new file mode 100644 index 00000000000..a8e99a01efc --- /dev/null +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/testsupport/setop/SetRows.scala @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.testsupport.setop + +import org.apache.texera.amber.core.tuple.{AttributeType, Schema, Tuple, TupleLike} + +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.util.concurrent.TimeUnit +import scala.io.Source + +/** The two inputs the set operators' parity tests share, fed to the native + * executor as tuples and to the export as nullable Arrow-style columns. + * + * Each side holds a finite value and a null the other side also holds, as + * controls, next to a NaN and a zero of each sign. The left holds its NaN + * twice, since the engine keeps two NaN rows apart as well. + */ +object SetRows { + private val schema = Schema().add("k", AttributeType.STRING).add("v", AttributeType.DOUBLE) + + private val leftValues: Seq[java.lang.Double] = Seq(1.0, Double.NaN, 0.0, null, 2.0, Double.NaN) + private val rightValues: Seq[java.lang.Double] = Seq(1.0, Double.NaN, -0.0, null, 3.0) + + val left: Seq[Tuple] = leftValues.map(v => Tuple(schema, Array[Any]("a", v))) + val right: Seq[Tuple] = rightValues.map(v => Tuple(schema, Array[Any]("a", v))) + + /** The native output, one sorted line per row, written as the driver writes it. */ + def rendered(rows: Iterator[TupleLike]): Seq[String] = + rows.map(_.getFields.map(render).mkString(" ")).toSeq.sorted + + /** Runs `exported` over the same rows and returns what it answered, as above. */ + def runExport(python: String, exported: String, prefix: String): Seq[String] = { + val driver = + s"""import pandas as pd, numpy as np + | + |def col(vals): + | mask = np.array([v is None for v in vals]) + | return pd.arrays.FloatingArray(np.array([0.0 if v is None else v for v in vals]), mask) + | + |in1df = pd.DataFrame({"k": ["a"] * ${leftValues.size}, "v": col(${pyList(leftValues)})}) + |in2df = pd.DataFrame({"k": ["a"] * ${rightValues.size}, "v": col(${pyList(rightValues)})}) + |$exported + |def render(v): + | return "None" if v is pd.NA else v if isinstance(v, str) else repr(float(v)) + |for row in sorted(" ".join(render(v) for v in r) for r in out1df.itertuples(index=False)): + | print(row) + |""".stripMargin + val script = Files.createTempFile(prefix, ".py") + script.toFile.deleteOnExit() + Files.write(script, driver.getBytes(StandardCharsets.UTF_8)) + val process = new ProcessBuilder(python, script.toString).redirectErrorStream(true).start() + val out = Source.fromInputStream(process.getInputStream, "UTF-8").mkString + process.waitFor(120, TimeUnit.SECONDS) + if (process.exitValue() != 0) { + throw new AssertionError(s"python said:\n$out\nscript:\n$driver") + } + out.linesIterator.toSeq + } + + private def render(value: Any): String = + value match { + case null => "None" + case d: java.lang.Double if d.isNaN => "nan" + case other => other.toString + } + + private def pyList(values: Seq[java.lang.Double]): String = + values + .map(v => if (v == null) "None" else if (v.isNaN) "np.nan" else v.toString) + .mkString("[", ", ", "]") +} From ce2b893269630e62519a9317815929dfc84bd37a Mon Sep 17 00:00:00 2001 From: kary zheng Date: Mon, 12 Oct 2026 21:28:42 -0700 Subject: [PATCH 21/32] fix(workflow-operator): name a HashJoin probe's right columns from the declared build schema A probe worker read the left column names off its first build row, so one that received no build rows put an unmatched right row's columns into the left ones. The runtime now records each input port's declared schema on the executor before the port finishes, and the probe reads the left names from it. Co-Authored-By: Claude Opus 5.5 --- .../promisehandlers/EndChannelHandler.scala | 3 ++ .../EndChannelHandlerSpec.scala | 16 +++++++++ .../core/executor/OperatorExecutor.scala | 14 +++++++- .../hashJoin/HashJoinProbeOpExec.scala | 18 ++++++---- .../hashJoin/HashJoinProbeOpExecSpec.scala | 34 +++++++++++++++++++ 5 files changed, 78 insertions(+), 7 deletions(-) diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/EndChannelHandler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/EndChannelHandler.scala index f1b37d7d325..5d61032d60d 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/EndChannelHandler.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/EndChannelHandler.scala @@ -40,6 +40,9 @@ trait EndChannelHandler { val portId = dp.inputGateway.getChannel(channelId).getPortId dp.inputManager.getPort(portId).completed = true dp.inputManager.initBatch(channelId, Array.empty) + // Before the operator finishes the port, so one that received no rows on + // it can still say what its columns were. + dp.executor.inputSchemas(portId.id) = dp.inputManager.getPort(portId).schema try { val outputState = dp.executor.produceStateOnFinish(portId.id) if (outputState.isDefined) { diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/EndChannelHandlerSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/EndChannelHandlerSpec.scala index e9ba6369630..c55891316b6 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/EndChannelHandlerSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/EndChannelHandlerSpec.scala @@ -235,6 +235,18 @@ class EndChannelHandlerSpec extends AnyFlatSpec { assert(executor.finishOutputPorts.toList == List(currentPortId.id)) } + it should "give the operator the finished port's declared schema before it finishes the port" in { + // A HashJoin probe worker that received no build rows names its unmatched right rows from + // this, so it has to be in place when the operator is asked to finish, not after. + val executor = new RecordingExecutor() + val fixture = new Fixture(executor, otherInputPortCompleted = false) + + fixture.endChannel() + + assert(executor.schemasAtFinish.toList == List(Map(currentPortId.id -> schema))) + assert(!executor.inputSchemas.contains(otherPortId.id)) + } + it should "emit the boundary state the operator produced on finish, with no loop envelope" in { val state = State(Map("boundary" -> "end")) val fixture = new Fixture(new RecordingExecutor(onFinishState = _ => Some(state))) @@ -431,8 +443,12 @@ object EndChannelHandlerSpec { /** Ports handed to `onFinish`, i.e. what `onFinishMultiPort` forwarded. */ val finishOutputPorts: ArrayBuffer[Int] = ArrayBuffer() + /** `inputSchemas` as the first finish callback saw it. */ + val schemasAtFinish: ArrayBuffer[Map[Int, Schema]] = ArrayBuffer() + override def produceStateOnFinish(port: Int): Option[State] = { statePorts += port + schemasAtFinish += inputSchemas.toMap onFinishState(port) } diff --git a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/executor/OperatorExecutor.scala b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/executor/OperatorExecutor.scala index 9837213abbb..6e3d1d7d90d 100644 --- a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/executor/OperatorExecutor.scala +++ b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/executor/OperatorExecutor.scala @@ -20,11 +20,23 @@ package org.apache.texera.amber.core.executor import org.apache.texera.amber.core.state.State -import org.apache.texera.amber.core.tuple.{Tuple, TupleLike} +import org.apache.texera.amber.core.tuple.{Schema, Tuple, TupleLike} import org.apache.texera.amber.core.workflow.PortIdentity +import scala.collection.mutable + trait OperatorExecutor { + /** + * What each finished input port was declared to carry, keyed by port index + * and written by the runtime before `onFinish` for that port. + * + * The tuples say this too, so an operator only needs to ask when there were + * none: a port can finish having carried no rows, and its schema is then the + * only record of what its columns were. + */ + val inputSchemas: mutable.Map[Int, Schema] = mutable.Map() + def open(): Unit = {} def produceStateOnStart(port: Int): Option[State] = None diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinProbeOpExec.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinProbeOpExec.scala index e6783d3bf8c..c11bcb19ed4 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinProbeOpExec.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinProbeOpExec.scala @@ -74,9 +74,15 @@ class HashJoinProbeOpExec[K]( objectMapper.readValue(descString, classOf[HashJoinOpDesc[K]]) var buildTableHashMap: mutable.HashMap[K, (ListBuffer[Tuple], Boolean)] = _ - // The build side's column names, learned from its first row. An unmatched - // right row has no build row of its own to name them. - private var leftAttributeNames: List[String] = List.empty + // The build side's column names, from the schema its port declared. An + // unmatched right row has no build row of its own to name them, and a worker + // can receive no build rows at all. The build port finishes before the first + // right row arrives. + private def leftAttributeNames: List[String] = + inputSchemas + .get(0) + .map(_.getAttributeNames.filterNot(_ == HASH_JOIN_INTERNAL_KEY_NAME)) + .getOrElse(List.empty) override def open(): Unit = { buildTableHashMap = new mutable.HashMap[K, (mutable.ListBuffer[Tuple], Boolean)]() @@ -90,10 +96,10 @@ class HashJoinProbeOpExec[K]( if (port == 0) { // Load build hash map val key = tuple.getField[K](HASH_JOIN_INTERNAL_KEY_NAME) - val names = tuple.getSchema.getAttributeNames.filterNot(n => n == HASH_JOIN_INTERNAL_KEY_NAME) - if (leftAttributeNames.isEmpty) leftAttributeNames = names buildTableHashMap.getOrElseUpdate(key, (new ListBuffer[Tuple](), false))._1 += tuple - .getPartialTuple(names) + .getPartialTuple( + tuple.getSchema.getAttributeNames.filterNot(n => n == HASH_JOIN_INTERNAL_KEY_NAME) + ) Iterator.empty } else { // Probe phase diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinProbeOpExecSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinProbeOpExecSpec.scala index 660aba86acd..f249b50c228 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinProbeOpExecSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinProbeOpExecSpec.scala @@ -89,6 +89,8 @@ class HashJoinProbeOpExecSpec extends AnyFlatSpec { ) buildExec.close() + // The runtime records the port's declared schema before finishing it. + probeExec.inputSchemas(buildPort) = internalHashTableSchema assert(probeExec.onFinish(buildPort).isEmpty) probeExec } @@ -314,6 +316,9 @@ class HashJoinProbeOpExecSpec extends AnyFlatSpec { .foreach(t => probeExec.processTuple(t.asInstanceOf[SchemaEnforceable].enforceSchema(buildOut), 0) ) + // The runtime records the port's declared schema before finishing it. + probeExec.inputSchemas(0) = buildOut + probeExec.onFinish(0) val emitted = rightRows.flatMap(r => probeExec.processTuple(Tuple.builder(right).addSequentially(r).build(), 1) ) ++ probeExec.onFinish(1) @@ -374,4 +379,33 @@ class HashJoinProbeOpExecSpec extends AnyFlatSpec { assert(names == List("id", "key", "value")) assert(rows.toSet == Set(List(null, null, 9), List("1", "payload", null))) } + + // A worker can receive no build rows at all, when the left input is empty or + // no left key hashes to it. The left names then come from the build port's + // declared schema alone, and the two cases above must still hold. + it should "name an unmatched right row's columns when the build side sent no rows" in { + val (names, rows) = joinedRows( + JoinType.FULL_OUTER, + Schema().add("k", AttributeType.STRING).add("x", AttributeType.INTEGER), + Seq.empty, + Schema().add("k", AttributeType.STRING).add("x", AttributeType.DOUBLE), + Seq(Array[Any]("2", 2.5)) + ) + assert(names == List("k", "x", "x#@1")) + assert(rows == List(List("2", null, 2.5))) + } + + it should "not let an unmatched right key fill a left payload when the build side sent no rows" in { + val (names, rows) = joinedRows( + JoinType.FULL_OUTER, + Schema().add("id", AttributeType.STRING).add("key", AttributeType.STRING), + Seq.empty, + Schema().add("key", AttributeType.STRING).add("value", AttributeType.INTEGER), + Seq(Array[Any]("2", 9)), + buildKey = "id", + probeKey = "key" + ) + assert(names == List("id", "key", "value")) + assert(rows == List(List(null, null, 9))) + } } From fa079508f6504415ef4e7a23139cecd43e190127 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Tue, 13 Oct 2026 20:08:34 -0700 Subject: [PATCH 22/32] fix(workflow-operator): let a NaN key match nothing in Hash Join, as the engine does The probe looks each key up in a HashMap under Scala's ==, where a NaN finds nothing, while merge paired NaN with NaN. Merge on codes that follow the engine's equality instead, each NaN taking one of its own. Co-Authored-By: Claude Opus 5.5 --- .../operator/hashJoin/HashJoinOpDesc.scala | 39 +++++-- .../hashJoin/HashJoinOpDescSpec.scala | 101 +++++++++++++++++- 2 files changed, 132 insertions(+), 8 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDesc.scala index 790c6d06a8d..724f067bd3e 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDesc.scala @@ -265,8 +265,13 @@ class HashJoinOpDesc[K] extends LogicalOp with StandaloneCodeGenerator { // // JoinUtils sets the probe key aside before it renames anything, so a // payload column never has to step around the key that is about to leave. - // The key itself still rides along, because the merge joins on it, and it - // only has to land where nothing else already sits. + // The key itself still rides along, and it only has to land where nothing + // else already sits. + // + // The merge joins on codes and not on the keys. The probe looks each key up + // in a HashMap under Scala's ==, where a NaN finds nothing, not even another + // NaN, while 0.0 meets -0.0 and a null meets a null; merge pairs NaN with + // NaN. val merge = s"""_left_cols = set(in1df.columns) |_right_cols = list(in2df.columns) @@ -285,13 +290,33 @@ class HashJoinOpDesc[K] extends LogicalOp with StandaloneCodeGenerator { | _probe_key = _probe_key + "#@1" |if _probe_key != $probeKeyLit: | _rename[$probeKeyLit] = _probe_key + |_join_key = "_texera_join_key" + |while _join_key in _taken or _join_key == _probe_key: + | _join_key = _join_key + "_" + | + |def _texera_join_codes(left, right): + | # Each NaN takes a code of its own. A null is read off the column's + | # mask, since a stored NaN is not one. + | import math + | + | codes = {} + | + | def code(s): + | out = [] + | for v, missing in zip(s, s.isna().to_numpy()): + | key = None if missing else object() if isinstance(v, float) and math.isnan(v) else v + | out.append(codes.setdefault(key, len(codes))) + | return out + | + | return code(left), code(right) + | + |_left_codes, _right_codes = _texera_join_codes(in1df[$buildKeyLit], in2df[$probeKeyLit]) |""".stripMargin + widening + - s"""out1df = $leftFrame.merge( - | $rightFrame, + s"""out1df = $leftFrame.assign(**{_join_key: _left_codes}).merge( + | $rightFrame.assign(**{_join_key: _right_codes}), | how=${pyStringLiteral(how)}, - | left_on=$buildKeyLit, - | right_on=_probe_key, - |)""".stripMargin + | on=_join_key, + |).drop(columns=[_join_key])""".stripMargin // `_probe_key` is whatever the rename settled on, so this drops the right // column rather than a left one that happened to share its name. Asked of // the name the rename produced and not of the two the operator was given: diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDescSpec.scala index 1efef3e96f1..3c2514c2f24 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDescSpec.scala @@ -20,7 +20,13 @@ package org.apache.texera.amber.operator.hashJoin import com.typesafe.config.ConfigFactory -import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema} +import org.apache.texera.amber.core.tuple.{ + Attribute, + AttributeType, + Schema, + SchemaEnforceable, + Tuple +} import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.OperatorGroupConstants @@ -123,6 +129,99 @@ class HashJoinOpDescSpec extends AnyFlatSpec with Matchers { } } + // The probe looks each key up in a HashMap under Scala's ==: a NaN finds + // nothing, not even another NaN, while 0.0 meets -0.0 and a null meets a + // null. Each join type keeps or drops the unmatched NaN rows by its own rule. + for (joinType <- JoinType.values) { + it should s"pair NaN, signed-zero and null keys as the engine does in a $joinType join" taggedAs NeedsPythonPackages in { + val python = resolvePython().getOrElse(cancel("No runnable python executable")) + if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + + val d = new HashJoinOpDesc[java.lang.Double] + d.buildAttributeName = "k" + d.probeAttributeName = "k2" + d.joinType = joinType + val left = Schema().add("k", AttributeType.DOUBLE).add("lv", AttributeType.STRING) + val right = Schema().add("k2", AttributeType.DOUBLE).add("rv", AttributeType.STRING) + val leftRows: Seq[(java.lang.Double, String)] = + Seq((Double.NaN, "L-nan"), (0.0, "L+0"), (1.0, "L1"), (null, "L-null")) + val rightRows: Seq[(java.lang.Double, String)] = + Seq((Double.NaN, "R-nan"), (-0.0, "R-0"), (1.0, "R1"), (null, "R-null")) + + val outSchema = d + .getExternalOutputSchemas(Map(PortIdentity() -> left, PortIdentity(1) -> right)) + .values + .head + val build = new HashJoinBuildOpExec[java.lang.Double](objectMapper.writeValueAsString(d)) + build.open() + leftRows.foreach { case (k, v) => build.processTuple(Tuple(left, Array[Any](k, v)), 0) } + val probe = new HashJoinProbeOpExec[java.lang.Double](objectMapper.writeValueAsString(d)) + probe.open() + val internal = + Schema().add(HashJoinOpDesc.HASH_JOIN_INTERNAL_KEY_NAME, AttributeType.ANY).add(left) + build + .onFinish(0) + .foreach(t => + probe.processTuple(t.asInstanceOf[SchemaEnforceable].enforceSchema(internal), 0) + ) + probe.onFinish(0) + val probed = rightRows.flatMap { + case (k, v) => probe.processTuple(Tuple(right, Array[Any](k, v)), 1) + } ++ probe.onFinish(1) + val engine = probed + .map(_.asInstanceOf[SchemaEnforceable].enforceSchema(outSchema).getFields.map { + case null => "None" + case d: java.lang.Double if d.isNaN => "nan" + case other => other.toString + }) + .map(_.mkString(" ")) + .toList + .sorted + + def frame(rows: Seq[(java.lang.Double, String)], key: String, value: String): String = { + val keys = + rows.map(_._1).map(k => if (k == null) "None" else if (k.isNaN) "np.nan" else k.toString) + val values = rows.map(r => s""""${r._2}"""") + s"""pd.DataFrame({ + | "$key": col([${keys.mkString(", ")}]), + | "$value": [${values.mkString(", ")}], + |})""".stripMargin + } + val driver = + s"""import pandas as pd, numpy as np + | + |def col(vals): + | return pd.arrays.FloatingArray( + | np.array([0.0 if v is None else v for v in vals]), + | np.array([v is None for v in vals]), + | ) + | + |in1df = ${frame(leftRows, "k", "lv")} + |in2df = ${frame(rightRows, "k2", "rv")} + |${d.generateStandaloneCode(Map(PortIdentity() -> left, PortIdentity(1) -> right))} + |def show(v): + | if isinstance(v, float): + | return "nan" if v != v else repr(float(v)) + | return str(v) + |rows = [] + |for values, missing in zip(out1df.itertuples(index=False), out1df.isna().to_numpy()): + | rows.append(" ".join("None" if m else show(v) for v, m in zip(values, missing))) + |print("\\n".join(sorted(rows))) + |""".stripMargin + + val script = Files.createTempFile("hashjoin-nan-zero-", ".py") + script.toFile.deleteOnExit() + Files.write(script, driver.getBytes(StandardCharsets.UTF_8)) + val process = new ProcessBuilder(python, script.toString).redirectErrorStream(true).start() + val out = Source.fromInputStream(process.getInputStream).mkString + process.waitFor(120, TimeUnit.SECONDS) + withClue(s"python said:\n$out\nscript:\n$driver") { + process.exitValue() shouldBe 0 + out.trim.linesIterator.toList shouldBe engine + } + } + } + // Without a schema nothing says which columns were integers. it should "leave the widening alone when no schema is given" in { val d = new HashJoinOpDesc[Integer] From c59c02ff715709fb6d05a280d3bd06383e6ea19a Mon Sep 17 00:00:00 2001 From: kary zheng Date: Wed, 14 Oct 2026 16:47:27 -0700 Subject: [PATCH 23/32] fix(operator): compare interval join timestamps by the millisecond as the engine does IntervalJoinOpExec compares a timestamp by getTime, so two moments in the same millisecond are one point to it, and a left key just past the upper bound but inside its millisecond still joins. The export compared the full resolution and dropped those rows. Floor the two keys and the bound to the millisecond for the comparison, adding the bound before it is cut. Co-Authored-By: Claude Opus 5.5 --- .../intervalJoin/IntervalJoinOpDesc.scala | 6 +- .../intervalJoin/IntervalJoinOpDescSpec.scala | 67 ++++++++++++++++++- 2 files changed, 71 insertions(+), 2 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDesc.scala index 786ecda1e7e..f5df47b3f37 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDesc.scala @@ -176,7 +176,11 @@ class IntervalJoinOpDesc extends LogicalOp with StandaloneCodeGenerator { |_iv_l = _pairs[$leftLit] |_iv_r = _pairs[$rightLit if $rightLit not in in1df.columns else $rightLit + "#@1"] |if pd.api.types.is_datetime64_any_dtype(_iv_r): - | _iv_hi = _iv_r + pd.DateOffset($offsetUnit=$constant) + | # The engine compares by getTime, so a moment counts only to the + | # millisecond. The bound is added before it is cut, as there. + | _iv_hi = (_iv_r + pd.DateOffset($offsetUnit=$constant)).dt.floor("ms") + | _iv_l = _iv_l.dt.floor("ms") + | _iv_r = _iv_r.dt.floor("ms") |else: | _iv_hi = _iv_r + $constant |_iv_match = (_iv_l $loOp _iv_r) & (_iv_l $hiOp _iv_hi) diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDescSpec.scala index 6237a3cf0f3..d0abbf3be2b 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDescSpec.scala @@ -21,7 +21,7 @@ package org.apache.texera.amber.operator.intervalJoin import com.typesafe.config.ConfigFactory import org.apache.texera.amber.core.executor.OpExecWithClassName -import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema} +import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema, Tuple} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.{HashPartition, PortIdentity} import org.apache.texera.amber.operator.LogicalOp @@ -34,6 +34,7 @@ import org.scalatest.matchers.should.Matchers import java.nio.charset.StandardCharsets import java.nio.file.Files +import java.sql.Timestamp import java.util.concurrent.TimeUnit import scala.io.Source import scala.util.Try @@ -178,6 +179,70 @@ class IntervalJoinOpDescSpec extends AnyFlatSpec with Matchers { } } + // IntervalJoinOpExec compares timestamps by getTime, so only whole + // milliseconds count. The right key is 12:00:00.0009 with a three-second + // interval: the first left row shares its millisecond and the last shares + // the upper bound's while past it, so each joins only at that resolution. + private val millisecondLefts = + Seq("2024-01-01 12:00:00.0005", "2024-01-01 12:00:02", "2024-01-01 12:00:03.00095") + private val millisecondRight = "2024-01-01 12:00:00.0009" + + it should "compare timestamps by the millisecond as the engine does" taggedAs NeedsPythonPackages in { + val python = resolvePython().getOrElse( + cancel("No runnable python executable (udf.conf python.path, python3, python, py)") + ) + if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + + val left = Schema().add(new Attribute("l", AttributeType.TIMESTAMP)) + val right = Schema().add(new Attribute("r", AttributeType.TIMESTAMP)) + + def check(includeBounds: Boolean): Unit = { + val d = + new IntervalJoinOpDesc("l", "r", 3L, includeBounds, includeBounds, TimeIntervalType.SECOND) + + val exec = new IntervalJoinOpExec(objectMapper.writeValueAsString(d)) + exec.open() + val engine = + try { + exec.processTuple(Tuple(right, Array[Any](Timestamp.valueOf(millisecondRight))), 1) + millisecondLefts.flatMap { l => + exec + .processTuple(Tuple(left, Array[Any](Timestamp.valueOf(l))), 0) + .map(_.getFields.toSeq) + .map(_.map(_.asInstanceOf[Timestamp].toLocalDateTime.toString).mkString(" ")) + } + } finally exec.close() + + def literal(t: String) = "\"" + t.replace(' ', 'T') + "\"" + val lefts = millisecondLefts.map(literal).mkString("[", ", ", "]") + val driver = + s"""import numpy as np + |import pandas as pd + | + |in1df = pd.DataFrame({"l": np.array($lefts, dtype="datetime64[us]")}) + |in2df = pd.DataFrame({"r": np.array([${literal(millisecondRight)}], dtype="datetime64[us]")}) + |${d.generateStandaloneCode()} + |for row in out1df.itertuples(index=False): + | print(" ".join(pd.Timestamp(v).isoformat() for v in row)) + |""".stripMargin + + val script = Files.createTempFile("intervaljoin-millis-", ".py") + script.toFile.deleteOnExit() + Files.write(script, driver.getBytes(StandardCharsets.UTF_8)) + val process = new ProcessBuilder(python, script.toString).redirectErrorStream(true).start() + val out = Source.fromInputStream(process.getInputStream).mkString + process.waitFor(120, TimeUnit.SECONDS) + withClue(s"python said:\n$out\nscript:\n$driver") { + process.exitValue() shouldBe 0 + out.trim.linesIterator.toSeq shouldBe engine + } + if (includeBounds) engine should have size 3 + } + + check(includeBounds = true) + check(includeBounds = false) + } + "IntervalJoinOpDesc" should "round-trip its fields through the polymorphic base" in { val d = configured() d.constant = 42L From f4004e8a0da82c6f49050a12d5f7a1cf259169c6 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Thu, 15 Oct 2026 10:23:57 -0700 Subject: [PATCH 24/32] fix(operator): add Interval Join's constant in the key's declared type, as the engine does An INTEGER bound is an Int sum: the engine cuts the constant to 32 bits and the sum wraps past Int.MaxValue, where pandas held the column as int64 and did neither. Co-Authored-By: Claude Opus 5.5 --- .../intervalJoin/IntervalJoinOpDesc.scala | 22 ++++++- .../intervalJoin/IntervalJoinOpDescSpec.scala | 65 +++++++++++++++++++ 2 files changed, 84 insertions(+), 3 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDesc.scala index f5df47b3f37..3c4198be990 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDesc.scala @@ -24,7 +24,7 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle} import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.state.NoLoopVariable -import org.apache.texera.amber.core.tuple.{Attribute, Schema} +import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow._ import org.apache.texera.amber.operator.{LogicalOp, StandaloneCodeGenerator} @@ -159,7 +159,9 @@ class IntervalJoinOpDesc extends LogicalOp with StandaloneCodeGenerator { // happens to carry that name and then drop it. Only the right key needs // locating: the merge leaves every left name alone and suffixes a right one // that collides. - override def generateStandaloneCode(): String = { + override def generateStandaloneCode(): String = generateStandaloneCode(Map.empty) + + override def generateStandaloneCode(inputSchemas: Map[PortIdentity, Schema]): String = { val leftLit = objectMapper.writeValueAsString(leftAttributeName) val rightLit = objectMapper.writeValueAsString(rightAttributeName) val loOp = if (includeLeftBound) ">=" else ">" @@ -172,6 +174,20 @@ class IntervalJoinOpDesc extends LogicalOp with StandaloneCodeGenerator { case Some(TimeIntervalType.SECOND) => "seconds" case _ => "days" // DAY or unset } + // The engine adds the constant in the right key's declared type. An INTEGER + // bound is an Int sum, so the constant is cut to 32 bits and the sum wraps + // past Int.MaxValue, where pandas holds the column as int64 and does + // neither. A LONG wraps in int64 as it does in Java. The key is widened to + // float64 first, which holds every such sum exactly, since NumPy refuses + // 2**31 against an int32 column. + val declared = inputSchemas + .get(operatorInfo.inputPorts.last.id) + .flatMap(schema => scala.util.Try(schema.getAttribute(rightAttributeName)).toOption) + .map(_.getType) + val numericBound = + if (declared.contains(AttributeType.INTEGER)) + s"""(_iv_r.astype("float64") + ${constant.toInt} + 2**31) % 2**32 - 2**31""" + else s"_iv_r + $constant" s"""_pairs = in1df.merge(in2df, how="cross", suffixes=("", "#@1")) |_iv_l = _pairs[$leftLit] |_iv_r = _pairs[$rightLit if $rightLit not in in1df.columns else $rightLit + "#@1"] @@ -182,7 +198,7 @@ class IntervalJoinOpDesc extends LogicalOp with StandaloneCodeGenerator { | _iv_l = _iv_l.dt.floor("ms") | _iv_r = _iv_r.dt.floor("ms") |else: - | _iv_hi = _iv_r + $constant + | _iv_hi = $numericBound |_iv_match = (_iv_l $loOp _iv_r) & (_iv_l $hiOp _iv_hi) |out1df = _pairs[_iv_match].reset_index(drop=True)""".stripMargin } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDescSpec.scala index d0abbf3be2b..dc3f19482de 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDescSpec.scala @@ -243,6 +243,71 @@ class IntervalJoinOpDescSpec extends AnyFlatSpec with Matchers { check(includeBounds = false) } + // The engine adds the constant in the key's declared type: an INTEGER bound + // is an Int sum, with the constant cut to 32 bits, and wraps past + // Int.MaxValue, and a LONG one wraps past Long.MaxValue. Each case feeds the + // right row first and the left rows in ascending order, as the executor + // assumes. + it should "add the constant in the key's declared type as the engine does" taggedAs NeedsPythonPackages in { + val python = resolvePython().getOrElse( + cancel("No runnable python executable (udf.conf python.path, python3, python, py)") + ) + if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + + val cases = Seq( + (AttributeType.INTEGER, 10L, 0L, Seq(3L, 20L)), + (AttributeType.INTEGER, 10L, 2147483640L, Seq(2147483645L)), + (AttributeType.INTEGER, 4294967306L, 0L, Seq(5L, 20L)), + (AttributeType.LONG, 10L, 9223372036854775800L, Seq(9223372036854775805L)) + ) + cases.foreach { + case (attrType, constant, right, lefts) => + val dtype = if (attrType == AttributeType.INTEGER) "int32" else "int64" + def value(v: Long): Any = if (attrType == AttributeType.INTEGER) v.toInt else v + val leftSchema = Schema().add(new Attribute("l", attrType)) + val rightSchema = Schema().add(new Attribute("r", attrType)) + val d = new IntervalJoinOpDesc("l", "r", constant, true, true, TimeIntervalType.DAY) + + val exec = new IntervalJoinOpExec(objectMapper.writeValueAsString(d)) + exec.open() + val engine = + try { + exec.processTuple(Tuple(rightSchema, Array[Any](value(right))), 1) + lefts.flatMap { l => + exec + .processTuple(Tuple(leftSchema, Array[Any](value(l))), 0) + .map(_.getFields.mkString(" ")) + } + } finally exec.close() + + val code = d.generateStandaloneCode( + Map(PortIdentity() -> leftSchema, PortIdentity(1) -> rightSchema) + ) + val driver = + s"""import numpy as np + |import pandas as pd + | + |in1df = pd.DataFrame({"l": np.array(${lefts + .mkString("[", ", ", "]")}, dtype="$dtype")}) + |in2df = pd.DataFrame({"r": np.array([$right], dtype="$dtype")}) + |$code + |for row in out1df.itertuples(index=False): + | print(" ".join(str(int(v)) for v in row)) + |""".stripMargin + + val script = Files.createTempFile("intervaljoin-bound-", ".py") + script.toFile.deleteOnExit() + Files.write(script, driver.getBytes(StandardCharsets.UTF_8)) + val process = new ProcessBuilder(python, script.toString).redirectErrorStream(true).start() + val out = Source.fromInputStream(process.getInputStream).mkString + process.waitFor(120, TimeUnit.SECONDS) + withClue(s"$attrType constant $constant, python said:\n$out\nscript:\n$driver") { + process.exitValue() shouldBe 0 + out.trim.linesIterator.filter(_.nonEmpty).toSeq shouldBe engine + } + } + } + "IntervalJoinOpDesc" should "round-trip its fields through the polymorphic base" in { val d = configured() d.constant = 42L From 2036f159ae2a75e096ebb327620b8b985c385909 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Thu, 15 Oct 2026 14:26:19 -0700 Subject: [PATCH 25/32] fix(operator): add Interval Join's time interval on the machine's wall clock, as the engine does The executor adds the constant to toLocalDateTime and reads the sum back with Timestamp.valueOf, so a day across a daylight-saving change is 23 or 25 hours and a month counts from the local date. The export added it to the UTC wall clock. Co-Authored-By: Claude Opus 5.5 --- .../intervalJoin/IntervalJoinOpDesc.scala | 51 +++++++++- .../intervalJoin/IntervalJoinOpDescSpec.scala | 96 +++++++++++++++++++ 2 files changed, 145 insertions(+), 2 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDesc.scala index 3c4198be990..9c933407b78 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDesc.scala @@ -27,7 +27,7 @@ import org.apache.texera.amber.core.state.NoLoopVariable import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow._ -import org.apache.texera.amber.operator.{LogicalOp, StandaloneCodeGenerator} +import org.apache.texera.amber.operator.{LogicalOp, StandaloneCodeGenerator, StandaloneHelpers} import org.apache.texera.amber.operator.metadata.annotations.{ AutofillAttributeName, AutofillAttributeNameOnPort1 @@ -194,7 +194,7 @@ class IntervalJoinOpDesc extends LogicalOp with StandaloneCodeGenerator { |if pd.api.types.is_datetime64_any_dtype(_iv_r): | # The engine compares by getTime, so a moment counts only to the | # millisecond. The bound is added before it is cut, as there. - | _iv_hi = (_iv_r + pd.DateOffset($offsetUnit=$constant)).dt.floor("ms") + | _iv_hi = _texera_interval_upper(_iv_r, pd.DateOffset($offsetUnit=$constant)).dt.floor("ms") | _iv_l = _iv_l.dt.floor("ms") | _iv_r = _iv_r.dt.floor("ms") |else: @@ -203,6 +203,9 @@ class IntervalJoinOpDesc extends LogicalOp with StandaloneCodeGenerator { |out1df = _pairs[_iv_match].reset_index(drop=True)""".stripMargin } + override def standaloneHelpers(): Seq[String] = + Seq(StandaloneHelpers.AttributeCasts, IntervalJoinOpDesc.IntervalUpper) + def this( leftTableAttributeName: String, rightTableAttributeName: String, @@ -221,3 +224,47 @@ class IntervalJoinOpDesc extends LogicalOp with StandaloneCodeGenerator { } } + +object IntervalJoinOpDesc { + + /** The upper bound of each interval, as the executor computes it: + * `Timestamp.valueOf(t.toLocalDateTime.plusX(constant))`. The constant is + * added on the machine's wall clock, so a day across a daylight-saving + * change is 23 or 25 hours, and a month counts from the local date. A wall + * clock that falls in a gap or an overlap is read at the standard offset, + * as java.util.Date reads it, and one before 1900 at the standard offset + * today. Each distinct key is worked out once, since the column holds every + * pair of the cross join. + */ + val IntervalUpper: String = + """def _texera_interval_upper(s, offset): + | from dateutil.tz import tzlocal + | + | zone = _texera_local_zone() + | standard = _texera_standard_offset(zone) + | moved = _texera_local_wall_clocks(s.dt.tz_localize("UTC")) + offset + | seen = {} + | + | def instant(wall): + | if pd.isna(wall): + | return pd.NaT + | if wall not in seen: + | moment = wall.to_pydatetime(warn=False) + | if moment.year < 1900: + | seen[wall] = pd.Timestamp(moment - standard) + | return seen[wall] + | tz = zone if zone is not None else tzlocal() + | try: + | early = moment.replace(tzinfo=tz, fold=0) + | late = moment.replace(tzinfo=tz, fold=1) + | if early.utcoffset() == late.utcoffset() or not early.dst(): + | at = early.utcoffset() + | else: + | at = late.utcoffset() + | except (OSError, OverflowError): + | at = standard + | seen[wall] = pd.Timestamp(moment - at) + | return seen[wall] + | + | return pd.Series([instant(w) for w in moved], index=s.index, dtype="datetime64[us]")""".stripMargin +} diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDescSpec.scala index dc3f19482de..13039285daf 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDescSpec.scala @@ -221,6 +221,7 @@ class IntervalJoinOpDescSpec extends AnyFlatSpec with Matchers { | |in1df = pd.DataFrame({"l": np.array($lefts, dtype="datetime64[us]")}) |in2df = pd.DataFrame({"r": np.array([${literal(millisecondRight)}], dtype="datetime64[us]")}) + |${d.standaloneHelpers().mkString("\n\n")} |${d.generateStandaloneCode()} |for row in out1df.itertuples(index=False): | print(" ".join(pd.Timestamp(v).isoformat() for v in row)) @@ -243,6 +244,101 @@ class IntervalJoinOpDescSpec extends AnyFlatSpec with Matchers { check(includeBounds = false) } + // The executor adds the constant on the machine's wall clock and reads the + // result back in its zone, so in Chicago a day across the spring change is + // 23 hours, one across the autumn change is 25 (the overlap read at the + // standard offset), a day into the spring gap is 24, and a month from + // 2024-01-31T00:00Z counts from the local 01-30 and ends on 03-01. Each + // case puts one left row on the engine's bound and one just past the bound + // a plain UTC offset gives. The script runs with TZ set to the same zone. + it should "add a time interval on the machine's wall clock as the engine does" taggedAs NeedsPythonPackages in { + val python = resolvePython().getOrElse( + cancel("No runnable python executable (udf.conf python.path, python3, python, py)") + ) + if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + + val zone = "America/Chicago" + val cases = Seq( + ( + TimeIntervalType.DAY, + "2024-03-09T18:00:00Z", + Seq("2024-03-10T17:00:00Z", "2024-03-10T17:30:00Z") + ), + ( + TimeIntervalType.DAY, + "2024-11-02T06:30:00Z", + Seq("2024-11-03T07:15:00Z", "2024-11-03T07:45:00Z") + ), + ( + TimeIntervalType.DAY, + "2024-03-09T08:30:00Z", + Seq("2024-03-10T08:30:00Z", "2024-03-10T08:31:00Z") + ), + ( + TimeIntervalType.MONTH, + "2024-01-31T00:00:00Z", + Seq("2024-02-29T12:00:00Z", "2024-03-01T00:01:00Z") + ), + ( + TimeIntervalType.DAY, + "1850-06-01T12:00:00Z", + Seq("1850-06-02T12:00:00Z", "1850-06-02T12:01:00Z") + ) + ) + val left = Schema().add(new Attribute("l", AttributeType.TIMESTAMP)) + val right = Schema().add(new Attribute("r", AttributeType.TIMESTAMP)) + def at(instant: String) = Timestamp.from(java.time.Instant.parse(instant)) + + val saved = java.util.TimeZone.getDefault + java.util.TimeZone.setDefault(java.util.TimeZone.getTimeZone(zone)) + try { + cases.foreach { + case (unit, r, lefts) => + val d = new IntervalJoinOpDesc("l", "r", 1L, true, true, unit) + val exec = new IntervalJoinOpExec(objectMapper.writeValueAsString(d)) + exec.open() + val engine = + try { + exec.processTuple(Tuple(right, Array[Any](at(r))), 1) + lefts.flatMap { l => + exec + .processTuple(Tuple(left, Array[Any](at(l))), 0) + .map(_.getFields.map(_.asInstanceOf[Timestamp].getTime).mkString(" ")) + } + } finally exec.close() + + def literal(t: String) = "\"" + t.stripSuffix("Z") + "\"" + val driver = + s"""import numpy as np + |import pandas as pd + | + |in1df = pd.DataFrame({"l": np.array(${lefts + .map(literal) + .mkString("[", ", ", "]")}, dtype="datetime64[us]")}) + |in2df = pd.DataFrame({"r": np.array([${literal(r)}], dtype="datetime64[us]")}) + |${d.standaloneHelpers().mkString("\n\n")} + |${d.generateStandaloneCode()} + |for row in out1df.itertuples(index=False): + | print(" ".join(str((pd.Timestamp(v) - pd.Timestamp(0)) // pd.Timedelta(milliseconds=1)) for v in row)) + |""".stripMargin + + val script = Files.createTempFile("intervaljoin-zone-", ".py") + script.toFile.deleteOnExit() + Files.write(script, driver.getBytes(StandardCharsets.UTF_8)) + val builder = new ProcessBuilder(python, script.toString).redirectErrorStream(true) + builder.environment().put("TZ", zone) + val process = builder.start() + val out = Source.fromInputStream(process.getInputStream).mkString + process.waitFor(120, TimeUnit.SECONDS) + withClue(s"$unit from $r, python said:\n$out\nscript:\n$driver") { + process.exitValue() shouldBe 0 + out.trim.linesIterator.filter(_.nonEmpty).toSeq shouldBe engine + engine should have size 1 + } + } + } finally java.util.TimeZone.setDefault(saved) + } + // The engine adds the constant in the key's declared type: an INTEGER bound // is an Int sum, with the constant cut to 32 bits, and wraps past // Int.MaxValue, and a LONG one wraps past Long.MaxValue. Each case feeds the From b9620beb7af8e617cf47791f98cc8d822e74d5b5 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Fri, 23 Oct 2026 14:43:47 -0700 Subject: [PATCH 26/32] fix(operator): add Interval Join's interval to the wall clock the script is handed A source already hands the script a timestamp as the machine's wall clock, so labelling it UTC and converting it again moved the bound by the local offset. The interval is now added to that wall clock as it stands, and the keys and the bound are read back into instants as Timestamp.valueOf reads them. Co-Authored-By: Claude Opus 5.5 --- .../intervalJoin/IntervalJoinOpDesc.scala | 39 +++++++------ .../intervalJoin/IntervalJoinOpDescSpec.scala | 57 +++++++------------ 2 files changed, 42 insertions(+), 54 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDesc.scala index 9c933407b78..ac4fa6dd2bc 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDesc.scala @@ -192,11 +192,12 @@ class IntervalJoinOpDesc extends LogicalOp with StandaloneCodeGenerator { |_iv_l = _pairs[$leftLit] |_iv_r = _pairs[$rightLit if $rightLit not in in1df.columns else $rightLit + "#@1"] |if pd.api.types.is_datetime64_any_dtype(_iv_r): - | # The engine compares by getTime, so a moment counts only to the - | # millisecond. The bound is added before it is cut, as there. - | _iv_hi = _texera_interval_upper(_iv_r, pd.DateOffset($offsetUnit=$constant)).dt.floor("ms") - | _iv_l = _iv_l.dt.floor("ms") - | _iv_r = _iv_r.dt.floor("ms") + | # The keys arrive as wall clocks, and the engine adds the constant to + | # the wall clock. It compares by getTime, so a moment counts only to + | # the millisecond. The bound is added before it is cut, as there. + | _iv_hi = _texera_wall_clock_instants(_iv_r + pd.DateOffset($offsetUnit=$constant)).dt.floor("ms") + | _iv_l = _texera_wall_clock_instants(_iv_l).dt.floor("ms") + | _iv_r = _texera_wall_clock_instants(_iv_r).dt.floor("ms") |else: | _iv_hi = $numericBound |_iv_match = (_iv_l $loOp _iv_r) & (_iv_l $hiOp _iv_hi) @@ -204,7 +205,7 @@ class IntervalJoinOpDesc extends LogicalOp with StandaloneCodeGenerator { } override def standaloneHelpers(): Seq[String] = - Seq(StandaloneHelpers.AttributeCasts, IntervalJoinOpDesc.IntervalUpper) + Seq(StandaloneHelpers.AttributeCasts, IntervalJoinOpDesc.WallClockInstants) def this( leftTableAttributeName: String, @@ -227,22 +228,24 @@ class IntervalJoinOpDesc extends LogicalOp with StandaloneCodeGenerator { object IntervalJoinOpDesc { - /** The upper bound of each interval, as the executor computes it: - * `Timestamp.valueOf(t.toLocalDateTime.plusX(constant))`. The constant is - * added on the machine's wall clock, so a day across a daylight-saving - * change is 23 or 25 hours, and a month counts from the local date. A wall - * clock that falls in a gap or an overlap is read at the standard offset, - * as java.util.Date reads it, and one before 1900 at the standard offset - * today. Each distinct key is worked out once, since the column holds every - * pair of the cross join. + /** The instant the executor holds for each wall clock in a column. A source + * hands the script a timestamp as the wall clock the machine's zone shows, + * and the executor's upper bound is + * `Timestamp.valueOf(t.toLocalDateTime.plusX(constant))`, so the constant + * is added to that wall clock as it stands. A day across a daylight-saving + * change is then 23 or 25 hours, and a month counts from the local date. + * The comparison is by getTime, so each wall clock is read back into an + * instant before it. One that falls in a gap or an overlap is read at the + * standard offset, as java.util.Date reads it, and one before 1900 at the + * standard offset today. Each distinct key is worked out once, since the + * column holds every pair of the cross join. */ - val IntervalUpper: String = - """def _texera_interval_upper(s, offset): + val WallClockInstants: String = + """def _texera_wall_clock_instants(s): | from dateutil.tz import tzlocal | | zone = _texera_local_zone() | standard = _texera_standard_offset(zone) - | moved = _texera_local_wall_clocks(s.dt.tz_localize("UTC")) + offset | seen = {} | | def instant(wall): @@ -266,5 +269,5 @@ object IntervalJoinOpDesc { | seen[wall] = pd.Timestamp(moment - at) | return seen[wall] | - | return pd.Series([instant(w) for w in moved], index=s.index, dtype="datetime64[us]")""".stripMargin + | return pd.Series([instant(w) for w in s], index=s.index, dtype="datetime64[us]")""".stripMargin } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDescSpec.scala index 13039285daf..aac64c1ba41 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDescSpec.scala @@ -244,13 +244,14 @@ class IntervalJoinOpDescSpec extends AnyFlatSpec with Matchers { check(includeBounds = false) } - // The executor adds the constant on the machine's wall clock and reads the - // result back in its zone, so in Chicago a day across the spring change is - // 23 hours, one across the autumn change is 25 (the overlap read at the - // standard offset), a day into the spring gap is 24, and a month from - // 2024-01-31T00:00Z counts from the local 01-30 and ends on 03-01. Each - // case puts one left row on the engine's bound and one just past the bound - // a plain UTC offset gives. The script runs with TZ set to the same zone. + // A source hands the script each timestamp as the wall clock the machine's + // zone shows, with no zone attached, and the executor adds the constant to + // that wall clock. So in Chicago a day from 03-09 12:00 ends at 03-10 12:00 + // across the spring change, and the same holds across the autumn one. A day + // into the spring gap ends where Timestamp.valueOf reads 02:30, an hour + // later, and a month from 01-31 ends on 02-29. Each case puts one left row + // on or before the engine's bound and one past it. Both sides are fed the + // same wall clocks, and the script runs with TZ set to the same zone. it should "add a time interval on the machine's wall clock as the engine does" taggedAs NeedsPythonPackages in { val python = resolvePython().getOrElse( cancel("No runnable python executable (udf.conf python.path, python3, python, py)") @@ -259,35 +260,19 @@ class IntervalJoinOpDescSpec extends AnyFlatSpec with Matchers { val zone = "America/Chicago" val cases = Seq( - ( - TimeIntervalType.DAY, - "2024-03-09T18:00:00Z", - Seq("2024-03-10T17:00:00Z", "2024-03-10T17:30:00Z") - ), - ( - TimeIntervalType.DAY, - "2024-11-02T06:30:00Z", - Seq("2024-11-03T07:15:00Z", "2024-11-03T07:45:00Z") - ), - ( - TimeIntervalType.DAY, - "2024-03-09T08:30:00Z", - Seq("2024-03-10T08:30:00Z", "2024-03-10T08:31:00Z") - ), - ( - TimeIntervalType.MONTH, - "2024-01-31T00:00:00Z", - Seq("2024-02-29T12:00:00Z", "2024-03-01T00:01:00Z") - ), - ( - TimeIntervalType.DAY, - "1850-06-01T12:00:00Z", - Seq("1850-06-02T12:00:00Z", "1850-06-02T12:01:00Z") - ) + (TimeIntervalType.DAY, "2024-06-01T12:00", Seq("2024-06-02T12:00", "2024-06-02T12:01")), + (TimeIntervalType.DAY, "2024-03-09T12:00", Seq("2024-03-10T11:30", "2024-03-10T12:30")), + (TimeIntervalType.DAY, "2024-11-02T12:00", Seq("2024-11-03T11:30", "2024-11-03T12:30")), + (TimeIntervalType.DAY, "2024-03-09T02:30", Seq("2024-03-10T03:15", "2024-03-10T03:45")), + (TimeIntervalType.DAY, "2024-11-02T01:30", Seq("2024-11-03T01:15", "2024-11-03T01:45")), + (TimeIntervalType.MONTH, "2024-01-31T00:00", Seq("2024-02-29T00:00", "2024-02-29T00:01")), + (TimeIntervalType.DAY, "1850-06-01T12:00", Seq("1850-06-02T12:00", "1850-06-02T12:01")) ) val left = Schema().add(new Attribute("l", AttributeType.TIMESTAMP)) val right = Schema().add(new Attribute("r", AttributeType.TIMESTAMP)) - def at(instant: String) = Timestamp.from(java.time.Instant.parse(instant)) + def at(wall: String) = Timestamp.valueOf(java.time.LocalDateTime.parse(wall)) + def wall(t: Timestamp) = + t.toLocalDateTime.format(java.time.format.DateTimeFormatter.ISO_LOCAL_DATE_TIME) val saved = java.util.TimeZone.getDefault java.util.TimeZone.setDefault(java.util.TimeZone.getTimeZone(zone)) @@ -303,11 +288,11 @@ class IntervalJoinOpDescSpec extends AnyFlatSpec with Matchers { lefts.flatMap { l => exec .processTuple(Tuple(left, Array[Any](at(l))), 0) - .map(_.getFields.map(_.asInstanceOf[Timestamp].getTime).mkString(" ")) + .map(_.getFields.map(v => wall(v.asInstanceOf[Timestamp])).mkString(" ")) } } finally exec.close() - def literal(t: String) = "\"" + t.stripSuffix("Z") + "\"" + def literal(t: String) = "\"" + t + "\"" val driver = s"""import numpy as np |import pandas as pd @@ -319,7 +304,7 @@ class IntervalJoinOpDescSpec extends AnyFlatSpec with Matchers { |${d.standaloneHelpers().mkString("\n\n")} |${d.generateStandaloneCode()} |for row in out1df.itertuples(index=False): - | print(" ".join(str((pd.Timestamp(v) - pd.Timestamp(0)) // pd.Timedelta(milliseconds=1)) for v in row)) + | print(" ".join(pd.Timestamp(v).isoformat() for v in row)) |""".stripMargin val script = Files.createTempFile("intervaljoin-zone-", ".py") From 16d13e1fd60f74f7410a542b82f744dac7a43b46 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Fri, 23 Oct 2026 15:31:46 -0700 Subject: [PATCH 27/32] feat(workflow-operator): keep the set and join helpers' notes out of the exported script _texera_set_row_keys, _texera_set_select and _texera_join_codes explained themselves in Python comments, which every exported script with a set operator or a Hash Join printed. The notes now sit in Scala, one line per function. Co-Authored-By: Claude Opus 5.5 --- .../texera/amber/operator/StandaloneHelpers.scala | 15 +++++---------- .../amber/operator/hashJoin/HashJoinOpDesc.scala | 4 +--- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneHelpers.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneHelpers.scala index 6e26bbb7b03..d2c449af579 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneHelpers.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneHelpers.scala @@ -25,18 +25,15 @@ package org.apache.texera.amber.operator object StandaloneHelpers { /** - * Row equality as the set operators' `HashSet[Tuple]` decides it, for - * Intersect, Difference and Symmetric Difference. + * Row equality as the set operators' `HashSet[Tuple]` decides it, for Intersect, Difference + * and Symmetric Difference: NaN equals nothing and 0.0 differs from -0.0, where pandas counts + * both pairs as equal. * - * Two rows meet only when both their hashes and `equals` agree. `equals` is - * Scala's `==` on each field, under which NaN equals nothing, not even a - * copy of its own row. The hash is Java's, from each double's bits, so 0.0 - * and -0.0 land apart. pandas' duplicate checks count both pairs as equal. + * - `_texera_set_row_keys`: one key per row, equal exactly where the engine's rows are. + * - `_texera_set_select`: the first row of each key, among those `keep` accepts. */ val SetRowKeys: String = """def _texera_set_row_keys(df): - | # One key per row, equal exactly where the engine's rows are. A null - | # is read off the column's mask, since a stored NaN is not one. | import math | | def key(v, missing): @@ -57,8 +54,6 @@ object StandaloneHelpers { | | |def _texera_set_select(df, keep): - | # The rows the engine's HashSet holds, the first of each key, that - | # `keep` accepts. | seen = set() | picked = [] | for i, key in enumerate(_texera_set_row_keys(df)): diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDesc.scala index 724f067bd3e..4b71fcfddc0 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDesc.scala @@ -271,7 +271,7 @@ class HashJoinOpDesc[K] extends LogicalOp with StandaloneCodeGenerator { // The merge joins on codes and not on the keys. The probe looks each key up // in a HashMap under Scala's ==, where a NaN finds nothing, not even another // NaN, while 0.0 meets -0.0 and a null meets a null; merge pairs NaN with - // NaN. + // NaN. `_texera_join_codes` gives each NaN a code of its own. val merge = s"""_left_cols = set(in1df.columns) |_right_cols = list(in2df.columns) @@ -295,8 +295,6 @@ class HashJoinOpDesc[K] extends LogicalOp with StandaloneCodeGenerator { | _join_key = _join_key + "_" | |def _texera_join_codes(left, right): - | # Each NaN takes a code of its own. A null is read off the column's - | # mask, since a stored NaN is not one. | import math | | codes = {} From d183c3152037dc02675f356cea3b055629634f03 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Fri, 23 Oct 2026 16:33:17 -0700 Subject: [PATCH 28/32] feat(workflow-operator): keep the sampling helper's notes out of the exported script Co-Authored-By: Claude Opus 5.5 --- .../texera/amber/operator/SamplingHelpers.scala | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/SamplingHelpers.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/SamplingHelpers.scala index 1fa9210898b..6c81befdffd 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/SamplingHelpers.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/SamplingHelpers.scala @@ -25,18 +25,12 @@ package org.apache.texera.amber.operator object SamplingHelpers { /** - * A Python transcription of `java.util.Random`, for operators whose executor - * draws from one. - * - * A sampler decides per row whether to keep it, so which rows survive is - * fixed by the exact sequence the generator produces. Seeding Python's - * `random` or numpy's with the engine's seed selects a different set, and - * the script would then report a different sample than the workflow it came - * from. Only the same generator gives the same rows. + * `_TexeraJavaRandom`: `java.util.Random` in Python, so a sampler keeps the rows the engine + * keeps. Python's or numpy's generator with the same seed picks other rows. `next_int` + * rejects a draw where Java's int sum overflows, which Python's integers never do. */ val JavaRandom: String = - """# java.util.Random, transcribed so sampling matches the engine. - |class _TexeraJavaRandom: + """class _TexeraJavaRandom: | _MASK = (1 << 48) - 1 | _MULTIPLIER = 0x5DEECE66D | _ADDEND = 0xB @@ -60,8 +54,6 @@ object SamplingHelpers { | while True: | bits = self._next(31) | value = bits % bound - | # Java rejects a draw by letting this sum overflow an int. - | # Python would carry it and never reject. | probe = bits - value + (bound - 1) | if ((probe + (1 << 31)) % (1 << 32)) - (1 << 31) >= 0: | return value""".stripMargin From 6856d5062ec6880edcc984ec187d62bf7be0a1f9 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Sun, 25 Oct 2026 03:14:35 -0700 Subject: [PATCH 29/32] feat(workflow-operator): read Interval Join's wall clocks with the shared helper The shared helper also reads a wall clock in the hours around 1900-01-01 at the offset the engine does, where the old one was a whole offset off. Co-Authored-By: Claude Opus 5.5 --- .../intervalJoin/IntervalJoinOpDesc.scala | 55 ++----------------- 1 file changed, 4 insertions(+), 51 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDesc.scala index ac4fa6dd2bc..9dfd0a97e2f 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDesc.scala @@ -195,17 +195,16 @@ class IntervalJoinOpDesc extends LogicalOp with StandaloneCodeGenerator { | # The keys arrive as wall clocks, and the engine adds the constant to | # the wall clock. It compares by getTime, so a moment counts only to | # the millisecond. The bound is added before it is cut, as there. - | _iv_hi = _texera_wall_clock_instants(_iv_r + pd.DateOffset($offsetUnit=$constant)).dt.floor("ms") - | _iv_l = _texera_wall_clock_instants(_iv_l).dt.floor("ms") - | _iv_r = _texera_wall_clock_instants(_iv_r).dt.floor("ms") + | _iv_hi = _texera_wall_clock_instants(_iv_r + pd.DateOffset($offsetUnit=$constant)) + | _iv_l = _texera_wall_clock_instants(_iv_l) + | _iv_r = _texera_wall_clock_instants(_iv_r) |else: | _iv_hi = $numericBound |_iv_match = (_iv_l $loOp _iv_r) & (_iv_l $hiOp _iv_hi) |out1df = _pairs[_iv_match].reset_index(drop=True)""".stripMargin } - override def standaloneHelpers(): Seq[String] = - Seq(StandaloneHelpers.AttributeCasts, IntervalJoinOpDesc.WallClockInstants) + override def standaloneHelpers(): Seq[String] = Seq(StandaloneHelpers.AttributeCasts) def this( leftTableAttributeName: String, @@ -225,49 +224,3 @@ class IntervalJoinOpDesc extends LogicalOp with StandaloneCodeGenerator { } } - -object IntervalJoinOpDesc { - - /** The instant the executor holds for each wall clock in a column. A source - * hands the script a timestamp as the wall clock the machine's zone shows, - * and the executor's upper bound is - * `Timestamp.valueOf(t.toLocalDateTime.plusX(constant))`, so the constant - * is added to that wall clock as it stands. A day across a daylight-saving - * change is then 23 or 25 hours, and a month counts from the local date. - * The comparison is by getTime, so each wall clock is read back into an - * instant before it. One that falls in a gap or an overlap is read at the - * standard offset, as java.util.Date reads it, and one before 1900 at the - * standard offset today. Each distinct key is worked out once, since the - * column holds every pair of the cross join. - */ - val WallClockInstants: String = - """def _texera_wall_clock_instants(s): - | from dateutil.tz import tzlocal - | - | zone = _texera_local_zone() - | standard = _texera_standard_offset(zone) - | seen = {} - | - | def instant(wall): - | if pd.isna(wall): - | return pd.NaT - | if wall not in seen: - | moment = wall.to_pydatetime(warn=False) - | if moment.year < 1900: - | seen[wall] = pd.Timestamp(moment - standard) - | return seen[wall] - | tz = zone if zone is not None else tzlocal() - | try: - | early = moment.replace(tzinfo=tz, fold=0) - | late = moment.replace(tzinfo=tz, fold=1) - | if early.utcoffset() == late.utcoffset() or not early.dst(): - | at = early.utcoffset() - | else: - | at = late.utcoffset() - | except (OSError, OverflowError): - | at = standard - | seen[wall] = pd.Timestamp(moment - at) - | return seen[wall] - | - | return pd.Series([instant(w) for w in s], index=s.index, dtype="datetime64[us]")""".stripMargin -} From 770bbdcb78cdf2abfc9006b3dfc7ec3305d070c1 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Sun, 25 Oct 2026 04:45:45 -0700 Subject: [PATCH 30/32] feat(workflow-operator): match Hash Join's keys with the shared key codes Co-Authored-By: Claude Opus 5.5 --- .../operator/hashJoin/HashJoinOpDesc.scala | 24 +++++-------------- .../hashJoin/HashJoinOpDescSpec.scala | 2 ++ 2 files changed, 8 insertions(+), 18 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDesc.scala index 4b71fcfddc0..f68c691724c 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDesc.scala @@ -30,7 +30,7 @@ import org.apache.texera.amber.core.virtualidentity.{ WorkflowIdentity } import org.apache.texera.amber.core.workflow._ -import org.apache.texera.amber.operator.{LogicalOp, StandaloneCodeGenerator} +import org.apache.texera.amber.operator.{LogicalOp, StandaloneCodeGenerator, StandaloneHelpers} import org.apache.texera.amber.operator.hashJoin.HashJoinOpDesc.HASH_JOIN_INTERNAL_KEY_NAME import org.apache.texera.amber.operator.metadata.annotations.{ AutofillAttributeName, @@ -228,6 +228,8 @@ class HashJoinOpDesc[K] extends LogicalOp with StandaloneCodeGenerator { override def generateStandaloneCode(): String = generateStandaloneCode(List(), List()) + override def standaloneHelpers(): Seq[String] = Seq(StandaloneHelpers.AttributeCasts) + private def generateStandaloneCode( leftIntegerColumns: List[String], rightIntegerColumns: List[String] @@ -271,7 +273,8 @@ class HashJoinOpDesc[K] extends LogicalOp with StandaloneCodeGenerator { // The merge joins on codes and not on the keys. The probe looks each key up // in a HashMap under Scala's ==, where a NaN finds nothing, not even another // NaN, while 0.0 meets -0.0 and a null meets a null; merge pairs NaN with - // NaN. `_texera_join_codes` gives each NaN a code of its own. + // NaN. `_texera_key_codes` from StandaloneHelpers.AttributeCasts gives each + // NaN a code of its own. val merge = s"""_left_cols = set(in1df.columns) |_right_cols = list(in2df.columns) @@ -293,22 +296,7 @@ class HashJoinOpDesc[K] extends LogicalOp with StandaloneCodeGenerator { |_join_key = "_texera_join_key" |while _join_key in _taken or _join_key == _probe_key: | _join_key = _join_key + "_" - | - |def _texera_join_codes(left, right): - | import math - | - | codes = {} - | - | def code(s): - | out = [] - | for v, missing in zip(s, s.isna().to_numpy()): - | key = None if missing else object() if isinstance(v, float) and math.isnan(v) else v - | out.append(codes.setdefault(key, len(codes))) - | return out - | - | return code(left), code(right) - | - |_left_codes, _right_codes = _texera_join_codes(in1df[$buildKeyLit], in2df[$probeKeyLit]) + |_left_codes, _right_codes = _texera_key_codes(in1df[[$buildKeyLit]], in2df[[$probeKeyLit]]) |""".stripMargin + widening + s"""out1df = $leftFrame.assign(**{_join_key: _left_codes}).merge( | $rightFrame.assign(**{_join_key: _right_codes}), diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDescSpec.scala index 3c2514c2f24..ae1ceae57db 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDescSpec.scala @@ -112,6 +112,7 @@ class HashJoinOpDescSpec extends AnyFlatSpec with Matchers { | |in1df = pd.DataFrame({"k": col([0.0, np.nan], [True, False]), "lv": [1, 2]}) |in2df = pd.DataFrame({"k": col([0.0], [True]), "rv": [9]}) + |${d.standaloneHelpers().mkString("\n")} |${d.generateStandaloneCode()} |print(len(out1df), int(out1df["rv"].notna().sum())) |""".stripMargin @@ -198,6 +199,7 @@ class HashJoinOpDescSpec extends AnyFlatSpec with Matchers { | |in1df = ${frame(leftRows, "k", "lv")} |in2df = ${frame(rightRows, "k2", "rv")} + |${d.standaloneHelpers().mkString("\n")} |${d.generateStandaloneCode(Map(PortIdentity() -> left, PortIdentity(1) -> right))} |def show(v): | if isinstance(v, float): From 24d5d9b0895b15200e3bc41afbed3bbfab5a8d34 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Sun, 25 Oct 2026 04:52:01 -0700 Subject: [PATCH 31/32] feat(workflow-operator): keep Interval Join's exported Python free of comments Co-Authored-By: Claude Opus 5.5 --- .../amber/operator/intervalJoin/IntervalJoinOpDesc.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDesc.scala index 9dfd0a97e2f..9f61db9d70b 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDesc.scala @@ -188,13 +188,13 @@ class IntervalJoinOpDesc extends LogicalOp with StandaloneCodeGenerator { if (declared.contains(AttributeType.INTEGER)) s"""(_iv_r.astype("float64") + ${constant.toInt} + 2**31) % 2**32 - 2**31""" else s"_iv_r + $constant" + // Timestamp keys arrive as wall clocks, and the engine adds the constant to + // the wall clock. It compares by getTime, so a moment counts only to the + // millisecond. The bound is added before it is cut, as there. s"""_pairs = in1df.merge(in2df, how="cross", suffixes=("", "#@1")) |_iv_l = _pairs[$leftLit] |_iv_r = _pairs[$rightLit if $rightLit not in in1df.columns else $rightLit + "#@1"] |if pd.api.types.is_datetime64_any_dtype(_iv_r): - | # The keys arrive as wall clocks, and the engine adds the constant to - | # the wall clock. It compares by getTime, so a moment counts only to - | # the millisecond. The bound is added before it is cut, as there. | _iv_hi = _texera_wall_clock_instants(_iv_r + pd.DateOffset($offsetUnit=$constant)) | _iv_l = _texera_wall_clock_instants(_iv_l) | _iv_r = _texera_wall_clock_instants(_iv_r) From 9d7468d0708ef9841c02d4c8a65d018d334ce389 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Mon, 26 Oct 2026 02:18:00 -0700 Subject: [PATCH 32/32] feat(workflow-operator): select the set operators' rows with the shared key codes Co-Authored-By: Claude Opus 5.5 --- .../amber/operator/StandaloneHelpers.scala | 38 ------------------- .../difference/DifferenceOpDesc.scala | 9 +++-- .../operator/intersect/IntersectOpDesc.scala | 10 +++-- .../SymmetricDifferenceOpDesc.scala | 15 ++++---- 4 files changed, 18 insertions(+), 54 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneHelpers.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneHelpers.scala index d2c449af579..1414f6b043c 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneHelpers.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneHelpers.scala @@ -24,44 +24,6 @@ package org.apache.texera.amber.operator */ object StandaloneHelpers { - /** - * Row equality as the set operators' `HashSet[Tuple]` decides it, for Intersect, Difference - * and Symmetric Difference: NaN equals nothing and 0.0 differs from -0.0, where pandas counts - * both pairs as equal. - * - * - `_texera_set_row_keys`: one key per row, equal exactly where the engine's rows are. - * - `_texera_set_select`: the first row of each key, among those `keep` accepts. - */ - val SetRowKeys: String = - """def _texera_set_row_keys(df): - | import math - | - | def key(v, missing): - | if missing: - | return None - | if isinstance(v, float): - | if math.isnan(v): - | return object() - | return (v, math.copysign(1.0, v)) - | return v - | - | return [ - | tuple(key(v, m) for v, m in zip(values, missing)) - | for values, missing in zip( - | df.itertuples(index=False, name=None), df.isna().to_numpy() - | ) - | ] - | - | - |def _texera_set_select(df, keep): - | seen = set() - | picked = [] - | for i, key in enumerate(_texera_set_row_keys(df)): - | if key not in seen and keep(key): - | picked.append(i) - | seen.add(key) - | return df.iloc[picked].reset_index(drop=True)""".stripMargin - /** * A Python transcription of `AttributeTypeUtils`, for operators that cast a * column to a declared type. diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/difference/DifferenceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/difference/DifferenceOpDesc.scala index b4d2b5f2b22..765939bb85f 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/difference/DifferenceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/difference/DifferenceOpDesc.scala @@ -62,10 +62,11 @@ class DifferenceOpDesc extends LogicalOp with StandaloneCodeGenerator { ) // Distinct left (port 0) rows the right (port 1) does not hold, by the - // engine's row equality. + // engine's row equality, which `_texera_key_codes` with `signed` numbers. override def generateStandaloneCode(): String = - """_texera_set_right = set(_texera_set_row_keys(in2df)) - |out1df = _texera_set_select(in1df, lambda key: key not in _texera_set_right)""".stripMargin + """_texera_set_left, _texera_set_right = _texera_key_codes(in1df, in2df, signed=True) + |_texera_set_kept = ~_texera_set_left.duplicated() & ~_texera_set_left.isin(_texera_set_right) + |out1df = in1df[_texera_set_kept.to_numpy()].reset_index(drop=True)""".stripMargin - override def standaloneHelpers(): Seq[String] = Seq(StandaloneHelpers.SetRowKeys) + override def standaloneHelpers(): Seq[String] = Seq(StandaloneHelpers.AttributeCasts) } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/intersect/IntersectOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/intersect/IntersectOpDesc.scala index 092d0a380f8..1a68b0f8cfc 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/intersect/IntersectOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/intersect/IntersectOpDesc.scala @@ -52,10 +52,12 @@ class IntersectOpDesc extends LogicalOp with StandaloneCodeGenerator { outputPorts = List(OutputPort(blocking = true)) ) - // Distinct left rows the right also holds, by the engine's row equality. + // Distinct left rows the right also holds, by the engine's row equality, which + // `_texera_key_codes` with `signed` numbers. override def generateStandaloneCode(): String = - """_texera_set_right = set(_texera_set_row_keys(in2df)) - |out1df = _texera_set_select(in1df, lambda key: key in _texera_set_right)""".stripMargin + """_texera_set_left, _texera_set_right = _texera_key_codes(in1df, in2df, signed=True) + |_texera_set_kept = ~_texera_set_left.duplicated() & _texera_set_left.isin(_texera_set_right) + |out1df = in1df[_texera_set_kept.to_numpy()].reset_index(drop=True)""".stripMargin - override def standaloneHelpers(): Seq[String] = Seq(StandaloneHelpers.SetRowKeys) + override def standaloneHelpers(): Seq[String] = Seq(StandaloneHelpers.AttributeCasts) } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/symmetricDifference/SymmetricDifferenceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/symmetricDifference/SymmetricDifferenceOpDesc.scala index de4699f6d01..e438ce6c608 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/symmetricDifference/SymmetricDifferenceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/symmetricDifference/SymmetricDifferenceOpDesc.scala @@ -61,17 +61,16 @@ class SymmetricDifferenceOpDesc extends LogicalOp with StandaloneCodeGenerator { outputPorts = List(OutputPort(blocking = true)) ) - // Distinct rows in exactly one input, by the engine's row equality. + // Distinct rows in exactly one input, by the engine's row equality, which + // `_texera_key_codes` with `signed` numbers. override def generateStandaloneCode(): String = - """_texera_set_left = set(_texera_set_row_keys(in1df)) - |_texera_set_right = set(_texera_set_row_keys(in2df)) + """_texera_set_left, _texera_set_right = _texera_key_codes(in1df, in2df, signed=True) + |_texera_set_left_kept = ~_texera_set_left.duplicated() & ~_texera_set_left.isin(_texera_set_right) + |_texera_set_right_kept = ~_texera_set_right.duplicated() & ~_texera_set_right.isin(_texera_set_left) |out1df = pd.concat( - | [ - | _texera_set_select(in1df, lambda key: key not in _texera_set_right), - | _texera_set_select(in2df, lambda key: key not in _texera_set_left), - | ], + | [in1df[_texera_set_left_kept.to_numpy()], in2df[_texera_set_right_kept.to_numpy()]], | ignore_index=True, |)""".stripMargin - override def standaloneHelpers(): Seq[String] = Seq(StandaloneHelpers.SetRowKeys) + override def standaloneHelpers(): Seq[String] = Seq(StandaloneHelpers.AttributeCasts) }