diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceIrisLogisticRegressionOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceIrisLogisticRegressionOpDesc.scala index 3cac29dd53a..42028c7cf64 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceIrisLogisticRegressionOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceIrisLogisticRegressionOpDesc.scala @@ -137,6 +137,7 @@ class HuggingFaceIrisLogisticRegressionOpDesc override def generateStandaloneCode(): String = { val lengthLit = pyStringLiteral(petalLengthCmAttribute) val widthLit = pyStringLiteral(petalWidthCmAttribute) + // The operator's guard: pd.isna answers for a JSON frame's NaN and a Tuple's None. s"""import numpy as np |import torch |import torch.nn as nn @@ -159,8 +160,6 @@ class HuggingFaceIrisLogisticRegressionOpDesc |_classes = [] |_probs = [] |for _length, _width in zip(out1df[$lengthLit], out1df[$widthLit]): - | # The operator's guard: pandas answers for the NaN a frame read from - | # JSON holds and for the None a Tuple hands over. | if pd.isna(_length) or pd.isna(_width): | _classes.append(None) | _probs.append(None) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceSentimentAnalysisOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceSentimentAnalysisOpDesc.scala index 0467d1e0b37..6d7c62bff4f 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceSentimentAnalysisOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceSentimentAnalysisOpDesc.scala @@ -121,6 +121,7 @@ class HuggingFaceSentimentAnalysisOpDesc val positiveLit = pyStringLiteral(resultAttributePositive) val neutralLit = pyStringLiteral(resultAttributeNeutral) val negativeLit = pyStringLiteral(resultAttributeNegative) + // An empty cell, which the tokenizer rejects, keeps its row with the scores left empty. s"""from transformers import AutoModelForSequenceClassification |from transformers import AutoTokenizer, AutoConfig |import numpy as np @@ -137,10 +138,6 @@ class HuggingFaceSentimentAnalysisOpDesc |for _col in ($positiveLit, $neutralLit, $negativeLit): | out1df[_col] = 0.0 |for _idx, _text in out1df[${pyStringLiteral(attribute)}].items(): - | # An empty cell reaches the frame as None, or as NaN once a column holds - | # nothing else, and the tokenizer rejects both. Keep the row and leave the - | # scores empty rather than ending the run over a value the model has - | # nothing to say about. | if pd.isna(_text) or (isinstance(_text, str) and not _text.strip()): | for _col in ($positiveLit, $neutralLit, $negativeLit): | out1df.at[_idx, _col] = None diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceSpamSMSDetectionOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceSpamSMSDetectionOpDesc.scala index e305a354cd6..e0a9138d82c 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceSpamSMSDetectionOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceSpamSMSDetectionOpDesc.scala @@ -100,6 +100,7 @@ class HuggingFaceSpamSMSDetectionOpDesc val attributeLit = pyStringLiteral(attribute) val spamLit = pyStringLiteral(resultAttributeSpam) val probabilityLit = pyStringLiteral(resultAttributeProbability) + // An empty cell, which the pipeline rejects, keeps its row with the results left empty. s"""from transformers import pipeline |import pandas as pd | @@ -107,10 +108,6 @@ class HuggingFaceSpamSMSDetectionOpDesc |out1df = in1df.copy() | |def _classify(_t): - | # An empty cell reaches the frame as None, or as NaN once a column holds - | # nothing else, and the pipeline rejects both. Keep the row and leave the - | # results empty rather than ending the run over a value the model has - | # nothing to say about. | if pd.isna(_t) or (isinstance(_t, str) and not _t.strip()): | return None | return _pipeline(_t)[0] diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceTextSummarizationOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceTextSummarizationOpDesc.scala index 50589907e2b..2f6b113bfff 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceTextSummarizationOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceTextSummarizationOpDesc.scala @@ -98,6 +98,7 @@ class HuggingFaceTextSummarizationOpDesc // once, generate a summary per row, and add the STRING result column to // produce out1df. override def generateStandaloneCode(): String = { + // An empty cell, which the tokenizer rejects, keeps its row with the summary left empty. s"""from transformers import BertTokenizerFast, EncoderDecoderModel |import pandas as pd |import torch @@ -110,10 +111,6 @@ class HuggingFaceTextSummarizationOpDesc |out1df = in1df.copy() |_summaries = [] |for _text in out1df[${pyStringLiteral(attribute)}]: - | # An empty cell reaches the frame as None, or as NaN once a column holds - | # nothing else, and the tokenizer rejects both. Keep the row and leave the - | # summary empty rather than ending the run over a value the model has - | # nothing to say about. | if pd.isna(_text) or (isinstance(_text, str) and not _text.strip()): | _summaries.append(None) | continue diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/machineLearning/Scorer/MachineLearningScorerOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/machineLearning/Scorer/MachineLearningScorerOpDesc.scala index e7ba0a7259f..1a8b5d79c12 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/machineLearning/Scorer/MachineLearningScorerOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/machineLearning/Scorer/MachineLearningScorerOpDesc.scala @@ -300,6 +300,8 @@ class MachineLearningScorerOpDesc extends PythonOperatorDescriptor with Standalo override def generateStandaloneCode(): String = { val isRegressionStr = if (isRegression) "True" else "False" + // Rows come back Class first, then the metrics in the order picked, as the operator declares. + // The drop goes to its own name, and an empty result raises rather than scoring nothing. s"""import pandas as pd |from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, mean_squared_error, root_mean_squared_error, mean_absolute_error, r2_score | @@ -320,9 +322,6 @@ class MachineLearningScorerOpDesc extends PythonOperatorDescriptor with Standalo | | labels = ['class_' + str(label) if type(label) != str else label for label in labels] | result['Class'] = labels - | # Class first and the metrics in the order they were picked, which is - | # the column order this operator declares and so the order of the rows - | # the workflow shows. | return pd.DataFrame(result)[['Class'] + metric_list] | |def regression_metrics(y_true, y_pred, metric_list): @@ -332,14 +331,9 @@ class MachineLearningScorerOpDesc extends PythonOperatorDescriptor with Standalo | result[metric] = metrics_func[metric](y_true, y_pred) | return pd.DataFrame(result, index=[0]) | - |# Bound to a name of its own: the same frame can feed another branch of the - |# plan, which must still see every row. |scored_df = in1df.dropna(subset=[${pyStringLiteral(actualValueColumn)}, ${pyStringLiteral( predictValueColumn )}]) - |# Nothing survived the drop. The metrics answer that badly and each in its own - |# way: the regression ones raise from inside scikit-learn, and the - |# classification ones return a NaN score, which reads as a result. |if scored_df.empty: | raise ValueError("No rows left to score: every row is missing the actual value, the predicted value, or both.") |y_true = scored_df[${pyStringLiteral(actualValueColumn)}] diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnClassifierOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnClassifierOpDesc.scala index 00657c905fb..6988f1a7213 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnClassifierOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnClassifierOpDesc.scala @@ -87,6 +87,7 @@ $reportMissingKept val narrowTrain = dropNonFeatureColumns("X_train", "") val narrowTest = dropNonFeatureColumns("X_test", "") + // Both frames drop the rows the operator drops, into local names, and each reports its skips. s"""${getImportStatements} |from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score |from sklearn.pipeline import make_pipeline @@ -95,10 +96,6 @@ $reportMissingKept |import numpy as np |import pandas as pd | - |# The same rows the operator drops, and on both frames: the model is fitted - |# on one and scored on the other, so narrowing only one side would fit and - |# score on different data. A local name rather than a reassignment, since - |# the input variable belongs to whichever operator produced it. |_train = ${dropMissingRowsStandalone("in1df")} |if len(_train) < len(in1df): | print("Skipped", len(in1df) - len(_train), "of", len(in1df), "rows with missing values") @@ -108,8 +105,6 @@ $reportMissingKept |model = make_pipeline(${vectorizerStage(c => pyStringLiteral(c))}$tfidfPart$estimator()).fit(X_train, Y_train) | |_test = ${dropMissingRowsStandalone("in2df")} - |# The operator runs once per port, so it says this for the table it scores - |# as well as for the one it fits. |if len(_test) < len(in2df): | print("Skipped", len(in2df) - len(_test), "of", len(in2df), "rows with missing values") |Y_test = _test[$targetLit] diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnLinearRegressionOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnLinearRegressionOpDesc.scala index 948b2b30610..0ac8789a9ec 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnLinearRegressionOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnLinearRegressionOpDesc.scala @@ -104,16 +104,13 @@ class SklearnLinearRegressionOpDesc override def generateStandaloneCode(): String = { val targetLit = pyStringLiteral(target) + // Both frames drop the rows the operator drops, into local names, and each reports its skips. s"""from sklearn.metrics import mean_absolute_error, r2_score |from sklearn.pipeline import make_pipeline |from sklearn.linear_model import LinearRegression |from sklearn.preprocessing import PolynomialFeatures |import pandas as pd | - |# The same rows the operator drops, and on both frames: the model is fitted - |# on one and scored on the other, so narrowing only one side would fit and - |# score on different data. Local names rather than reassignments, since the - |# input variables belong to whichever operators produced them. |_train = in1df.dropna() |if len(_train) < len(in1df): | print("Skipped", len(in1df) - len(_train), "of", len(in1df), "rows with missing values") @@ -127,8 +124,6 @@ class SklearnLinearRegressionOpDesc |model = pipeline.fit(X_train, Y_train) | |_test = in2df.dropna() - |# The operator runs once per port, so it says this for the table it scores - |# as well as for the one it fits. |if len(_test) < len(in2df): | print("Skipped", len(in2df) - len(_test), "of", len(in2df), "rows with missing values") |Y_test = _test[$targetLit] diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/testing/SklearnTestingOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/testing/SklearnTestingOpDesc.scala index 5786f041d94..c9373f3d28c 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/testing/SklearnTestingOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/testing/SklearnTestingOpDesc.scala @@ -132,13 +132,11 @@ class SklearnTestingOpDesc val isRegressionStr = if (isRegression) "True" else "False" val modelLit = pyStringLiteral(model) val targetLit = pyStringLiteral(target) + // Drops on every column as the executor does, since a fitted model hides its estimator. + // Each row's model scores the frame itself, kept two-dimensional for predict. s"""from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score, root_mean_squared_error, mean_absolute_error, r2_score | |out1df = in1df.copy() - |# The same drop the executor makes before it scores: the model arrives - |# already fitted, so this operator cannot ask which estimator it holds and - |# drops on every column to be safe. Scoring the rows it skips would answer - |# with a number the run never reported, or fail inside scikit-learn. |rows_read = len(in2df) |scored_df = in2df.dropna() |if len(scored_df) < rows_read: @@ -146,14 +144,6 @@ class SklearnTestingOpDesc |Y = scored_df[$targetLit] |X = scored_df.drop($targetLit, axis=1) |${narrowToFittableColumns("X", "")} - |# predict wants (n_samples, n_features). Squeezing the frame first collapsed - |# a single feature, or a single surviving row, to one dimension, which - |# scikit-learn rejects. - |# - |# The model port carries one model per row and the executor scores each of - |# them on its own, so a row has to be given the score of the model it holds. - |# One score written down the column would report the first model's answer - |# for all of them. |predictions = [_model.predict(X) for _model in in1df[$modelLit]] |if $isRegressionStr: | out1df["R2"] = [r2_score(Y, _p) for _p in predictions] diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingOpDesc.scala index 9dab7c27b56..fcc209cb2ba 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingOpDesc.scala @@ -74,15 +74,13 @@ $reportMissingKept val modelNameLit = pyStringLiteral(getUserFriendlyModelName) val narrowX = dropNonFeatureColumns("X", "") + // The rows the operator drops, into a local name, since the input belongs to its producer. s"""${getImportStatements} |from sklearn.pipeline import make_pipeline |from sklearn.compose import ColumnTransformer |from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer |import pandas as pd | - |# The same rows the operator drops. A local name rather than a - |# reassignment, since the input variable belongs to whichever operator - |# produced it. |_train = ${dropMissingRowsStandalone("in1df")} |if len(_train) < len(in1df): | print("Skipped", len(in1df) - len(_train), "of", len(in1df), "rows with missing values") diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/unneststring/UnnestStringOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/unneststring/UnnestStringOpDesc.scala index 3a90db3009c..afa0d87a335 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/unneststring/UnnestStringOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/unneststring/UnnestStringOpDesc.scala @@ -152,10 +152,8 @@ class UnnestStringOpDesc extends FlatMapOpDesc with StandaloneCodeGenerator { .flatMap(schema => scala.util.Try(schema.getAttribute(attribute)).toOption) .map(_.getType) val column = renderedAsText(s"out1df[$attributeLit]", declared) - s"""# Nothing in the column unnests to nothing, the way the operator answers a null - |# field with no rows at all. Dropped before the split rather than after: the - |# rendering would turn the empty cell into text and unnest that. - |out1df = in1df[in1df[$attributeLit].notna()].copy() + // A null unnests to no rows, so it is dropped before the rendering turns it into text. + s"""out1df = in1df[in1df[$attributeLit].notna()].copy() |out1df[$resultLit] = $column.str.split($delim, regex=True) |out1df = out1df.explode($resultLit, ignore_index=True) |out1df = out1df[(out1df[$resultLit].notna()) & (out1df[$resultLit] != "")].reset_index(drop=True)""".stripMargin diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/IcicleChart/IcicleChartOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/IcicleChart/IcicleChartOpDesc.scala index faeafba2ab8..e05f71fb174 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/IcicleChart/IcicleChartOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/IcicleChart/IcicleChartOpDesc.scala @@ -147,7 +147,7 @@ class IcicleChartOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCo // reason for "no chart" is where the reader looks for the chart — printing it // to the terminal alone left that file absent. render_error's continuation // line keeps the runtime path's indentation, since the HTML is triple-quoted - // and those spaces reach the browser. + // and those spaces reach the browser. chart_df is a copy: the input can feed another branch. s"""def render_error(error_msg): | return '''

Icicle chart is not available.

|

Reason is: {}

@@ -161,8 +161,6 @@ class IcicleChartOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCo |if in1df.empty: | fail("input table is empty.") |else: - | # On a copy: the same frame can feed another branch of the plan, and - | # both the assignment and the drop below would otherwise reach it. | chart_df = in1df.copy() | chart_df[$valueLit] = chart_df[chart_df[$valueLit] > 0][$valueLit] | chart_df = chart_df.dropna(subset=[$attributes]) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/barChart/BarChartOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/barChart/BarChartOpDesc.scala index f7013513210..7a34062ef2d 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/barChart/BarChartOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/barChart/BarChartOpDesc.scala @@ -180,7 +180,8 @@ class BarChartOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCode // reason for "no chart" is where the reader looks for the chart — printing it // to the terminal alone left that file absent. render_error's continuation // line keeps the runtime path's indentation, since the HTML is triple-quoted - // and those spaces reach the browser. + // and those spaces reach the browser. Empty is checked before the dropna, since an empty + // table read back from JSONL has no columns to name, and chart_df keeps the input whole. s"""def render_error(error_msg): | return '''

Bar chart is not available.

|

Reason is: {}

@@ -194,13 +195,8 @@ class BarChartOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCode |if $fieldsLit == $valueLit: | fail("Fields should not have the same value.") |elif in1df.empty: - | # Checked before the dropna, unlike the runtime path: an empty table read - | # back from JSONL carries no columns at all, so naming them in `subset` - | # would raise a KeyError instead of reporting the empty table. | fail("Table should not have any empty/null values or fields.") |else: - | # Bound to a name of its own: the same frame can feed another branch - | # of the plan, which must still see every row. | chart_df = in1df.dropna(subset=[$valueLit, $fieldsLit]) | if chart_df.empty: | fail("Table should not have any empty/null values or fields.") diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/boxViolinPlot/BoxViolinPlotOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/boxViolinPlot/BoxViolinPlotOpDesc.scala index 1b1200b0e8a..fc8bef249a0 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/boxViolinPlot/BoxViolinPlotOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/boxViolinPlot/BoxViolinPlotOpDesc.scala @@ -171,7 +171,7 @@ class BoxViolinPlotOpDesc extends PythonOperatorDescriptor with PlotlyStandalone // reason for "no chart" is where the reader looks for the chart — printing it // to the terminal alone left that file absent. render_error's continuation // line keeps the runtime path's indentation, since the HTML is triple-quoted - // and those spaces reach the browser. + // and those spaces reach the browser. chart_df keeps the input whole for other branches. s"""def render_error(error_msg): | return '''

Box/Violin Plot is not available.

|

Reason is: {}

@@ -185,8 +185,6 @@ class BoxViolinPlotOpDesc extends PythonOperatorDescriptor with PlotlyStandalone |if in1df.empty: | fail("input table is empty.") |else: - | # Bound to a name of its own: the same frame can feed another branch - | # of the plan, which must still see every row. | chart_df = in1df.dropna(subset=[$valueLit]) | if chart_df.empty: | fail("value column contains only non-positive numbers or nulls.") diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/bubbleChart/BubbleChartOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/bubbleChart/BubbleChartOpDesc.scala index be05f4dfec9..9478c345d5e 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/bubbleChart/BubbleChartOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/bubbleChart/BubbleChartOpDesc.scala @@ -181,7 +181,8 @@ class BubbleChartOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCo // to the terminal alone left that file absent. The heading is the // runtime path's, TreeMap and all, so both paths say the same thing. // render_error's continuation line keeps the runtime path's indentation, since - // the HTML is triple-quoted and those spaces reach the browser. + // the HTML is triple-quoted and those spaces reach the browser. chart_df keeps the + // input whole for other branches. s"""def render_error(error_msg): | return '''

TreeMap is not available.

|

Reasons are: {}

@@ -195,8 +196,6 @@ class BubbleChartOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCo |if in1df.empty: | fail("Input table is empty.") |else: - | # Bound to a name of its own: the same frame can feed another branch - | # of the plan, which must still see every row. | chart_df = in1df.dropna(subset=[$xLit, $yLit, $zLit]) | if chart_df.empty: | fail("No valid rows left (every row has at least 1 missing value).") diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/bulletChart/BulletChartOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/bulletChart/BulletChartOpDesc.scala index 14277cece25..bd4a33dabe1 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/bulletChart/BulletChartOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/bulletChart/BulletChartOpDesc.scala @@ -268,7 +268,8 @@ class BulletChartOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCo .mkString("[", ", ", "]") // render_error's continuation line keeps the runtime path's indentation — the - // HTML is triple-quoted, so those spaces reach the browser. + // HTML is triple-quoted, so those spaces reach the browser. The JSON holds one chart + // per row, and a lone figure is written as one object, as every one-chart operator does. s"""def render_error(error_msg): | return '''

Bullet chart is not available.

|

Reason: {}

'''.format(error_msg) @@ -353,9 +354,6 @@ class BulletChartOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCo | html_chunks.append(html_chunk) | except Exception as e: | html_chunks.append(render_error(f"Error generating bullet chart: {str(e)}")) - | # One chart per row, so the file holds the whole sequence. - | # A single figure still writes as a lone object, which is - | # what every one-chart operator writes. | if len(figs) == 1: | figs[0].write_json(outputJson) | elif figs: diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/choroplethMap/ChoroplethMapOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/choroplethMap/ChoroplethMapOpDesc.scala index 1dee0093f82..bf0e57ccd7a 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/choroplethMap/ChoroplethMapOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/choroplethMap/ChoroplethMapOpDesc.scala @@ -143,7 +143,7 @@ class ChoroplethMapOpDesc extends PythonOperatorDescriptor with PlotlyStandalone // reason for "no chart" is where the reader looks for the chart — printing it // to the terminal alone left that file absent. render_error's continuation // line keeps the runtime path's indentation, since the HTML is triple-quoted - // and those spaces reach the browser. + // and those spaces reach the browser. chart_df keeps the input whole for other branches. s"""def render_error(error_msg): | return '''

Choropleth map is not available.

|

Reason is: {}

@@ -157,8 +157,6 @@ class ChoroplethMapOpDesc extends PythonOperatorDescriptor with PlotlyStandalone |if in1df.empty: | fail("Input table is empty.") |else: - | # Bound to a name of its own: the same frame can feed another branch - | # of the plan, which must still see every row. | chart_df = in1df.dropna(subset=[$locationsLit, $colorLit]) | if chart_df.empty: | fail("No valid rows left (every row has at least 1 missing value).") diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/contourPlot/ContourPlotOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/contourPlot/ContourPlotOpDesc.scala index 88adfc2f93e..9f2c0f7c768 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/contourPlot/ContourPlotOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/contourPlot/ContourPlotOpDesc.scala @@ -166,7 +166,8 @@ class ContourPlotOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCo override def generateStandaloneCode(): String = // render_error's continuation line keeps the runtime path's indentation — the - // HTML is triple-quoted, so those spaces reach the browser. + // HTML is triple-quoted, so those spaces reach the browser. griddata refuses a NaN, so + // chart_df drops it and keeps the input whole; collinear points get an error page, not an exit. s"""import numpy as np |from scipy.interpolate import griddata | @@ -179,9 +180,6 @@ class ContourPlotOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCo | with open(outputHtml, "w", encoding="utf-8") as output: | output.write(render_error(message)) | - |# A row missing any of the three has no point to contribute, and griddata - |# refuses a NaN coordinate outright. Bound to a name of its own: the same - |# frame can feed another branch of the plan, which must still see every row. |chart_df = in1df.dropna(subset=[${pyStringLiteral(x)}, ${pyStringLiteral( y )}, ${pyStringLiteral( @@ -194,12 +192,7 @@ class ContourPlotOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCo | y = chart_df[${pyStringLiteral(y)}].values | z = chart_df[${pyStringLiteral(z)}].values | - | # Cubic interpolation triangulates the points before it interpolates, which - | # needs them to span a plane. Points that all fall on one line leave Qhull - | # without a simplex to start from and it raises instead. | points = np.unique(np.column_stack((x, y)), axis=0) - | # An else rather than an exit: this block shares a script with the rest of - | # the plan, which has to go on running after a plot that cannot be drawn. | if np.linalg.matrix_rank(points - points.mean(axis=0)) < 2: | _write_error("The x and y values all fall on one line, so there is no area to contour.") | else: diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/dendrogram/DendrogramOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/dendrogram/DendrogramOpDesc.scala index c8f4d728d71..f996842d36e 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/dendrogram/DendrogramOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/dendrogram/DendrogramOpDesc.scala @@ -158,7 +158,8 @@ class DendrogramOpDesc extends PythonOperatorDescriptor with StandaloneCodeGener override def generateStandaloneCode(): String = { // render_error's continuation line keeps the runtime path's indentation — the - // HTML is triple-quoted, so those spaces reach the browser. + // HTML is triple-quoted, so those spaces reach the browser. scipy refuses a NaN and a + // single row, so both get an error page; chart_df keeps the input whole. s"""import numpy as np |import plotly.figure_factory as ff | @@ -174,15 +175,9 @@ class DendrogramOpDesc extends PythonOperatorDescriptor with StandaloneCodeGener |if in1df.empty: | _write_error("input table is empty.") |else: - | # A row missing either coordinate has no position to cluster from, and - | # scipy refuses a NaN anywhere in the distance matrix. Bound to a name - | # of its own: the same frame can feed another branch of the plan, which - | # must still see every row. | chart_df = in1df.dropna(subset=[${pyStringLiteral(xVal)}, ${pyStringLiteral(yVal)}]) | if chart_df.empty: | _write_error("input table has no rows with all of the configured columns filled in.") - | # Clustering starts from the distances between rows, so a single row - | # leaves scipy an empty distance matrix and it raises rather than draws. | elif len(chart_df) < 2: | _write_error("input table has fewer than two rows to cluster.") | else: diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/gaugeChart/GaugeChartOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/gaugeChart/GaugeChartOpDesc.scala index 324d7c68403..2bb17dad0a0 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/gaugeChart/GaugeChartOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/gaugeChart/GaugeChartOpDesc.scala @@ -207,7 +207,8 @@ class GaugeChartOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCod val thresholdExpr = numberOrNone(threshold) val stepsExpr = stepsLiteral // render_error's continuation line keeps the runtime path's indentation — the - // HTML is triple-quoted, so those spaces reach the browser. + // HTML is triple-quoted, so those spaces reach the browser. Drawing sits in a try so a + // surprise is an error page, and a lone figure is written as one object, not a list. s"""import plotly.graph_objects as go |import plotly.io as pio | @@ -226,9 +227,6 @@ class GaugeChartOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCod | with open(outputHtml, "w", encoding="utf-8") as output: | output.write(render_error("Input table is empty.")) |else: - | # Everything the operator draws inside its own try, so anything it did - | # not foresee (a column that is not there, a step that is not a number) - | # is a page saying so rather than the end of the exported run. | try: | gauge_value = $valueLit | delta_ref = $deltaExpr @@ -291,9 +289,6 @@ class GaugeChartOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCod | | with open(outputHtml, "w", encoding="utf-8") as output: | output.write("
" + "".join(html_chunks) + "
") - | # One chart per row, so the file holds the whole sequence. A - | # single figure still writes as a lone object, which is what - | # every one-chart operator writes. | if len(figs) == 1: | figs[0].write_json(outputJson) | elif figs: diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/hierarchychart/HierarchyChartOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/hierarchychart/HierarchyChartOpDesc.scala index b57aa440f9c..14c452844a8 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/hierarchychart/HierarchyChartOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/hierarchychart/HierarchyChartOpDesc.scala @@ -152,7 +152,7 @@ class HierarchyChartOpDesc extends PythonOperatorDescriptor with PlotlyStandalon // reason for "no chart" is where the reader looks for the chart — printing it // to the terminal alone left that file absent. render_error's continuation // line keeps the runtime path's indentation, since the HTML is triple-quoted - // and those spaces reach the browser. + // and those spaces reach the browser. chart_df is a copy: the input can feed another branch. s"""def render_error(error_msg): | return '''

Hierarchy chart is not available.

|

Reason is: {}

@@ -166,8 +166,6 @@ class HierarchyChartOpDesc extends PythonOperatorDescriptor with PlotlyStandalon |if in1df.empty: | fail("input table is empty.") |else: - | # On a copy: the same frame can feed another branch of the plan, and - | # both the assignment and the drop below would otherwise reach it. | chart_df = in1df.copy() | chart_df[$valueLit] = chart_df[chart_df[$valueLit] > 0][$valueLit] | chart_df = chart_df.dropna(subset=[$attributes]) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/histogram2d/Histogram2DOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/histogram2d/Histogram2DOpDesc.scala index 1da0ffa4f53..61b89224c38 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/histogram2d/Histogram2DOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/histogram2d/Histogram2DOpDesc.scala @@ -133,6 +133,7 @@ class Histogram2DOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCo val xLit = pyStringLiteral(xColumn) val yLit = pyStringLiteral(yColumn) + // chart_df keeps the input whole for other branches. s"""import plotly.express as px | |def render_error(msg): @@ -142,8 +143,6 @@ class Histogram2DOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCo | with open(outputHtml, "w", encoding="utf-8") as output: | output.write(render_error("Input table is empty.")) |else: - | # Bound to a name of its own: the same frame can feed another branch - | # of the plan, which must still see every row. | chart_df = in1df.dropna(subset=[$xLit, $yLit]) | if chart_df.empty: | with open(outputHtml, "w", encoding="utf-8") as output: diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/radarPlot/RadarPlotOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/radarPlot/RadarPlotOpDesc.scala index 659e41097db..ba80d721c07 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/radarPlot/RadarPlotOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/radarPlot/RadarPlotOpDesc.scala @@ -310,7 +310,6 @@ class RadarPlotOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCode | fig = go.Figure() | | for idx, row in enumerate(selected_table): - | # To connect ensure all points in the radar trace are connected | closed_row = row.tolist() + [row[0]] | closed_categories = categories + [categories[0]] | closed_hover_texts = hover_texts[idx] + [hover_texts[idx][0]] diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/treeplot/TreePlotOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/treeplot/TreePlotOpDesc.scala index 1df3d94e4ec..15aa6d24e79 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/treeplot/TreePlotOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/treeplot/TreePlotOpDesc.scala @@ -249,11 +249,11 @@ class TreePlotOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCode override def producesDataFrame(): Boolean = false + // Only a layout failure renders an error page, as in the operator. A label keeps its cell + // value, so None is not "None"; layout is a tidy top-down tree, each node placed once. override def generateStandaloneCode(): String = { s"""import ast | - |# Only a layout failure renders an error page; everything else propagates, - |# matching the operator's own error handling. |class TreeLayoutError(Exception): | pass | @@ -269,9 +269,6 @@ class TreePlotOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCode | for node_name, coords in pos.items(): | annotations.append( | dict( - | # The label goes in as it came out of the cell, the way the - | # operator passes it: plotly stringifies it, and a str() here - | # would turn a None node into the text "None". | text=node_name, | x=coords[0], | y=coords[1], @@ -289,8 +286,6 @@ class TreePlotOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCode | return annotations | |def build_tree_layout(edges): - | # Tidy top-down tree: depth picks the row, a leaf takes the next - | # free column and a parent sits centred over its own children. | labels = [] | known = set() | for parent, child in edges: @@ -329,8 +324,6 @@ class TreePlotOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCode | next_column += 1 | continue | depth[node] = level - | # A node belongs to whichever parent reaches it first, so a - | # cycle or a shared child is never laid out twice. | kids = [kid for kid in children[node] if kid not in placed] | placed.update(kids) | claimed[node] = kids @@ -341,11 +334,9 @@ class TreePlotOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCode | for label in labels: | if label not in placed and label not in has_parent: | grow(label) - | # Whatever is left sits in a cycle that no root can reach. | for label in labels: | if label not in placed: | grow(label) - | # The y-axis is inverted here so the tree grows top-down. | return labels, [(column[label], -depth[label]) for label in labels] | |def compute_tree_layout(edges): diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/urlviz/UrlVizOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/urlviz/UrlVizOpDesc.scala index 4409adbcd8e..ec1f141b396 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/urlviz/UrlVizOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/urlviz/UrlVizOpDesc.scala @@ -86,19 +86,14 @@ class UrlVizOpDesc extends LogicalOp with StandaloneCodeGenerator { ) // Mirrors UrlVizOpExec: wrap each urlContentAttrName value in the exact same - // iframe HTML document and emit it as the "html-content" column. + // iframe HTML document and emit it as the "html-content" column. A null renders as the + // JVM's "null", and the cell is escaped the way the executor's escapeAttribute does. override def generateStandaloneCode(): String = { val urlLit = pyStringLiteral(urlContentAttrName) s"""import html | |def _texera_urlviz_iframe(u): - | # "null", not Python's "None": the operator interpolates the field into a - | # Scala string, and the JVM renders a null that way. | u = "null" if pd.isna(u) else str(u) - | # The cell lands where an attribute is expected, so a quote in it would - | # close src= and leave the rest of the cell standing as attributes of the - | # iframe. escapeAttribute on the executor writes the same five characters - | # in the same order, so the two paths write the page the same way. | u = html.escape(u, quote=True) | return ( | '\\n'