From 7ac5b181986c8adb9a4247ca4f37e18956de18a7 Mon Sep 17 00:00:00 2001 From: kraysent Date: Mon, 6 Jul 2026 22:06:48 +0100 Subject: [PATCH 01/10] mvp fo expressions using eval --- tests/test_formula_evaluate.py | 307 ++++++++++++++++++++++++++ tests/test_formula_parse.py | 32 +++ uploader/app/lib/formula/__init__.py | 20 ++ uploader/app/lib/formula/core.py | 72 ++++++ uploader/app/lib/formula/errors.py | 10 + uploader/app/lib/formula/namespace.py | 51 +++++ uploader/app/lib/formula/values.py | 19 ++ 7 files changed, 511 insertions(+) create mode 100644 tests/test_formula_evaluate.py create mode 100644 tests/test_formula_parse.py create mode 100644 uploader/app/lib/formula/__init__.py create mode 100644 uploader/app/lib/formula/core.py create mode 100644 uploader/app/lib/formula/errors.py create mode 100644 uploader/app/lib/formula/namespace.py create mode 100644 uploader/app/lib/formula/values.py diff --git a/tests/test_formula_evaluate.py b/tests/test_formula_evaluate.py new file mode 100644 index 0000000..81ce451 --- /dev/null +++ b/tests/test_formula_evaluate.py @@ -0,0 +1,307 @@ +from dataclasses import dataclass + +import astropy.units as u +import numpy as np +import pytest + +from uploader.app.lib import expression as legacy_expression +from uploader.app.lib.formula import ( + ExpressionEvaluationError, + Value, + column_quantity, + evaluate, + parse, +) +from uploader.app.lib.formula.namespace import FUNCTIONS, NAMED_CONSTANTS + + +@dataclass +class Col: + value: float | str + unit: str = "" + + +def evaluate_expr(source: str, columns: dict[str, Col]) -> Value: + built = {name: column_quantity(col.value, col.unit) for name, col in columns.items()} + return evaluate(parse(source), built) + + +def _sample_columns() -> dict[str, Col]: + return { + "logd25": Col(1.5), + "logr25": Col(0.3), + "e_logd25": Col(0.05), + "e_logr25": Col(0.04), + "pa": Col(190.0, "deg"), + } + + +def _legacy_values_units(columns: dict[str, Col]) -> tuple[dict[str, float], dict[str, str]]: + values = {name: float(col.value) for name, col in columns.items() if isinstance(col.value, int | float)} + units = {name: col.unit for name, col in columns.items()} + return values, units + + +def test_isophotal_axis_expressions() -> None: + columns = _sample_columns() + a = evaluate_expr('3 * 10 ** col("logd25") * arcsec', columns) + assert isinstance(a, u.Quantity) + assert a.unit == u.arcsec + assert abs(a.value - 94.86832980505137) < 1e-6 + + e_a = evaluate_expr('3 * 10 ** col("logd25") * 2.302585093 * e_logd25 * arcsec', columns) + assert isinstance(e_a, u.Quantity) + assert e_a.unit == u.arcsec + assert e_a.value > 0 + + b = evaluate_expr('3 * 10 ** (col("logd25") - col("logr25")) * arcsec', columns) + assert isinstance(b, u.Quantity) + assert b.unit == u.arcsec + assert b.value > 0 + + e_b = evaluate_expr( + '3 * 10 ** (col("logd25") - col("logr25")) * 2.302585093 ' + '* (col("e_logd25") ** 2 + col("e_logr25") ** 2) ** 0.5 * arcsec', + columns, + ) + assert isinstance(e_b, u.Quantity) + assert e_b.unit == u.arcsec + assert e_b.value > 0 + + +def test_unit_aware_position_angle_modulo() -> None: + columns = _sample_columns() + pa = evaluate_expr('col("pa") % (180.0 * deg)', columns) + assert isinstance(pa, u.Quantity) + assert pa.unit == u.deg + assert pa.value == 10.0 + + +def test_isophotal_axis_expressions_with_logarithmic_column_units() -> None: + columns = _sample_columns() + for log_unit in ("mag", "dex"): + columns_with_log = { + **columns, + "logd25": Col(1.5, log_unit), + "logr25": Col(0.3, log_unit), + "e_logd25": Col(0.05, log_unit), + "e_logr25": Col(0.04, log_unit), + } + a = evaluate_expr('3 * 10 ** col("logd25") * arcsec', columns_with_log) + assert isinstance(a, u.Quantity) + assert a.unit == u.arcsec + assert abs(a.value - 94.86832980505137) < 1e-6 + + e_a = evaluate_expr('3 * 10 ** col("logd25") * 2.302585093 * e_logd25 * arcsec', columns_with_log) + assert isinstance(e_a, u.Quantity) + assert e_a.unit == u.arcsec + assert e_a.value > 0 + + e_b = evaluate_expr( + '3 * 10 ** (col("logd25") - col("logr25")) * 2.302585093 ' + '* (col("e_logd25") ** 2 + col("e_logr25") ** 2) ** 0.5 * arcsec', + columns_with_log, + ) + assert isinstance(e_b, u.Quantity) + assert e_b.unit == u.arcsec + assert e_b.value > 0 + + +def test_isophotal_axis_expressions_with_hyperleda_units() -> None: + columns = { + "logd25": Col(0.697, "dex(0.1 arcmin)"), + "logr25": Col(0.13, "dex"), + "e_logd25": Col(0.079, "dex(0.1 arcmin)"), + "e_logr25": Col(0.028, "dex"), + "pa": Col(161.14, "deg"), + } + a = evaluate_expr('3 * 10 ** col("logd25") * arcsec', columns) + assert isinstance(a, u.Quantity) + assert a.unit == u.arcsec + assert a.value > 0 + + e_a = evaluate_expr('3 * 10 ** col("logd25") * 2.302585093 * e_logd25 * arcsec', columns) + assert isinstance(e_a, u.Quantity) + assert e_a.unit == u.arcsec + assert e_a.value > 0 + + b = evaluate_expr('3 * 10 ** (col("logd25") - col("logr25")) * arcsec', columns) + assert isinstance(b, u.Quantity) + assert b.unit == u.arcsec + assert b.value > 0 + + e_b = evaluate_expr( + '3 * 10 ** (col("logd25") - col("logr25")) * 2.302585093 ' + '* (col("e_logd25") ** 2 + col("e_logr25") ** 2) ** 0.5 * arcsec', + columns, + ) + assert isinstance(e_b, u.Quantity) + assert e_b.unit == u.arcsec + assert e_b.value > 0 + + +def test_surface_brightness_column_keeps_units() -> None: + columns = {"bri25": Col(23.162, "mag / arcsec2")} + bri25 = evaluate_expr('col("bri25")', columns) + assert isinstance(bri25, u.Quantity) + assert bri25.unit == u.Unit("mag/arcsec2") + + +STRING_CASES: list[tuple[str, dict[str, Col], str]] = [ + ('"abc"', {}, "abc"), + ('col("name")', {"name": Col("NGC 123")}, "NGC 123"), + ('col("a") + col("b")', {"a": Col("M"), "b": Col("82")}, "M82"), + ('"M " + col("id")', {"id": Col("82")}, "M 82"), + ('col("a") + " " + col("b")', {"a": Col("NGC"), "b": Col("905")}, "NGC 905"), +] + + +@pytest.mark.parametrize("source,columns,expected", STRING_CASES) +def test_string_evaluation(source: str, columns: dict[str, Col], expected: str) -> None: + assert evaluate_expr(source, columns) == expected + + +COERCION_CASES: list[tuple[float | str, str, object]] = [ + ("hello", "", "hello"), + (1.5, "", 1.5), + (1.5, "deg", u.Quantity(1.5, u.deg)), + (1.5, "mag", 1.5), + (1.5, "dex", 1.5), + (0.697, "dex(0.1 arcmin)", 0.697), + (23.162, "mag / arcsec2", u.Quantity(23.162, u.Unit("mag/arcsec2"))), +] + + +@pytest.mark.parametrize("value,unit,expected", COERCION_CASES) +def test_column_quantity(value: float | str, unit: str, expected: object) -> None: + result = column_quantity(value, unit) + if isinstance(expected, u.Quantity): + assert isinstance(result, u.Quantity) + assert result.unit == expected.unit + assert result.value == pytest.approx(expected.value) + else: + assert result == expected + + +def test_name_precedence_constant_over_column() -> None: + columns = {"pi": Col(1.0)} + result = evaluate_expr("pi", columns) + assert isinstance(result, u.Quantity) + assert result.value == pytest.approx(np.pi) + + col_result = evaluate_expr('col("pi")', columns) + assert isinstance(col_result, u.Quantity) + assert col_result.value == 1.0 + assert col_result.unit == u.dimensionless_unscaled + + +def test_trig_on_angle_column() -> None: + columns = {"pa": Col(30.0, "deg")} + result = evaluate_expr('sin(col("pa"))', columns) + assert isinstance(result, u.Quantity) + assert result.value == pytest.approx(0.5) + + +EVAL_ERROR_CASES: list[tuple[str, dict[str, Col]]] = [ + ("missing_col", {}), + ('col("missing")', {}), + ("arcsec + mag", {}), + ('col("name") + col("logd25")', {"name": Col("x"), "logd25": Col(1.5)}), + ('col("pa") % 180.0', {"pa": Col(190.0, "deg")}), + ('sin(col("x"))', {"x": Col(0.5)}), +] + + +@pytest.mark.parametrize("source,columns", EVAL_ERROR_CASES) +def test_evaluation_errors(source: str, columns: dict[str, Col]) -> None: + with pytest.raises(ExpressionEvaluationError): + evaluate_expr(source, columns) + + +PARITY_CASES: list[tuple[str, dict[str, Col]]] = [ + ('3 * 10 ** col("logd25") * arcsec', _sample_columns()), + ('3 * 10 ** col("logd25") * 2.302585093 * e_logd25 * arcsec', _sample_columns()), + ('3 * 10 ** (col("logd25") - col("logr25")) * arcsec', _sample_columns()), + ( + '3 * 10 ** (col("logd25") - col("logr25")) * 2.302585093 ' + '* (col("e_logd25") ** 2 + col("e_logr25") ** 2) ** 0.5 * arcsec', + _sample_columns(), + ), + ('col("bri25")', {"bri25": Col(23.162, "mag / arcsec2")}), +] + + +@pytest.mark.parametrize("source,columns", PARITY_CASES) +def test_parity_with_legacy_expression(source: str, columns: dict[str, Col]) -> None: + formula_result = evaluate_expr(source, columns) + values, units = _legacy_values_units(columns) + legacy_result = legacy_expression.parse(source).evaluate(values, units) + assert isinstance(formula_result, u.Quantity) + assert formula_result.unit == legacy_result.unit + assert formula_result.value == pytest.approx(legacy_result.value) + + +@pytest.mark.parametrize( + "source,columns", + [ + ( + '3 * 10 ** col("logd25") * arcsec', + { + **{k: Col(v.value, v.unit) for k, v in _sample_columns().items()}, + "logd25": Col(1.5, "mag"), + "logr25": Col(0.3, "mag"), + "e_logd25": Col(0.05, "mag"), + "e_logr25": Col(0.04, "mag"), + }, + ), + ( + '3 * 10 ** col("logd25") * arcsec', + { + **{k: Col(v.value, v.unit) for k, v in _sample_columns().items()}, + "logd25": Col(1.5, "dex"), + "logr25": Col(0.3, "dex"), + "e_logd25": Col(0.05, "dex"), + "e_logr25": Col(0.04, "dex"), + }, + ), + ( + '3 * 10 ** col("logd25") * arcsec', + { + "logd25": Col(0.697, "dex(0.1 arcmin)"), + "logr25": Col(0.13, "dex"), + "e_logd25": Col(0.079, "dex(0.1 arcmin)"), + "e_logr25": Col(0.028, "dex"), + "pa": Col(161.14, "deg"), + }, + ), + ], +) +def test_parity_with_logarithmic_units(source: str, columns: dict[str, Col]) -> None: + formula_result = evaluate_expr(source, columns) + values, units = _legacy_values_units(columns) + legacy_result = legacy_expression.parse(source).evaluate(values, units) + assert isinstance(formula_result, u.Quantity) + assert formula_result.unit == legacy_result.unit + assert formula_result.value == pytest.approx(legacy_result.value) + + +CONSTANT_GUARD_CASES = [(name, f"{name}") for name in NAMED_CONSTANTS] + + +@pytest.mark.parametrize("name,source", CONSTANT_GUARD_CASES) +def test_named_constants_are_usable(name: str, source: str) -> None: + result = evaluate_expr(source, {}) + assert isinstance(result, u.Quantity) + + +FUNCTION_GUARD_CASES = [ + ("sin", "sin(30 * deg)"), + ("cos", "cos(0 * rad)"), +] + + +@pytest.mark.parametrize("name,source", FUNCTION_GUARD_CASES) +def test_functions_are_usable(name: str, source: str) -> None: + assert name in FUNCTIONS + result = evaluate_expr(source, {}) + assert isinstance(result, u.Quantity) diff --git a/tests/test_formula_parse.py b/tests/test_formula_parse.py new file mode 100644 index 0000000..7a0ccb0 --- /dev/null +++ b/tests/test_formula_parse.py @@ -0,0 +1,32 @@ +import pytest + +from uploader.app.lib.formula import ExpressionSyntaxError, parse + +PARSE_CASES: list[tuple[str, set[str] | type[ExpressionSyntaxError]]] = [ + ("e_logd25 + logd25", {"e_logd25", "logd25"}), + ('col("a") + col("b")', {"a", "b"}), + ('col("weird name")', {"weird name"}), + ('sin(col("pa")) + pi', {"pa"}), + ("logd25 + logd25", {"logd25"}), + ('3 * 10 ** col("logd25") * e_logd25 * arcsec', {"logd25", "e_logd25"}), + ('"M " + col("id")', {"id"}), + ("1 + 2", set()), + ("", ExpressionSyntaxError), + ("1 +", ExpressionSyntaxError), + ("col(", ExpressionSyntaxError), + ("* 2", ExpressionSyntaxError), + ("a = 1", ExpressionSyntaxError), + ("col()", ExpressionSyntaxError), + ("col(x)", ExpressionSyntaxError), + ('col("a", "b")', ExpressionSyntaxError), + ("col(1)", ExpressionSyntaxError), +] + + +@pytest.mark.parametrize("source,expected", PARSE_CASES) +def test_parse(source: str, expected: set[str] | type[ExpressionSyntaxError]) -> None: + if isinstance(expected, type): + with pytest.raises(expected): + parse(source) + else: + assert parse(source).referenced_columns == expected diff --git a/uploader/app/lib/formula/__init__.py b/uploader/app/lib/formula/__init__.py new file mode 100644 index 0000000..b5f3f85 --- /dev/null +++ b/uploader/app/lib/formula/__init__.py @@ -0,0 +1,20 @@ +from uploader.app.lib.formula.core import Expression, evaluate, parse +from uploader.app.lib.formula.errors import ( + ExpressionError, + ExpressionEvaluationError, + ExpressionSyntaxError, +) +from uploader.app.lib.formula.namespace import expression_syntax_help +from uploader.app.lib.formula.values import Value, column_quantity + +__all__ = [ + "Expression", + "ExpressionError", + "ExpressionEvaluationError", + "ExpressionSyntaxError", + "Value", + "column_quantity", + "evaluate", + "expression_syntax_help", + "parse", +] diff --git a/uploader/app/lib/formula/core.py b/uploader/app/lib/formula/core.py new file mode 100644 index 0000000..cf1fdfa --- /dev/null +++ b/uploader/app/lib/formula/core.py @@ -0,0 +1,72 @@ +import ast +from collections.abc import Mapping +from dataclasses import dataclass +from types import CodeType +from typing import final + +import astropy.units as u + +from uploader.app.lib.formula.errors import ExpressionEvaluationError, ExpressionSyntaxError +from uploader.app.lib.formula.namespace import COL_FUNCTION, FUNCTIONS, NAMED_CONSTANTS, build_namespace +from uploader.app.lib.formula.values import Value + + +def _column_from_call(node: ast.Call) -> str | None: + if not isinstance(node.func, ast.Name) or node.func.id != COL_FUNCTION: + return None + if node.keywords or len(node.args) != 1: + raise ValueError(f"{COL_FUNCTION}() takes exactly one string argument") + arg = node.args[0] + if not isinstance(arg, ast.Constant) or not isinstance(arg.value, str): + raise ValueError(f"{COL_FUNCTION}() argument must be a string literal") + return arg.value + + +@final +@dataclass(frozen=True) +class Expression: + referenced_columns: frozenset[str] + code: CodeType + + +def parse(source: str) -> Expression: + try: + tree = ast.parse(source.strip(), mode="eval") + except SyntaxError as e: + raise ExpressionSyntaxError(str(e)) from e + try: + referenced_columns = frozenset(_ColumnCollector().collect(tree.body)) + except ValueError as e: + raise ExpressionSyntaxError(str(e)) from e + code = compile(tree, "", "eval") + return Expression(referenced_columns=referenced_columns, code=code) + + +def evaluate(expression: Expression, columns: Mapping[str, Value]) -> Value: + try: + return eval(expression.code, build_namespace(columns)) # noqa: S307 + except (KeyError, NameError, TypeError, ValueError, ZeroDivisionError, u.UnitsError) as e: + raise ExpressionEvaluationError(str(e)) from e + + +@final +class _ColumnCollector(ast.NodeVisitor): + def __init__(self) -> None: + self.columns: set[str] = set() + + def collect(self, node: ast.AST) -> set[str]: + self.visit(node) + return self.columns + + def visit_Call(self, node: ast.Call) -> None: + column = _column_from_call(node) + if column is not None: + self.columns.add(column) + return + for arg in node.args: + self.visit(arg) + + def visit_Name(self, node: ast.Name) -> None: + if node.id in NAMED_CONSTANTS or node.id in FUNCTIONS: + return + self.columns.add(node.id) diff --git a/uploader/app/lib/formula/errors.py b/uploader/app/lib/formula/errors.py new file mode 100644 index 0000000..aee0e07 --- /dev/null +++ b/uploader/app/lib/formula/errors.py @@ -0,0 +1,10 @@ +class ExpressionError(Exception): + pass + + +class ExpressionSyntaxError(ExpressionError): + pass + + +class ExpressionEvaluationError(ExpressionError): + pass diff --git a/uploader/app/lib/formula/namespace.py b/uploader/app/lib/formula/namespace.py new file mode 100644 index 0000000..3d0a70f --- /dev/null +++ b/uploader/app/lib/formula/namespace.py @@ -0,0 +1,51 @@ +from collections.abc import Mapping + +import astropy.constants as const +import astropy.units as u +import numpy as np + +from uploader.app.lib.formula.values import Value + +COL_FUNCTION = "col" + +NAMED_CONSTANTS: dict[str, u.Quantity] = { + "pi": np.pi * u.dimensionless_unscaled, + "c": const.c, + "deg": 1 * u.deg, + "rad": 1 * u.rad, + "arcmin": 1 * u.arcmin, + "arcsec": 1 * u.arcsec, + "mag": 1 * u.mag, +} + +FUNCTIONS: dict[str, object] = { + "sin": np.sin, + "cos": np.cos, +} + + +def build_namespace(columns: Mapping[str, Value]) -> dict[str, object]: + bare = {name: value for name, value in columns.items() if name.isidentifier()} + return { + "__builtins__": {}, + COL_FUNCTION: lambda name: columns[name], + **bare, + **NAMED_CONSTANTS, + **FUNCTIONS, + } + + +def expression_syntax_help() -> str: + constants = ", ".join(sorted(NAMED_CONSTANTS)) + return ( + f'Use {COL_FUNCTION}("name") or bare identifiers to refer to rawdata columns ' + '(e.g. col("a"), e_logd25).\n' + "Bare identifiers that match predefined constants use those constants.\n" + "Operators: + - * / ** %.\n" + "Functions: sin(x), cos(x) (argument must be an angle).\n" + "Numbers are dimensionless.\n" + "String literals and + concatenation are supported.\n" + 'Modulo divisors must carry units (e.g. col("pa") % (180 * deg)).\n' + "Log columns (mag/dex) yield the bare exponent; multiply by the scale yourself.\n" + f"Available constants: {constants}." + ) diff --git a/uploader/app/lib/formula/values.py b/uploader/app/lib/formula/values.py new file mode 100644 index 0000000..7d570b0 --- /dev/null +++ b/uploader/app/lib/formula/values.py @@ -0,0 +1,19 @@ +import astropy.units as u +from astropy.units.function.core import FunctionUnitBase + +type Value = u.Quantity | str + + +def _is_logarithmic_column_unit(unit: u.Unit) -> bool: + return unit == u.mag or unit == u.dex or isinstance(unit, FunctionUnitBase) + + +def column_quantity(value: float | str, unit: str) -> Value: + if isinstance(value, str): + return value + if not unit: + return float(value) * u.dimensionless_unscaled + parsed = u.Unit(unit) + if _is_logarithmic_column_unit(parsed): + return float(value) * u.dimensionless_unscaled + return float(value) * parsed From eef481cac7b5fc70efdc940fd4ba60b498180307 Mon Sep 17 00:00:00 2001 From: kraysent Date: Mon, 6 Jul 2026 22:15:07 +0100 Subject: [PATCH 02/10] fix tests --- tests/test_formula_evaluate.py | 548 +++++++++++++++++---------------- 1 file changed, 288 insertions(+), 260 deletions(-) diff --git a/tests/test_formula_evaluate.py b/tests/test_formula_evaluate.py index 81ce451..6c6f97d 100644 --- a/tests/test_formula_evaluate.py +++ b/tests/test_formula_evaluate.py @@ -4,15 +4,7 @@ import numpy as np import pytest -from uploader.app.lib import expression as legacy_expression -from uploader.app.lib.formula import ( - ExpressionEvaluationError, - Value, - column_quantity, - evaluate, - parse, -) -from uploader.app.lib.formula.namespace import FUNCTIONS, NAMED_CONSTANTS +from uploader.app.lib.formula import ExpressionEvaluationError, column_quantity, evaluate, parse @dataclass @@ -21,7 +13,17 @@ class Col: unit: str = "" -def evaluate_expr(source: str, columns: dict[str, Col]) -> Value: +@dataclass +class EvalCase: + expression: str + columns: dict[str, Col] + result_val: str | float | None = None + result_unit: u.Unit | None = None + error: bool = False + name: str = "" + + +def evaluate_expr(source: str, columns: dict[str, Col]) -> object: built = {name: column_quantity(col.value, col.unit) for name, col in columns.items()} return evaluate(parse(source), built) @@ -36,272 +38,298 @@ def _sample_columns() -> dict[str, Col]: } -def _legacy_values_units(columns: dict[str, Col]) -> tuple[dict[str, float], dict[str, str]]: - values = {name: float(col.value) for name, col in columns.items() if isinstance(col.value, int | float)} - units = {name: col.unit for name, col in columns.items()} - return values, units - - -def test_isophotal_axis_expressions() -> None: - columns = _sample_columns() - a = evaluate_expr('3 * 10 ** col("logd25") * arcsec', columns) - assert isinstance(a, u.Quantity) - assert a.unit == u.arcsec - assert abs(a.value - 94.86832980505137) < 1e-6 - - e_a = evaluate_expr('3 * 10 ** col("logd25") * 2.302585093 * e_logd25 * arcsec', columns) - assert isinstance(e_a, u.Quantity) - assert e_a.unit == u.arcsec - assert e_a.value > 0 - - b = evaluate_expr('3 * 10 ** (col("logd25") - col("logr25")) * arcsec', columns) - assert isinstance(b, u.Quantity) - assert b.unit == u.arcsec - assert b.value > 0 - - e_b = evaluate_expr( - '3 * 10 ** (col("logd25") - col("logr25")) * 2.302585093 ' - '* (col("e_logd25") ** 2 + col("e_logr25") ** 2) ** 0.5 * arcsec', - columns, - ) - assert isinstance(e_b, u.Quantity) - assert e_b.unit == u.arcsec - assert e_b.value > 0 - - -def test_unit_aware_position_angle_modulo() -> None: - columns = _sample_columns() - pa = evaluate_expr('col("pa") % (180.0 * deg)', columns) - assert isinstance(pa, u.Quantity) - assert pa.unit == u.deg - assert pa.value == 10.0 - - -def test_isophotal_axis_expressions_with_logarithmic_column_units() -> None: - columns = _sample_columns() - for log_unit in ("mag", "dex"): - columns_with_log = { - **columns, - "logd25": Col(1.5, log_unit), - "logr25": Col(0.3, log_unit), - "e_logd25": Col(0.05, log_unit), - "e_logr25": Col(0.04, log_unit), - } - a = evaluate_expr('3 * 10 ** col("logd25") * arcsec', columns_with_log) - assert isinstance(a, u.Quantity) - assert a.unit == u.arcsec - assert abs(a.value - 94.86832980505137) < 1e-6 - - e_a = evaluate_expr('3 * 10 ** col("logd25") * 2.302585093 * e_logd25 * arcsec', columns_with_log) - assert isinstance(e_a, u.Quantity) - assert e_a.unit == u.arcsec - assert e_a.value > 0 - - e_b = evaluate_expr( - '3 * 10 ** (col("logd25") - col("logr25")) * 2.302585093 ' - '* (col("e_logd25") ** 2 + col("e_logr25") ** 2) ** 0.5 * arcsec', - columns_with_log, - ) - assert isinstance(e_b, u.Quantity) - assert e_b.unit == u.arcsec - assert e_b.value > 0 +def _log_columns(log_unit: str) -> dict[str, Col]: + return { + **_sample_columns(), + "logd25": Col(1.5, log_unit), + "logr25": Col(0.3, log_unit), + "e_logd25": Col(0.05, log_unit), + "e_logr25": Col(0.04, log_unit), + } -def test_isophotal_axis_expressions_with_hyperleda_units() -> None: - columns = { +def _hyperleda_columns() -> dict[str, Col]: + return { "logd25": Col(0.697, "dex(0.1 arcmin)"), "logr25": Col(0.13, "dex"), "e_logd25": Col(0.079, "dex(0.1 arcmin)"), "e_logr25": Col(0.028, "dex"), "pa": Col(161.14, "deg"), } - a = evaluate_expr('3 * 10 ** col("logd25") * arcsec', columns) - assert isinstance(a, u.Quantity) - assert a.unit == u.arcsec - assert a.value > 0 - - e_a = evaluate_expr('3 * 10 ** col("logd25") * 2.302585093 * e_logd25 * arcsec', columns) - assert isinstance(e_a, u.Quantity) - assert e_a.unit == u.arcsec - assert e_a.value > 0 - - b = evaluate_expr('3 * 10 ** (col("logd25") - col("logr25")) * arcsec', columns) - assert isinstance(b, u.Quantity) - assert b.unit == u.arcsec - assert b.value > 0 - - e_b = evaluate_expr( - '3 * 10 ** (col("logd25") - col("logr25")) * 2.302585093 ' - '* (col("e_logd25") ** 2 + col("e_logr25") ** 2) ** 0.5 * arcsec', - columns, - ) - assert isinstance(e_b, u.Quantity) - assert e_b.unit == u.arcsec - assert e_b.value > 0 - - -def test_surface_brightness_column_keeps_units() -> None: - columns = {"bri25": Col(23.162, "mag / arcsec2")} - bri25 = evaluate_expr('col("bri25")', columns) - assert isinstance(bri25, u.Quantity) - assert bri25.unit == u.Unit("mag/arcsec2") - - -STRING_CASES: list[tuple[str, dict[str, Col], str]] = [ - ('"abc"', {}, "abc"), - ('col("name")', {"name": Col("NGC 123")}, "NGC 123"), - ('col("a") + col("b")', {"a": Col("M"), "b": Col("82")}, "M82"), - ('"M " + col("id")', {"id": Col("82")}, "M 82"), - ('col("a") + " " + col("b")', {"a": Col("NGC"), "b": Col("905")}, "NGC 905"), -] - - -@pytest.mark.parametrize("source,columns,expected", STRING_CASES) -def test_string_evaluation(source: str, columns: dict[str, Col], expected: str) -> None: - assert evaluate_expr(source, columns) == expected - - -COERCION_CASES: list[tuple[float | str, str, object]] = [ - ("hello", "", "hello"), - (1.5, "", 1.5), - (1.5, "deg", u.Quantity(1.5, u.deg)), - (1.5, "mag", 1.5), - (1.5, "dex", 1.5), - (0.697, "dex(0.1 arcmin)", 0.697), - (23.162, "mag / arcsec2", u.Quantity(23.162, u.Unit("mag/arcsec2"))), -] - - -@pytest.mark.parametrize("value,unit,expected", COERCION_CASES) -def test_column_quantity(value: float | str, unit: str, expected: object) -> None: - result = column_quantity(value, unit) - if isinstance(expected, u.Quantity): - assert isinstance(result, u.Quantity) - assert result.unit == expected.unit - assert result.value == pytest.approx(expected.value) - else: - assert result == expected - - -def test_name_precedence_constant_over_column() -> None: - columns = {"pi": Col(1.0)} - result = evaluate_expr("pi", columns) - assert isinstance(result, u.Quantity) - assert result.value == pytest.approx(np.pi) - - col_result = evaluate_expr('col("pi")', columns) - assert isinstance(col_result, u.Quantity) - assert col_result.value == 1.0 - assert col_result.unit == u.dimensionless_unscaled - - -def test_trig_on_angle_column() -> None: - columns = {"pa": Col(30.0, "deg")} - result = evaluate_expr('sin(col("pa"))', columns) - assert isinstance(result, u.Quantity) - assert result.value == pytest.approx(0.5) - - -EVAL_ERROR_CASES: list[tuple[str, dict[str, Col]]] = [ - ("missing_col", {}), - ('col("missing")', {}), - ("arcsec + mag", {}), - ('col("name") + col("logd25")', {"name": Col("x"), "logd25": Col(1.5)}), - ('col("pa") % 180.0', {"pa": Col(190.0, "deg")}), - ('sin(col("x"))', {"x": Col(0.5)}), -] - - -@pytest.mark.parametrize("source,columns", EVAL_ERROR_CASES) -def test_evaluation_errors(source: str, columns: dict[str, Col]) -> None: - with pytest.raises(ExpressionEvaluationError): - evaluate_expr(source, columns) - - -PARITY_CASES: list[tuple[str, dict[str, Col]]] = [ - ('3 * 10 ** col("logd25") * arcsec', _sample_columns()), - ('3 * 10 ** col("logd25") * 2.302585093 * e_logd25 * arcsec', _sample_columns()), - ('3 * 10 ** (col("logd25") - col("logr25")) * arcsec', _sample_columns()), - ( - '3 * 10 ** (col("logd25") - col("logr25")) * 2.302585093 ' - '* (col("e_logd25") ** 2 + col("e_logr25") ** 2) ** 0.5 * arcsec', - _sample_columns(), - ), - ('col("bri25")', {"bri25": Col(23.162, "mag / arcsec2")}), -] -@pytest.mark.parametrize("source,columns", PARITY_CASES) -def test_parity_with_legacy_expression(source: str, columns: dict[str, Col]) -> None: - formula_result = evaluate_expr(source, columns) - values, units = _legacy_values_units(columns) - legacy_result = legacy_expression.parse(source).evaluate(values, units) - assert isinstance(formula_result, u.Quantity) - assert formula_result.unit == legacy_result.unit - assert formula_result.value == pytest.approx(legacy_result.value) - - -@pytest.mark.parametrize( - "source,columns", - [ - ( - '3 * 10 ** col("logd25") * arcsec', - { - **{k: Col(v.value, v.unit) for k, v in _sample_columns().items()}, - "logd25": Col(1.5, "mag"), - "logr25": Col(0.3, "mag"), - "e_logd25": Col(0.05, "mag"), - "e_logr25": Col(0.04, "mag"), - }, +EVAL_CASES: list[EvalCase] = [ + EvalCase( + name="isophotal_major_axis", + expression='3 * 10 ** col("logd25") * arcsec', + columns=_sample_columns(), + result_val=94.8683, + result_unit=u.arcsec, + ), + EvalCase( + name="isophotal_major_axis_error", + expression='3 * 10 ** col("logd25") * 2.302585093 * e_logd25 * arcsec', + columns=_sample_columns(), + result_val=10.9221, + result_unit=u.arcsec, + ), + EvalCase( + name="isophotal_minor_axis", + expression='3 * 10 ** (col("logd25") - col("logr25")) * arcsec', + columns=_sample_columns(), + result_val=47.5468, + result_unit=u.arcsec, + ), + EvalCase( + name="isophotal_minor_axis_error", + expression=( + '3 * 10 ** (col("logd25") - col("logr25")) * 2.302585093 ' + '* (col("e_logd25") ** 2 + col("e_logr25") ** 2) ** 0.5 * arcsec' + ), + columns=_sample_columns(), + result_val=7.0102, + result_unit=u.arcsec, + ), + EvalCase( + name="position_angle_modulo", + expression='col("pa") % (180.0 * deg)', + columns=_sample_columns(), + result_val=10.0, + result_unit=u.deg, + ), + EvalCase( + name="isophotal_major_axis_mag_units", + expression='3 * 10 ** col("logd25") * arcsec', + columns=_log_columns("mag"), + result_val=94.8683, + result_unit=u.arcsec, + ), + EvalCase( + name="isophotal_major_axis_error_mag_units", + expression='3 * 10 ** col("logd25") * 2.302585093 * e_logd25 * arcsec', + columns=_log_columns("mag"), + result_val=10.9221, + result_unit=u.arcsec, + ), + EvalCase( + name="isophotal_minor_axis_error_mag_units", + expression=( + '3 * 10 ** (col("logd25") - col("logr25")) * 2.302585093 ' + '* (col("e_logd25") ** 2 + col("e_logr25") ** 2) ** 0.5 * arcsec' ), - ( - '3 * 10 ** col("logd25") * arcsec', - { - **{k: Col(v.value, v.unit) for k, v in _sample_columns().items()}, - "logd25": Col(1.5, "dex"), - "logr25": Col(0.3, "dex"), - "e_logd25": Col(0.05, "dex"), - "e_logr25": Col(0.04, "dex"), - }, + columns=_log_columns("mag"), + result_val=7.0102, + result_unit=u.arcsec, + ), + EvalCase( + name="isophotal_major_axis_dex_units", + expression='3 * 10 ** col("logd25") * arcsec', + columns=_log_columns("dex"), + result_val=94.8683, + result_unit=u.arcsec, + ), + EvalCase( + name="isophotal_major_axis_error_dex_units", + expression='3 * 10 ** col("logd25") * 2.302585093 * e_logd25 * arcsec', + columns=_log_columns("dex"), + result_val=10.9221, + result_unit=u.arcsec, + ), + EvalCase( + name="isophotal_minor_axis_error_dex_units", + expression=( + '3 * 10 ** (col("logd25") - col("logr25")) * 2.302585093 ' + '* (col("e_logd25") ** 2 + col("e_logr25") ** 2) ** 0.5 * arcsec' ), - ( - '3 * 10 ** col("logd25") * arcsec', - { - "logd25": Col(0.697, "dex(0.1 arcmin)"), - "logr25": Col(0.13, "dex"), - "e_logd25": Col(0.079, "dex(0.1 arcmin)"), - "e_logr25": Col(0.028, "dex"), - "pa": Col(161.14, "deg"), - }, + columns=_log_columns("dex"), + result_val=7.0102, + result_unit=u.arcsec, + ), + EvalCase( + name="hyperleda_major_axis", + expression='3 * 10 ** col("logd25") * arcsec', + columns=_hyperleda_columns(), + result_val=14.9321, + result_unit=u.arcsec, + ), + EvalCase( + name="hyperleda_major_axis_error", + expression='3 * 10 ** col("logd25") * 2.302585093 * e_logd25 * arcsec', + columns=_hyperleda_columns(), + result_val=2.7162, + result_unit=u.arcsec, + ), + EvalCase( + name="hyperleda_minor_axis", + expression='3 * 10 ** (col("logd25") - col("logr25")) * arcsec', + columns=_hyperleda_columns(), + result_val=11.0693, + result_unit=u.arcsec, + ), + EvalCase( + name="hyperleda_minor_axis_error", + expression=( + '3 * 10 ** (col("logd25") - col("logr25")) * 2.302585093 ' + '* (col("e_logd25") ** 2 + col("e_logr25") ** 2) ** 0.5 * arcsec' ), - ], -) -def test_parity_with_logarithmic_units(source: str, columns: dict[str, Col]) -> None: - formula_result = evaluate_expr(source, columns) - values, units = _legacy_values_units(columns) - legacy_result = legacy_expression.parse(source).evaluate(values, units) - assert isinstance(formula_result, u.Quantity) - assert formula_result.unit == legacy_result.unit - assert formula_result.value == pytest.approx(legacy_result.value) - - -CONSTANT_GUARD_CASES = [(name, f"{name}") for name in NAMED_CONSTANTS] - + columns=_hyperleda_columns(), + result_val=2.1363, + result_unit=u.arcsec, + ), + EvalCase( + name="surface_brightness_column", + expression='col("bri25")', + columns={"bri25": Col(23.162, "mag / arcsec2")}, + result_val=23.162, + result_unit=u.Unit("mag/arcsec2"), + ), + EvalCase(name="string_literal", expression='"abc"', columns={}, result_val="abc"), + EvalCase( + name="string_column", + expression='col("name")', + columns={"name": Col("NGC 123")}, + result_val="NGC 123", + ), + EvalCase( + name="string_concat_columns", + expression='col("a") + col("b")', + columns={"a": Col("M"), "b": Col("82")}, + result_val="M82", + ), + EvalCase( + name="string_concat_literal_prefix", + expression='"M " + col("id")', + columns={"id": Col("82")}, + result_val="M 82", + ), + EvalCase( + name="string_concat_literal_separator", + expression='col("a") + " " + col("b")', + columns={"a": Col("NGC"), "b": Col("905")}, + result_val="NGC 905", + ), + EvalCase( + name="column_string_value", + expression='col("x")', + columns={"x": Col("hello")}, + result_val="hello", + ), + EvalCase( + name="column_dimensionless", + expression='col("x")', + columns={"x": Col(1.5)}, + result_val=1.5, + result_unit=u.dimensionless_unscaled, + ), + EvalCase( + name="column_angle_unit", + expression='col("x")', + columns={"x": Col(1.5, "deg")}, + result_val=1.5, + result_unit=u.deg, + ), + EvalCase( + name="column_mag_unit", + expression='col("x")', + columns={"x": Col(1.5, "mag")}, + result_val=1.5, + result_unit=u.dimensionless_unscaled, + ), + EvalCase( + name="column_dex_unit", + expression='col("x")', + columns={"x": Col(1.5, "dex")}, + result_val=1.5, + result_unit=u.dimensionless_unscaled, + ), + EvalCase( + name="column_dex_function_unit", + expression='col("x")', + columns={"x": Col(0.697, "dex(0.1 arcmin)")}, + result_val=0.697, + result_unit=u.dimensionless_unscaled, + ), + EvalCase( + name="constant_over_column", + expression="pi", + columns={"pi": Col(1.0)}, + result_val=3.1416, + result_unit=u.dimensionless_unscaled, + ), + EvalCase( + name="column_over_constant", + expression='col("pi")', + columns={"pi": Col(1.0)}, + result_val=1.0, + result_unit=u.dimensionless_unscaled, + ), + EvalCase( + name="trig_on_angle_column", + expression='sin(col("pa"))', + columns={"pa": Col(30.0, "deg")}, + result_val=0.5, + result_unit=u.dimensionless_unscaled, + ), + EvalCase( + name="constant_pi", + expression="pi", + columns={}, + result_val=3.1416, + result_unit=u.dimensionless_unscaled, + ), + EvalCase(name="constant_c", expression="c", columns={}, result_val=299792458.0, result_unit=u.Unit("m/s")), + EvalCase(name="constant_deg", expression="deg", columns={}, result_val=1.0, result_unit=u.deg), + EvalCase(name="constant_rad", expression="rad", columns={}, result_val=1.0, result_unit=u.rad), + EvalCase(name="constant_arcmin", expression="arcmin", columns={}, result_val=1.0, result_unit=u.arcmin), + EvalCase(name="constant_arcsec", expression="arcsec", columns={}, result_val=1.0, result_unit=u.arcsec), + EvalCase(name="constant_mag", expression="mag", columns={}, result_val=1.0, result_unit=u.mag), + EvalCase( + name="function_sin", + expression="sin(30 * deg)", + columns={}, + result_val=0.5, + result_unit=u.dimensionless_unscaled, + ), + EvalCase( + name="function_cos", expression="cos(0 * rad)", columns={}, result_val=1.0, result_unit=u.dimensionless_unscaled + ), + EvalCase(name="error_missing_column_bare", expression="missing_col", columns={}, error=True), + EvalCase(name="error_missing_column_call", expression='col("missing")', columns={}, error=True), + EvalCase(name="error_incompatible_units", expression="arcsec + mag", columns={}, error=True), + EvalCase( + name="error_string_plus_number", + expression='col("name") + col("logd25")', + columns={"name": Col("x"), "logd25": Col(1.5)}, + error=True, + ), + EvalCase( + name="error_modulo_dimensionless_divisor", + expression='col("pa") % 180.0', + columns={"pa": Col(190.0, "deg")}, + error=True, + ), + EvalCase( + name="error_trig_on_dimensionless", + expression='sin(col("x"))', + columns={"x": Col(0.5)}, + error=True, + ), +] -@pytest.mark.parametrize("name,source", CONSTANT_GUARD_CASES) -def test_named_constants_are_usable(name: str, source: str) -> None: - result = evaluate_expr(source, {}) - assert isinstance(result, u.Quantity) +@pytest.mark.parametrize("case", EVAL_CASES, ids=[case.name for case in EVAL_CASES]) +def test_evaluate(case: EvalCase) -> None: + if case.error: + with pytest.raises(ExpressionEvaluationError): + evaluate_expr(case.expression, case.columns) + return -FUNCTION_GUARD_CASES = [ - ("sin", "sin(30 * deg)"), - ("cos", "cos(0 * rad)"), -] + result = evaluate_expr(case.expression, case.columns) + if case.result_unit is None: + assert result == case.result_val + return -@pytest.mark.parametrize("name,source", FUNCTION_GUARD_CASES) -def test_functions_are_usable(name: str, source: str) -> None: - assert name in FUNCTIONS - result = evaluate_expr(source, {}) assert isinstance(result, u.Quantity) + assert result.unit == case.result_unit + np.testing.assert_almost_equal(result.value, case.result_val, decimal=4) From 08ce287f2c9666a45d77eb7aa58cd58c347c9681 Mon Sep 17 00:00:00 2001 From: kraysent Date: Mon, 6 Jul 2026 22:20:44 +0100 Subject: [PATCH 03/10] a bit better tests --- tests/test_formula_evaluate.py | 179 +++++++++++++++++---------------- 1 file changed, 94 insertions(+), 85 deletions(-) diff --git a/tests/test_formula_evaluate.py b/tests/test_formula_evaluate.py index 6c6f97d..6e5d782 100644 --- a/tests/test_formula_evaluate.py +++ b/tests/test_formula_evaluate.py @@ -28,55 +28,64 @@ def evaluate_expr(source: str, columns: dict[str, Col]) -> object: return evaluate(parse(source), built) -def _sample_columns() -> dict[str, Col]: - return { - "logd25": Col(1.5), - "logr25": Col(0.3), - "e_logd25": Col(0.05), - "e_logr25": Col(0.04), - "pa": Col(190.0, "deg"), - } - - -def _log_columns(log_unit: str) -> dict[str, Col]: - return { - **_sample_columns(), - "logd25": Col(1.5, log_unit), - "logr25": Col(0.3, log_unit), - "e_logd25": Col(0.05, log_unit), - "e_logr25": Col(0.04, log_unit), - } - - -def _hyperleda_columns() -> dict[str, Col]: - return { - "logd25": Col(0.697, "dex(0.1 arcmin)"), - "logr25": Col(0.13, "dex"), - "e_logd25": Col(0.079, "dex(0.1 arcmin)"), - "e_logr25": Col(0.028, "dex"), - "pa": Col(161.14, "deg"), - } +_COLUMNS: dict[str, Col] = { + "logd25": Col(1.5), + "logr25": Col(0.3), + "e_logd25": Col(0.05), + "e_logr25": Col(0.04), + "pa": Col(190.0, "deg"), + "logd25_mag": Col(1.5, "mag"), + "logr25_mag": Col(0.3, "mag"), + "e_logd25_mag": Col(0.05, "mag"), + "e_logr25_mag": Col(0.04, "mag"), + "logd25_dex": Col(1.5, "dex"), + "logr25_dex": Col(0.3, "dex"), + "e_logd25_dex": Col(0.05, "dex"), + "e_logr25_dex": Col(0.04, "dex"), + "logd25_hl": Col(0.697, "dex(0.1 arcmin)"), + "logr25_hl": Col(0.13, "dex"), + "e_logd25_hl": Col(0.079, "dex(0.1 arcmin)"), + "e_logr25_hl": Col(0.028, "dex"), + "pa_hl": Col(161.14, "deg"), + "bri25": Col(23.162, "mag / arcsec2"), + "designation": Col("NGC 123"), + "str_a": Col("M"), + "str_b": Col("82"), + "id": Col("82"), + "ngc_a": Col("NGC"), + "ngc_b": Col("905"), + "x_str": Col("hello"), + "x_dim": Col(1.5), + "x_deg": Col(1.5, "deg"), + "x_mag": Col(1.5, "mag"), + "x_dex": Col(1.5, "dex"), + "x_dex_fn": Col(0.697, "dex(0.1 arcmin)"), + "pi": Col(1.0), + "pa_trig": Col(30.0, "deg"), + "err_name": Col("x"), + "x_dimless": Col(0.5), +} EVAL_CASES: list[EvalCase] = [ EvalCase( name="isophotal_major_axis", expression='3 * 10 ** col("logd25") * arcsec', - columns=_sample_columns(), + columns=_COLUMNS, result_val=94.8683, result_unit=u.arcsec, ), EvalCase( name="isophotal_major_axis_error", expression='3 * 10 ** col("logd25") * 2.302585093 * e_logd25 * arcsec', - columns=_sample_columns(), + columns=_COLUMNS, result_val=10.9221, result_unit=u.arcsec, ), EvalCase( name="isophotal_minor_axis", expression='3 * 10 ** (col("logd25") - col("logr25")) * arcsec', - columns=_sample_columns(), + columns=_COLUMNS, result_val=47.5468, result_unit=u.arcsec, ), @@ -86,187 +95,187 @@ def _hyperleda_columns() -> dict[str, Col]: '3 * 10 ** (col("logd25") - col("logr25")) * 2.302585093 ' '* (col("e_logd25") ** 2 + col("e_logr25") ** 2) ** 0.5 * arcsec' ), - columns=_sample_columns(), + columns=_COLUMNS, result_val=7.0102, result_unit=u.arcsec, ), EvalCase( name="position_angle_modulo", expression='col("pa") % (180.0 * deg)', - columns=_sample_columns(), + columns=_COLUMNS, result_val=10.0, result_unit=u.deg, ), EvalCase( name="isophotal_major_axis_mag_units", - expression='3 * 10 ** col("logd25") * arcsec', - columns=_log_columns("mag"), + expression='3 * 10 ** col("logd25_mag") * arcsec', + columns=_COLUMNS, result_val=94.8683, result_unit=u.arcsec, ), EvalCase( name="isophotal_major_axis_error_mag_units", - expression='3 * 10 ** col("logd25") * 2.302585093 * e_logd25 * arcsec', - columns=_log_columns("mag"), + expression='3 * 10 ** col("logd25_mag") * 2.302585093 * e_logd25_mag * arcsec', + columns=_COLUMNS, result_val=10.9221, result_unit=u.arcsec, ), EvalCase( name="isophotal_minor_axis_error_mag_units", expression=( - '3 * 10 ** (col("logd25") - col("logr25")) * 2.302585093 ' - '* (col("e_logd25") ** 2 + col("e_logr25") ** 2) ** 0.5 * arcsec' + '3 * 10 ** (col("logd25_mag") - col("logr25_mag")) * 2.302585093 ' + '* (col("e_logd25_mag") ** 2 + col("e_logr25_mag") ** 2) ** 0.5 * arcsec' ), - columns=_log_columns("mag"), + columns=_COLUMNS, result_val=7.0102, result_unit=u.arcsec, ), EvalCase( name="isophotal_major_axis_dex_units", - expression='3 * 10 ** col("logd25") * arcsec', - columns=_log_columns("dex"), + expression='3 * 10 ** col("logd25_dex") * arcsec', + columns=_COLUMNS, result_val=94.8683, result_unit=u.arcsec, ), EvalCase( name="isophotal_major_axis_error_dex_units", - expression='3 * 10 ** col("logd25") * 2.302585093 * e_logd25 * arcsec', - columns=_log_columns("dex"), + expression='3 * 10 ** col("logd25_dex") * 2.302585093 * e_logd25_dex * arcsec', + columns=_COLUMNS, result_val=10.9221, result_unit=u.arcsec, ), EvalCase( name="isophotal_minor_axis_error_dex_units", expression=( - '3 * 10 ** (col("logd25") - col("logr25")) * 2.302585093 ' - '* (col("e_logd25") ** 2 + col("e_logr25") ** 2) ** 0.5 * arcsec' + '3 * 10 ** (col("logd25_dex") - col("logr25_dex")) * 2.302585093 ' + '* (col("e_logd25_dex") ** 2 + col("e_logr25_dex") ** 2) ** 0.5 * arcsec' ), - columns=_log_columns("dex"), + columns=_COLUMNS, result_val=7.0102, result_unit=u.arcsec, ), EvalCase( name="hyperleda_major_axis", - expression='3 * 10 ** col("logd25") * arcsec', - columns=_hyperleda_columns(), + expression='3 * 10 ** col("logd25_hl") * arcsec', + columns=_COLUMNS, result_val=14.9321, result_unit=u.arcsec, ), EvalCase( name="hyperleda_major_axis_error", - expression='3 * 10 ** col("logd25") * 2.302585093 * e_logd25 * arcsec', - columns=_hyperleda_columns(), + expression='3 * 10 ** col("logd25_hl") * 2.302585093 * e_logd25_hl * arcsec', + columns=_COLUMNS, result_val=2.7162, result_unit=u.arcsec, ), EvalCase( name="hyperleda_minor_axis", - expression='3 * 10 ** (col("logd25") - col("logr25")) * arcsec', - columns=_hyperleda_columns(), + expression='3 * 10 ** (col("logd25_hl") - col("logr25_hl")) * arcsec', + columns=_COLUMNS, result_val=11.0693, result_unit=u.arcsec, ), EvalCase( name="hyperleda_minor_axis_error", expression=( - '3 * 10 ** (col("logd25") - col("logr25")) * 2.302585093 ' - '* (col("e_logd25") ** 2 + col("e_logr25") ** 2) ** 0.5 * arcsec' + '3 * 10 ** (col("logd25_hl") - col("logr25_hl")) * 2.302585093 ' + '* (col("e_logd25_hl") ** 2 + col("e_logr25_hl") ** 2) ** 0.5 * arcsec' ), - columns=_hyperleda_columns(), + columns=_COLUMNS, result_val=2.1363, result_unit=u.arcsec, ), EvalCase( name="surface_brightness_column", expression='col("bri25")', - columns={"bri25": Col(23.162, "mag / arcsec2")}, + columns=_COLUMNS, result_val=23.162, result_unit=u.Unit("mag/arcsec2"), ), EvalCase(name="string_literal", expression='"abc"', columns={}, result_val="abc"), EvalCase( name="string_column", - expression='col("name")', - columns={"name": Col("NGC 123")}, + expression='col("designation")', + columns=_COLUMNS, result_val="NGC 123", ), EvalCase( name="string_concat_columns", - expression='col("a") + col("b")', - columns={"a": Col("M"), "b": Col("82")}, + expression='col("str_a") + col("str_b")', + columns=_COLUMNS, result_val="M82", ), EvalCase( name="string_concat_literal_prefix", expression='"M " + col("id")', - columns={"id": Col("82")}, + columns=_COLUMNS, result_val="M 82", ), EvalCase( name="string_concat_literal_separator", - expression='col("a") + " " + col("b")', - columns={"a": Col("NGC"), "b": Col("905")}, + expression='col("ngc_a") + " " + col("ngc_b")', + columns=_COLUMNS, result_val="NGC 905", ), EvalCase( name="column_string_value", - expression='col("x")', - columns={"x": Col("hello")}, + expression='col("x_str")', + columns=_COLUMNS, result_val="hello", ), EvalCase( name="column_dimensionless", - expression='col("x")', - columns={"x": Col(1.5)}, + expression='col("x_dim")', + columns=_COLUMNS, result_val=1.5, result_unit=u.dimensionless_unscaled, ), EvalCase( name="column_angle_unit", - expression='col("x")', - columns={"x": Col(1.5, "deg")}, + expression='col("x_deg")', + columns=_COLUMNS, result_val=1.5, result_unit=u.deg, ), EvalCase( name="column_mag_unit", - expression='col("x")', - columns={"x": Col(1.5, "mag")}, + expression='col("x_mag")', + columns=_COLUMNS, result_val=1.5, result_unit=u.dimensionless_unscaled, ), EvalCase( name="column_dex_unit", - expression='col("x")', - columns={"x": Col(1.5, "dex")}, + expression='col("x_dex")', + columns=_COLUMNS, result_val=1.5, result_unit=u.dimensionless_unscaled, ), EvalCase( name="column_dex_function_unit", - expression='col("x")', - columns={"x": Col(0.697, "dex(0.1 arcmin)")}, + expression='col("x_dex_fn")', + columns=_COLUMNS, result_val=0.697, result_unit=u.dimensionless_unscaled, ), EvalCase( name="constant_over_column", expression="pi", - columns={"pi": Col(1.0)}, + columns=_COLUMNS, result_val=3.1416, result_unit=u.dimensionless_unscaled, ), EvalCase( name="column_over_constant", expression='col("pi")', - columns={"pi": Col(1.0)}, + columns=_COLUMNS, result_val=1.0, result_unit=u.dimensionless_unscaled, ), EvalCase( name="trig_on_angle_column", - expression='sin(col("pa"))', - columns={"pa": Col(30.0, "deg")}, + expression='sin(col("pa_trig"))', + columns=_COLUMNS, result_val=0.5, result_unit=u.dimensionless_unscaled, ), @@ -298,20 +307,20 @@ def _hyperleda_columns() -> dict[str, Col]: EvalCase(name="error_incompatible_units", expression="arcsec + mag", columns={}, error=True), EvalCase( name="error_string_plus_number", - expression='col("name") + col("logd25")', - columns={"name": Col("x"), "logd25": Col(1.5)}, + expression='col("err_name") + col("logd25")', + columns=_COLUMNS, error=True, ), EvalCase( name="error_modulo_dimensionless_divisor", expression='col("pa") % 180.0', - columns={"pa": Col(190.0, "deg")}, + columns=_COLUMNS, error=True, ), EvalCase( name="error_trig_on_dimensionless", - expression='sin(col("x"))', - columns={"x": Col(0.5)}, + expression='sin(col("x_dimless"))', + columns=_COLUMNS, error=True, ), ] From c08f6fdda218200970d2643177330a7a7b2dfbaf Mon Sep 17 00:00:00 2001 From: kraysent Date: Mon, 6 Jul 2026 22:23:13 +0100 Subject: [PATCH 04/10] remove duplicate cases --- tests/test_formula_evaluate.py | 50 ---------------------------------- 1 file changed, 50 deletions(-) diff --git a/tests/test_formula_evaluate.py b/tests/test_formula_evaluate.py index 6e5d782..44a6681 100644 --- a/tests/test_formula_evaluate.py +++ b/tests/test_formula_evaluate.py @@ -38,10 +38,6 @@ def evaluate_expr(source: str, columns: dict[str, Col]) -> object: "logr25_mag": Col(0.3, "mag"), "e_logd25_mag": Col(0.05, "mag"), "e_logr25_mag": Col(0.04, "mag"), - "logd25_dex": Col(1.5, "dex"), - "logr25_dex": Col(0.3, "dex"), - "e_logd25_dex": Col(0.05, "dex"), - "e_logr25_dex": Col(0.04, "dex"), "logd25_hl": Col(0.697, "dex(0.1 arcmin)"), "logr25_hl": Col(0.13, "dex"), "e_logd25_hl": Col(0.079, "dex(0.1 arcmin)"), @@ -54,11 +50,9 @@ def evaluate_expr(source: str, columns: dict[str, Col]) -> object: "id": Col("82"), "ngc_a": Col("NGC"), "ngc_b": Col("905"), - "x_str": Col("hello"), "x_dim": Col(1.5), "x_deg": Col(1.5, "deg"), "x_mag": Col(1.5, "mag"), - "x_dex": Col(1.5, "dex"), "x_dex_fn": Col(0.697, "dex(0.1 arcmin)"), "pi": Col(1.0), "pa_trig": Col(30.0, "deg"), @@ -130,30 +124,6 @@ def evaluate_expr(source: str, columns: dict[str, Col]) -> object: result_val=7.0102, result_unit=u.arcsec, ), - EvalCase( - name="isophotal_major_axis_dex_units", - expression='3 * 10 ** col("logd25_dex") * arcsec', - columns=_COLUMNS, - result_val=94.8683, - result_unit=u.arcsec, - ), - EvalCase( - name="isophotal_major_axis_error_dex_units", - expression='3 * 10 ** col("logd25_dex") * 2.302585093 * e_logd25_dex * arcsec', - columns=_COLUMNS, - result_val=10.9221, - result_unit=u.arcsec, - ), - EvalCase( - name="isophotal_minor_axis_error_dex_units", - expression=( - '3 * 10 ** (col("logd25_dex") - col("logr25_dex")) * 2.302585093 ' - '* (col("e_logd25_dex") ** 2 + col("e_logr25_dex") ** 2) ** 0.5 * arcsec' - ), - columns=_COLUMNS, - result_val=7.0102, - result_unit=u.arcsec, - ), EvalCase( name="hyperleda_major_axis", expression='3 * 10 ** col("logd25_hl") * arcsec', @@ -217,12 +187,6 @@ def evaluate_expr(source: str, columns: dict[str, Col]) -> object: columns=_COLUMNS, result_val="NGC 905", ), - EvalCase( - name="column_string_value", - expression='col("x_str")', - columns=_COLUMNS, - result_val="hello", - ), EvalCase( name="column_dimensionless", expression='col("x_dim")', @@ -244,13 +208,6 @@ def evaluate_expr(source: str, columns: dict[str, Col]) -> object: result_val=1.5, result_unit=u.dimensionless_unscaled, ), - EvalCase( - name="column_dex_unit", - expression='col("x_dex")', - columns=_COLUMNS, - result_val=1.5, - result_unit=u.dimensionless_unscaled, - ), EvalCase( name="column_dex_function_unit", expression='col("x_dex_fn")', @@ -258,13 +215,6 @@ def evaluate_expr(source: str, columns: dict[str, Col]) -> object: result_val=0.697, result_unit=u.dimensionless_unscaled, ), - EvalCase( - name="constant_over_column", - expression="pi", - columns=_COLUMNS, - result_val=3.1416, - result_unit=u.dimensionless_unscaled, - ), EvalCase( name="column_over_constant", expression='col("pi")', From 88372bdc7b7d779f988baaa3e8028aa878ed77aa Mon Sep 17 00:00:00 2001 From: kraysent Date: Mon, 6 Jul 2026 22:32:03 +0100 Subject: [PATCH 05/10] rename columns and tests --- tests/test_formula_evaluate.py | 145 +++++++++++++++------------------ 1 file changed, 66 insertions(+), 79 deletions(-) diff --git a/tests/test_formula_evaluate.py b/tests/test_formula_evaluate.py index 44a6681..e2de01f 100644 --- a/tests/test_formula_evaluate.py +++ b/tests/test_formula_evaluate.py @@ -29,135 +29,130 @@ def evaluate_expr(source: str, columns: dict[str, Col]) -> object: _COLUMNS: dict[str, Col] = { - "logd25": Col(1.5), - "logr25": Col(0.3), - "e_logd25": Col(0.05), - "e_logr25": Col(0.04), - "pa": Col(190.0, "deg"), - "logd25_mag": Col(1.5, "mag"), - "logr25_mag": Col(0.3, "mag"), - "e_logd25_mag": Col(0.05, "mag"), - "e_logr25_mag": Col(0.04, "mag"), - "logd25_hl": Col(0.697, "dex(0.1 arcmin)"), - "logr25_hl": Col(0.13, "dex"), - "e_logd25_hl": Col(0.079, "dex(0.1 arcmin)"), - "e_logr25_hl": Col(0.028, "dex"), - "pa_hl": Col(161.14, "deg"), - "bri25": Col(23.162, "mag / arcsec2"), - "designation": Col("NGC 123"), - "str_a": Col("M"), - "str_b": Col("82"), - "id": Col("82"), - "ngc_a": Col("NGC"), - "ngc_b": Col("905"), - "x_dim": Col(1.5), - "x_deg": Col(1.5, "deg"), - "x_mag": Col(1.5, "mag"), - "x_dex_fn": Col(0.697, "dex(0.1 arcmin)"), - "pi": Col(1.0), - "pa_trig": Col(30.0, "deg"), - "err_name": Col("x"), - "x_dimless": Col(0.5), + "float_col_1": Col(1.5), + "float_col_2": Col(0.3), + "float_col_3": Col(0.05), + "float_col_4": Col(0.04), + "angle_col_1": Col(190.0, "deg"), + "float_col_1_mag": Col(1.5, "mag"), + "float_col_2_mag": Col(0.3, "mag"), + "float_col_3_mag": Col(0.05, "mag"), + "float_col_4_mag": Col(0.04, "mag"), + "float_col_1_dex": Col(0.697, "dex(0.1 arcmin)"), + "float_col_2_dex": Col(0.13, "dex"), + "float_col_3_dex": Col(0.079, "dex(0.1 arcmin)"), + "float_col_4_dex": Col(0.028, "dex"), + "brightness_col_1": Col(23.162, "mag / arcsec2"), + "string_col_1": Col("NGC 123"), + "string_col_2": Col("M"), + "string_col_3": Col("82"), + "string_col_4": Col("NGC"), + "string_col_5": Col("905"), + "angle_col_2": Col(1.5, "deg"), + "const_col_1": Col(1.0), + "angle_col_3": Col(30.0, "deg"), + "string_col_6": Col("x"), + "float_col_dimless": Col(0.5), } EVAL_CASES: list[EvalCase] = [ EvalCase( - name="isophotal_major_axis", - expression='3 * 10 ** col("logd25") * arcsec', + name="power_ten_scale_length", + expression='3 * 10 ** col("float_col_1") * arcsec', columns=_COLUMNS, result_val=94.8683, result_unit=u.arcsec, ), EvalCase( - name="isophotal_major_axis_error", - expression='3 * 10 ** col("logd25") * 2.302585093 * e_logd25 * arcsec', + name="power_ten_scale_length_error", + expression='3 * 10 ** col("float_col_1") * 2.302585093 * float_col_3 * arcsec', columns=_COLUMNS, result_val=10.9221, result_unit=u.arcsec, ), EvalCase( - name="isophotal_minor_axis", - expression='3 * 10 ** (col("logd25") - col("logr25")) * arcsec', + name="power_ten_scale_length_diff", + expression='3 * 10 ** (col("float_col_1") - col("float_col_2")) * arcsec', columns=_COLUMNS, result_val=47.5468, result_unit=u.arcsec, ), EvalCase( - name="isophotal_minor_axis_error", + name="power_ten_scale_length_diff_error", expression=( - '3 * 10 ** (col("logd25") - col("logr25")) * 2.302585093 ' - '* (col("e_logd25") ** 2 + col("e_logr25") ** 2) ** 0.5 * arcsec' + '3 * 10 ** (col("float_col_1") - col("float_col_2")) * 2.302585093 ' + '* (col("float_col_3") ** 2 + col("float_col_4") ** 2) ** 0.5 * arcsec' ), columns=_COLUMNS, result_val=7.0102, result_unit=u.arcsec, ), EvalCase( - name="position_angle_modulo", - expression='col("pa") % (180.0 * deg)', + name="angle_modulo", + expression='col("angle_col_1") % (180.0 * deg)', columns=_COLUMNS, result_val=10.0, result_unit=u.deg, ), EvalCase( - name="isophotal_major_axis_mag_units", - expression='3 * 10 ** col("logd25_mag") * arcsec', + name="power_ten_scale_length_mag_units", + expression='3 * 10 ** col("float_col_1_mag") * arcsec', columns=_COLUMNS, result_val=94.8683, result_unit=u.arcsec, ), EvalCase( - name="isophotal_major_axis_error_mag_units", - expression='3 * 10 ** col("logd25_mag") * 2.302585093 * e_logd25_mag * arcsec', + name="power_ten_scale_length_error_mag_units", + expression='3 * 10 ** col("float_col_1_mag") * 2.302585093 * float_col_3_mag * arcsec', columns=_COLUMNS, result_val=10.9221, result_unit=u.arcsec, ), EvalCase( - name="isophotal_minor_axis_error_mag_units", + name="power_ten_scale_length_diff_error_mag_units", expression=( - '3 * 10 ** (col("logd25_mag") - col("logr25_mag")) * 2.302585093 ' - '* (col("e_logd25_mag") ** 2 + col("e_logr25_mag") ** 2) ** 0.5 * arcsec' + '3 * 10 ** (col("float_col_1_mag") - col("float_col_2_mag")) * 2.302585093 ' + '* (col("float_col_3_mag") ** 2 + col("float_col_4_mag") ** 2) ** 0.5 * arcsec' ), columns=_COLUMNS, result_val=7.0102, result_unit=u.arcsec, ), EvalCase( - name="hyperleda_major_axis", - expression='3 * 10 ** col("logd25_hl") * arcsec', + name="dex_scale_length", + expression='3 * 10 ** col("float_col_1_dex") * arcsec', columns=_COLUMNS, result_val=14.9321, result_unit=u.arcsec, ), EvalCase( - name="hyperleda_major_axis_error", - expression='3 * 10 ** col("logd25_hl") * 2.302585093 * e_logd25_hl * arcsec', + name="dex_scale_length_error", + expression='3 * 10 ** col("float_col_1_dex") * 2.302585093 * float_col_3_dex * arcsec', columns=_COLUMNS, result_val=2.7162, result_unit=u.arcsec, ), EvalCase( - name="hyperleda_minor_axis", - expression='3 * 10 ** (col("logd25_hl") - col("logr25_hl")) * arcsec', + name="dex_scale_length_diff", + expression='3 * 10 ** (col("float_col_1_dex") - col("float_col_2_dex")) * arcsec', columns=_COLUMNS, result_val=11.0693, result_unit=u.arcsec, ), EvalCase( - name="hyperleda_minor_axis_error", + name="dex_scale_length_diff_error", expression=( - '3 * 10 ** (col("logd25_hl") - col("logr25_hl")) * 2.302585093 ' - '* (col("e_logd25_hl") ** 2 + col("e_logr25_hl") ** 2) ** 0.5 * arcsec' + '3 * 10 ** (col("float_col_1_dex") - col("float_col_2_dex")) * 2.302585093 ' + '* (col("float_col_3_dex") ** 2 + col("float_col_4_dex") ** 2) ** 0.5 * arcsec' ), columns=_COLUMNS, result_val=2.1363, result_unit=u.arcsec, ), EvalCase( - name="surface_brightness_column", - expression='col("bri25")', + name="brightness_column", + expression='col("brightness_col_1")', columns=_COLUMNS, result_val=23.162, result_unit=u.Unit("mag/arcsec2"), @@ -165,66 +160,66 @@ def evaluate_expr(source: str, columns: dict[str, Col]) -> object: EvalCase(name="string_literal", expression='"abc"', columns={}, result_val="abc"), EvalCase( name="string_column", - expression='col("designation")', + expression='col("string_col_1")', columns=_COLUMNS, result_val="NGC 123", ), EvalCase( name="string_concat_columns", - expression='col("str_a") + col("str_b")', + expression='col("string_col_2") + col("string_col_3")', columns=_COLUMNS, result_val="M82", ), EvalCase( name="string_concat_literal_prefix", - expression='"M " + col("id")', + expression='"M " + col("string_col_3")', columns=_COLUMNS, result_val="M 82", ), EvalCase( name="string_concat_literal_separator", - expression='col("ngc_a") + " " + col("ngc_b")', + expression='col("string_col_4") + " " + col("string_col_5")', columns=_COLUMNS, result_val="NGC 905", ), EvalCase( name="column_dimensionless", - expression='col("x_dim")', + expression='col("float_col_1")', columns=_COLUMNS, result_val=1.5, result_unit=u.dimensionless_unscaled, ), EvalCase( name="column_angle_unit", - expression='col("x_deg")', + expression='col("angle_col_2")', columns=_COLUMNS, result_val=1.5, result_unit=u.deg, ), EvalCase( name="column_mag_unit", - expression='col("x_mag")', + expression='col("float_col_1_mag")', columns=_COLUMNS, result_val=1.5, result_unit=u.dimensionless_unscaled, ), EvalCase( name="column_dex_function_unit", - expression='col("x_dex_fn")', + expression='col("float_col_1_dex")', columns=_COLUMNS, result_val=0.697, result_unit=u.dimensionless_unscaled, ), EvalCase( name="column_over_constant", - expression='col("pi")', + expression='col("const_col_1")', columns=_COLUMNS, result_val=1.0, result_unit=u.dimensionless_unscaled, ), EvalCase( name="trig_on_angle_column", - expression='sin(col("pa_trig"))', + expression='sin(col("angle_col_3"))', columns=_COLUMNS, result_val=0.5, result_unit=u.dimensionless_unscaled, @@ -236,12 +231,7 @@ def evaluate_expr(source: str, columns: dict[str, Col]) -> object: result_val=3.1416, result_unit=u.dimensionless_unscaled, ), - EvalCase(name="constant_c", expression="c", columns={}, result_val=299792458.0, result_unit=u.Unit("m/s")), - EvalCase(name="constant_deg", expression="deg", columns={}, result_val=1.0, result_unit=u.deg), - EvalCase(name="constant_rad", expression="rad", columns={}, result_val=1.0, result_unit=u.rad), EvalCase(name="constant_arcmin", expression="arcmin", columns={}, result_val=1.0, result_unit=u.arcmin), - EvalCase(name="constant_arcsec", expression="arcsec", columns={}, result_val=1.0, result_unit=u.arcsec), - EvalCase(name="constant_mag", expression="mag", columns={}, result_val=1.0, result_unit=u.mag), EvalCase( name="function_sin", expression="sin(30 * deg)", @@ -249,27 +239,24 @@ def evaluate_expr(source: str, columns: dict[str, Col]) -> object: result_val=0.5, result_unit=u.dimensionless_unscaled, ), - EvalCase( - name="function_cos", expression="cos(0 * rad)", columns={}, result_val=1.0, result_unit=u.dimensionless_unscaled - ), EvalCase(name="error_missing_column_bare", expression="missing_col", columns={}, error=True), EvalCase(name="error_missing_column_call", expression='col("missing")', columns={}, error=True), EvalCase(name="error_incompatible_units", expression="arcsec + mag", columns={}, error=True), EvalCase( name="error_string_plus_number", - expression='col("err_name") + col("logd25")', + expression='col("string_col_6") + col("float_col_1")', columns=_COLUMNS, error=True, ), EvalCase( name="error_modulo_dimensionless_divisor", - expression='col("pa") % 180.0', + expression='col("angle_col_1") % 180.0', columns=_COLUMNS, error=True, ), EvalCase( name="error_trig_on_dimensionless", - expression='sin(col("x_dimless"))', + expression='sin(col("float_col_dimless"))', columns=_COLUMNS, error=True, ), From ecdf397f65684f690bad8abf2294babb53784469 Mon Sep 17 00:00:00 2001 From: kraysent Date: Mon, 6 Jul 2026 22:39:55 +0100 Subject: [PATCH 06/10] trim test suite --- tests/test_formula_evaluate.py | 186 ++++----------------------------- 1 file changed, 21 insertions(+), 165 deletions(-) diff --git a/tests/test_formula_evaluate.py b/tests/test_formula_evaluate.py index e2de01f..91f867b 100644 --- a/tests/test_formula_evaluate.py +++ b/tests/test_formula_evaluate.py @@ -29,130 +29,35 @@ def evaluate_expr(source: str, columns: dict[str, Col]) -> object: _COLUMNS: dict[str, Col] = { - "float_col_1": Col(1.5), - "float_col_2": Col(0.3), - "float_col_3": Col(0.05), - "float_col_4": Col(0.04), - "angle_col_1": Col(190.0, "deg"), - "float_col_1_mag": Col(1.5, "mag"), - "float_col_2_mag": Col(0.3, "mag"), - "float_col_3_mag": Col(0.05, "mag"), - "float_col_4_mag": Col(0.04, "mag"), - "float_col_1_dex": Col(0.697, "dex(0.1 arcmin)"), - "float_col_2_dex": Col(0.13, "dex"), - "float_col_3_dex": Col(0.079, "dex(0.1 arcmin)"), - "float_col_4_dex": Col(0.028, "dex"), - "brightness_col_1": Col(23.162, "mag / arcsec2"), + "float_col": Col(1.5), + "angle_col": Col(190.0, "deg"), + "float_col_mag": Col(1.5, "mag"), + "float_col_dex": Col(0.697, "dex(0.1 arcmin)"), + "brightness_col": Col(23.162, "mag / arcsec2"), "string_col_1": Col("NGC 123"), "string_col_2": Col("M"), - "string_col_3": Col("82"), - "string_col_4": Col("NGC"), - "string_col_5": Col("905"), - "angle_col_2": Col(1.5, "deg"), - "const_col_1": Col(1.0), - "angle_col_3": Col(30.0, "deg"), - "string_col_6": Col("x"), "float_col_dimless": Col(0.5), } EVAL_CASES: list[EvalCase] = [ EvalCase( - name="power_ten_scale_length", - expression='3 * 10 ** col("float_col_1") * arcsec', + name="dimentionless_expr", + expression='3 * 10 ** col("float_col") * arcsec', columns=_COLUMNS, result_val=94.8683, result_unit=u.arcsec, ), EvalCase( - name="power_ten_scale_length_error", - expression='3 * 10 ** col("float_col_1") * 2.302585093 * float_col_3 * arcsec', - columns=_COLUMNS, - result_val=10.9221, - result_unit=u.arcsec, - ), - EvalCase( - name="power_ten_scale_length_diff", - expression='3 * 10 ** (col("float_col_1") - col("float_col_2")) * arcsec', - columns=_COLUMNS, - result_val=47.5468, - result_unit=u.arcsec, - ), - EvalCase( - name="power_ten_scale_length_diff_error", - expression=( - '3 * 10 ** (col("float_col_1") - col("float_col_2")) * 2.302585093 ' - '* (col("float_col_3") ** 2 + col("float_col_4") ** 2) ** 0.5 * arcsec' - ), - columns=_COLUMNS, - result_val=7.0102, - result_unit=u.arcsec, - ), - EvalCase( - name="angle_modulo", - expression='col("angle_col_1") % (180.0 * deg)', + name="modulo_edgecase", + expression='col("angle_col") % (180.0 * deg)', columns=_COLUMNS, result_val=10.0, result_unit=u.deg, ), EvalCase( - name="power_ten_scale_length_mag_units", - expression='3 * 10 ** col("float_col_1_mag") * arcsec', - columns=_COLUMNS, - result_val=94.8683, - result_unit=u.arcsec, - ), - EvalCase( - name="power_ten_scale_length_error_mag_units", - expression='3 * 10 ** col("float_col_1_mag") * 2.302585093 * float_col_3_mag * arcsec', - columns=_COLUMNS, - result_val=10.9221, - result_unit=u.arcsec, - ), - EvalCase( - name="power_ten_scale_length_diff_error_mag_units", - expression=( - '3 * 10 ** (col("float_col_1_mag") - col("float_col_2_mag")) * 2.302585093 ' - '* (col("float_col_3_mag") ** 2 + col("float_col_4_mag") ** 2) ** 0.5 * arcsec' - ), - columns=_COLUMNS, - result_val=7.0102, - result_unit=u.arcsec, - ), - EvalCase( - name="dex_scale_length", - expression='3 * 10 ** col("float_col_1_dex") * arcsec', - columns=_COLUMNS, - result_val=14.9321, - result_unit=u.arcsec, - ), - EvalCase( - name="dex_scale_length_error", - expression='3 * 10 ** col("float_col_1_dex") * 2.302585093 * float_col_3_dex * arcsec', - columns=_COLUMNS, - result_val=2.7162, - result_unit=u.arcsec, - ), - EvalCase( - name="dex_scale_length_diff", - expression='3 * 10 ** (col("float_col_1_dex") - col("float_col_2_dex")) * arcsec', - columns=_COLUMNS, - result_val=11.0693, - result_unit=u.arcsec, - ), - EvalCase( - name="dex_scale_length_diff_error", - expression=( - '3 * 10 ** (col("float_col_1_dex") - col("float_col_2_dex")) * 2.302585093 ' - '* (col("float_col_3_dex") ** 2 + col("float_col_4_dex") ** 2) ** 0.5 * arcsec' - ), - columns=_COLUMNS, - result_val=2.1363, - result_unit=u.arcsec, - ), - EvalCase( - name="brightness_column", - expression='col("brightness_col_1")', + name="single_column", + expression='col("brightness_col")', columns=_COLUMNS, result_val=23.162, result_unit=u.Unit("mag/arcsec2"), @@ -166,91 +71,42 @@ def evaluate_expr(source: str, columns: dict[str, Col]) -> object: ), EvalCase( name="string_concat_columns", - expression='col("string_col_2") + col("string_col_3")', + expression='col("string_col_2") + " " + col("string_col_1")', columns=_COLUMNS, - result_val="M82", - ), - EvalCase( - name="string_concat_literal_prefix", - expression='"M " + col("string_col_3")', - columns=_COLUMNS, - result_val="M 82", - ), - EvalCase( - name="string_concat_literal_separator", - expression='col("string_col_4") + " " + col("string_col_5")', - columns=_COLUMNS, - result_val="NGC 905", - ), - EvalCase( - name="column_dimensionless", - expression='col("float_col_1")', - columns=_COLUMNS, - result_val=1.5, - result_unit=u.dimensionless_unscaled, + result_val="M NGC 123", ), EvalCase( - name="column_angle_unit", - expression='col("angle_col_2")', - columns=_COLUMNS, - result_val=1.5, - result_unit=u.deg, - ), - EvalCase( - name="column_mag_unit", - expression='col("float_col_1_mag")', - columns=_COLUMNS, - result_val=1.5, - result_unit=u.dimensionless_unscaled, - ), - EvalCase( - name="column_dex_function_unit", - expression='col("float_col_1_dex")', + name="function_unit", + expression='col("float_col_dex")', columns=_COLUMNS, result_val=0.697, result_unit=u.dimensionless_unscaled, ), EvalCase( - name="column_over_constant", - expression='col("const_col_1")', + name="nested_functions", + expression='sin(col("angle_col"))', columns=_COLUMNS, - result_val=1.0, + result_val=-0.1736, result_unit=u.dimensionless_unscaled, ), EvalCase( - name="trig_on_angle_column", - expression='sin(col("angle_col_3"))', - columns=_COLUMNS, - result_val=0.5, - result_unit=u.dimensionless_unscaled, - ), - EvalCase( - name="constant_pi", + name="constant", expression="pi", columns={}, result_val=3.1416, result_unit=u.dimensionless_unscaled, ), - EvalCase(name="constant_arcmin", expression="arcmin", columns={}, result_val=1.0, result_unit=u.arcmin), - EvalCase( - name="function_sin", - expression="sin(30 * deg)", - columns={}, - result_val=0.5, - result_unit=u.dimensionless_unscaled, - ), - EvalCase(name="error_missing_column_bare", expression="missing_col", columns={}, error=True), EvalCase(name="error_missing_column_call", expression='col("missing")', columns={}, error=True), EvalCase(name="error_incompatible_units", expression="arcsec + mag", columns={}, error=True), EvalCase( name="error_string_plus_number", - expression='col("string_col_6") + col("float_col_1")', + expression='col("string_col_1") + col("float_col")', columns=_COLUMNS, error=True, ), EvalCase( name="error_modulo_dimensionless_divisor", - expression='col("angle_col_1") % 180.0', + expression='col("angle_col") % 180.0', columns=_COLUMNS, error=True, ), From e492d9865f267f309a2ec4eb2f681d108f70bb5a Mon Sep 17 00:00:00 2001 From: kraysent Date: Mon, 6 Jul 2026 22:46:20 +0100 Subject: [PATCH 07/10] support columns --- tests/test_formula_evaluate.py | 58 +++++++++++++++++++++++++++--- uploader/app/lib/formula/values.py | 17 ++++++--- 2 files changed, 66 insertions(+), 9 deletions(-) diff --git a/tests/test_formula_evaluate.py b/tests/test_formula_evaluate.py index 91f867b..286678f 100644 --- a/tests/test_formula_evaluate.py +++ b/tests/test_formula_evaluate.py @@ -9,7 +9,7 @@ @dataclass class Col: - value: float | str + value: float | str | list[float] | list[str] unit: str = "" @@ -17,7 +17,7 @@ class Col: class EvalCase: expression: str columns: dict[str, Col] - result_val: str | float | None = None + result_val: str | float | list[float] | list[str] | None = None result_unit: u.Unit | None = None error: bool = False name: str = "" @@ -37,6 +37,10 @@ def evaluate_expr(source: str, columns: dict[str, Col]) -> object: "string_col_1": Col("NGC 123"), "string_col_2": Col("M"), "float_col_dimless": Col(0.5), + "vec_col": Col([1.0, 2.0, 3.0]), + "vec_angle_col": Col([0.0, 90.0, 180.0], "deg"), + "vec_string_a": Col(["NGC", "IC", "M"]), + "vec_string_b": Col(["123", "456", "789"]), } @@ -116,6 +120,40 @@ def evaluate_expr(source: str, columns: dict[str, Col]) -> object: columns=_COLUMNS, error=True, ), + EvalCase( + name="vector_column_with_unit", + expression='col("vec_angle_col")', + columns=_COLUMNS, + result_val=[0.0, 90.0, 180.0], + result_unit=u.deg, + ), + EvalCase( + name="vector_arithmetic", + expression='3 * 10 ** col("vec_col") * arcsec', + columns=_COLUMNS, + result_val=[30.0, 300.0, 3000.0], + result_unit=u.arcsec, + ), + EvalCase( + name="vector_trig", + expression='sin(col("vec_angle_col"))', + columns=_COLUMNS, + result_val=[0.0, 1.0, 0.0], + result_unit=u.dimensionless_unscaled, + ), + EvalCase( + name="scalar_vector_broadcast", + expression='col("float_col") + col("vec_col")', + columns=_COLUMNS, + result_val=[2.5, 3.5, 4.5], + result_unit=u.dimensionless_unscaled, + ), + EvalCase( + name="vector_string_concat", + expression='col("vec_string_a") + " " + col("vec_string_b")', + columns=_COLUMNS, + result_val=["NGC 123", "IC 456", "M 789"], + ), ] @@ -129,9 +167,21 @@ def test_evaluate(case: EvalCase) -> None: result = evaluate_expr(case.expression, case.columns) if case.result_unit is None: - assert result == case.result_val + if isinstance(case.result_val, list): + assert isinstance(result, np.ndarray) + np.testing.assert_array_equal(result, case.result_val) + else: + assert result == case.result_val return assert isinstance(result, u.Quantity) assert result.unit == case.result_unit - np.testing.assert_almost_equal(result.value, case.result_val, decimal=4) + if isinstance(case.result_val, list): + np.testing.assert_allclose( + np.asarray(result.value), + np.asarray(case.result_val), + rtol=1e-4, + atol=1e-10, + ) + else: + np.testing.assert_almost_equal(result.value, case.result_val, decimal=4) diff --git a/uploader/app/lib/formula/values.py b/uploader/app/lib/formula/values.py index 7d570b0..102a4ba 100644 --- a/uploader/app/lib/formula/values.py +++ b/uploader/app/lib/formula/values.py @@ -1,19 +1,26 @@ +from collections.abc import Sequence + import astropy.units as u +import numpy as np from astropy.units.function.core import FunctionUnitBase -type Value = u.Quantity | str +type Value = u.Quantity | str | np.ndarray def _is_logarithmic_column_unit(unit: u.Unit) -> bool: return unit == u.mag or unit == u.dex or isinstance(unit, FunctionUnitBase) -def column_quantity(value: float | str, unit: str) -> Value: +def column_quantity(value: float | str | Sequence[float] | Sequence[str], unit: str) -> Value: if isinstance(value, str): return value + if isinstance(value, Sequence) and not isinstance(value, (str, bytes)): + if all(isinstance(x, str) for x in value): + return np.asarray(value) + numeric = np.asarray(value, dtype=float) if not unit: - return float(value) * u.dimensionless_unscaled + return numeric * u.dimensionless_unscaled parsed = u.Unit(unit) if _is_logarithmic_column_unit(parsed): - return float(value) * u.dimensionless_unscaled - return float(value) * parsed + return numeric * u.dimensionless_unscaled + return numeric * parsed From a9e4ccc51612e0dbe74a6056b91899bb77671036 Mon Sep 17 00:00:00 2001 From: kraysent Date: Mon, 6 Jul 2026 22:55:27 +0100 Subject: [PATCH 08/10] use formula in designations --- tests/test_designation_upload.py | 124 ++++++++++++++++++ .../app/structured/designations/upload.py | 87 +++++++++++- uploader/forms/structured_designation.py | 11 +- uploader/task_registry.py | 6 +- 4 files changed, 216 insertions(+), 12 deletions(-) create mode 100644 tests/test_designation_upload.py diff --git a/tests/test_designation_upload.py b/tests/test_designation_upload.py new file mode 100644 index 0000000..872b279 --- /dev/null +++ b/tests/test_designation_upload.py @@ -0,0 +1,124 @@ +from unittest.mock import Mock, patch + +import pytest + +from uploader.app.structured.designations.upload import upload_designations +from uploader.clients.gen.client import adminapi + + +def _mock_storage(total: int = 1) -> Mock: + storage = Mock() + storage.query.return_value = [{"cnt": total}] + return storage + + +def _mock_client() -> Mock: + return Mock(spec=adminapi.AuthenticatedClient) + + +@patch("uploader.app.structured.designations.upload.rawdata_batches") +@patch("uploader.app.structured.designations.upload._fetch_column_units") +def test_simple_column_expression( + mock_fetch_column_units: Mock, + mock_rawdata_batches: Mock, +) -> None: + mock_fetch_column_units.return_value = ({"name"}, {"name": ""}) + mock_rawdata_batches.return_value = iter( + [[{"hyperleda_internal_id": "1", "name": "NGC 123"}]], + ) + + total = upload_designations( + _mock_storage(), + "test_table", + "name", + 100, + _mock_client(), + report_func=lambda _: None, + ) + + assert total == 1 + mock_rawdata_batches.assert_called_once() + assert mock_rawdata_batches.call_args.args[2] == ["name"] + + +@patch("uploader.app.structured.designations.upload.rawdata_batches") +@patch("uploader.app.structured.designations.upload._fetch_column_units") +def test_composed_string_expression( + mock_fetch_column_units: Mock, + mock_rawdata_batches: Mock, +) -> None: + mock_fetch_column_units.return_value = ({"prefix", "number"}, {"prefix": "", "number": ""}) + mock_rawdata_batches.return_value = iter( + [[{"hyperleda_internal_id": "1", "prefix": "NGC", "number": "123"}]], + ) + + total = upload_designations( + _mock_storage(), + "test_table", + 'prefix + " " + number', + 100, + _mock_client(), + report_func=lambda _: None, + ) + + assert total == 1 + + +@patch("uploader.app.structured.designations.upload._fetch_column_units") +def test_missing_referenced_columns(mock_fetch_column_units: Mock) -> None: + mock_fetch_column_units.return_value = ({"other"}, {}) + + with pytest.raises(RuntimeError, match="has no column\\(s\\): \\['name'\\]"): + upload_designations( + _mock_storage(), + "test_table", + "name", + 100, + _mock_client(), + report_func=lambda _: None, + ) + + +@patch("uploader.app.structured.designations.upload.rawdata_batches") +@patch("uploader.app.structured.designations.upload._fetch_column_units") +def test_null_referenced_values_counted_as_unmatched( + mock_fetch_column_units: Mock, + mock_rawdata_batches: Mock, +) -> None: + mock_fetch_column_units.return_value = ({"name"}, {"name": ""}) + mock_rawdata_batches.return_value = iter( + [[{"hyperleda_internal_id": "1", "name": None}]], + ) + + total = upload_designations( + _mock_storage(), + "test_table", + "name", + 100, + _mock_client(), + report_func=lambda _: None, + ) + + assert total == 1 + + +@patch("uploader.app.structured.designations.upload.rawdata_batches") +@patch("uploader.app.structured.designations.upload._fetch_column_units") +def test_expression_evaluation_error_becomes_runtime_error( + mock_fetch_column_units: Mock, + mock_rawdata_batches: Mock, +) -> None: + mock_fetch_column_units.return_value = ({"text_col", "num_col"}, {"text_col": "", "num_col": ""}) + mock_rawdata_batches.return_value = iter( + [[{"hyperleda_internal_id": "1", "text_col": "NGC 123", "num_col": 1.5}]], + ) + + with pytest.raises(RuntimeError, match="failed to evaluate expression for row 1"): + upload_designations( + _mock_storage(), + "test_table", + "text_col + num_col", + 100, + _mock_client(), + report_func=lambda _: None, + ) diff --git a/uploader/app/structured/designations/upload.py b/uploader/app/structured/designations/upload.py index 34a55a4..f76cd0c 100644 --- a/uploader/app/structured/designations/upload.py +++ b/uploader/app/structured/designations/upload.py @@ -1,18 +1,29 @@ from collections.abc import Callable +from typing import Any +import astropy.units as u import matplotlib.pyplot as plt +import numpy as np from psycopg import sql import uploader.app.action_description as action_description import uploader.app.report as report from uploader.app import log from uploader.app.display import format_table +from uploader.app.lib.formula import ( + Expression, + ExpressionEvaluationError, + Value, + column_quantity, + evaluate, + parse, +) from uploader.app.lib.rawdata import rawdata_batches from uploader.app.storage import PgStorage from uploader.app.structured.designations.rules import RULES, match from uploader.app.upload import handle_call from uploader.clients.gen.client import adminapi -from uploader.clients.gen.client.adminapi.api.default import save_structured_data +from uploader.clients.gen.client.adminapi.api.default import get_table, save_structured_data from uploader.clients.gen.client.adminapi.models.save_structured_data_request import ( SaveStructuredDataRequest, ) @@ -114,10 +125,67 @@ def pct(n: int) -> float: report_func(report.DoneEvent(message=summary)) +def _fetch_column_units( + client: adminapi.AuthenticatedClient, + table_name: str, +) -> tuple[set[str], dict[str, str]]: + resp = handle_call(get_table.sync_detailed(client=client, table_name=table_name)) + column_names: set[str] = set() + column_units: dict[str, str] = {} + for col in resp.data.column_info: + column_names.add(col.name) + if isinstance(col.unit, str): + column_units[col.name] = col.unit + return column_names, column_units + + +def _validate_columns( + table_name: str, + needed_cols: set[str], + column_names: set[str], +) -> None: + missing = sorted(col for col in needed_cols if col not in column_names) + if missing: + raise RuntimeError(f"Table {table_name} has no column(s): {missing}") + + +def _build_column_values( + row: dict[str, Any], + referenced_columns: frozenset[str], + column_units: dict[str, str], +) -> dict[str, Value]: + return {col: column_quantity(row[col], column_units.get(col, "")) for col in referenced_columns} + + +def _designation_string(value: Value) -> str: + if isinstance(value, str): + return value.strip() + if isinstance(value, u.Quantity): + scalar = value.value + if isinstance(scalar, np.ndarray) and scalar.shape != (): + raise RuntimeError("designation expression must evaluate to a scalar value per row") + return str(scalar).strip() + raise RuntimeError("designation expression must evaluate to a scalar value per row") + + +def _evaluate_designation( + parsed: Expression, + row: dict[str, Any], + column_units: dict[str, str], +) -> str: + columns = _build_column_values(row, parsed.referenced_columns, column_units) + try: + return _designation_string(evaluate(parsed, columns)) + except ExpressionEvaluationError as e: + raise RuntimeError( + f"failed to evaluate expression for row {row['hyperleda_internal_id']}: {e}", + ) from e + + def upload_designations( storage: PgStorage, table_name: str, - column_name: str, + expression: str, batch_size: int, client: adminapi.AuthenticatedClient, *, @@ -125,6 +193,11 @@ def upload_designations( print_unmatched: bool = False, report_func: Callable[[report.Event], None], ) -> int: + parsed = parse(expression) + needed_cols = set(parsed.referenced_columns) + column_names, column_units = _fetch_column_units(client, table_name) + _validate_columns(table_name, needed_cols, column_names) + rule_counts: dict[str, int] = {r.name: 0 for r in RULES} unmatched = 0 total_count = 0 @@ -136,17 +209,19 @@ def upload_designations( processed_rows = 0 - for rows in rawdata_batches(storage, table_name, [column_name], batch_size): + for rows in rawdata_batches(storage, table_name, sorted(needed_cols), batch_size): batch_ids: list[str] = [] batch_names: list[list[str]] = [] for row in rows: internal_id = row["hyperleda_internal_id"] - name_val = row[column_name] - if name_val is None or (isinstance(name_val, str) and not name_val.strip()): + if any(row[col] is None for col in needed_cols): + unmatched += 1 + continue + name_str = _evaluate_designation(parsed, row, column_units) + if not name_str: unmatched += 1 continue - name_str = str(name_val).strip() match_result = match(name_str) if match_result is not None: transformed, rule_name = match_result diff --git a/uploader/forms/structured_designation.py b/uploader/forms/structured_designation.py index 632ce13..f3f134b 100644 --- a/uploader/forms/structured_designation.py +++ b/uploader/forms/structured_designation.py @@ -25,10 +25,13 @@ class StructuredDesignationAdvancedSettings(BaseModel): class StructuredDesignationForm(BaseModel): table_name: str = Field(..., title="Name of the table") - column_name: str = Field( + expression: str = Field( ..., - title="Object name column", - description="Name of the column that represents object designation in the table.", + title="Designation expression", + description=( + "Expression yielding the object designation per row. " + 'Examples: designation, col("weird name"), prefix + " " + number.' + ), ) write: bool = Field( default=False, @@ -61,7 +64,7 @@ def handle_structured_designation( run_upload_designations( storage, f.table_name.strip(), - f.column_name.strip(), + f.expression.strip(), advanced.batch_size, client, write=f.write, diff --git a/uploader/task_registry.py b/uploader/task_registry.py index fc6670a..c4fdba7 100644 --- a/uploader/task_registry.py +++ b/uploader/task_registry.py @@ -1,4 +1,4 @@ -from uploader.app.lib.expression import expression_syntax_help +from uploader.app.lib.formula import expression_syntax_help from uploader.forms.authenticate import AuthenticateForm, handle_authenticate from uploader.forms.crossmatch_layered import CrossmatchLayeredForm, handle_crossmatch_layered from uploader.forms.structured_designation import ( @@ -80,7 +80,9 @@ def register_all_tasks() -> None: TaskDefinition( id="upload-structured-designation", title="Designations", - description="Convert designations to common format and upload them to the database.", + description=( + f"Convert designations to common format and upload them to the database.\n\n{expression_syntax_help()}" + ), form_model=StructuredDesignationForm, handler=handle_structured_designation, group="Catalogs", From 43988019e406db01cf547f3548ac4c3a52c38f1e Mon Sep 17 00:00:00 2001 From: kraysent Date: Mon, 6 Jul 2026 23:06:44 +0100 Subject: [PATCH 09/10] fix string and add output file --- tests/test_designation_upload.py | 27 ++++ tests/test_formula_evaluate.py | 12 ++ uploader/app/lib/formula/namespace.py | 15 +- .../app/structured/designations/upload.py | 149 +++++++++++------- uploader/forms/structured_designation.py | 6 + 5 files changed, 147 insertions(+), 62 deletions(-) diff --git a/tests/test_designation_upload.py b/tests/test_designation_upload.py index 872b279..659677b 100644 --- a/tests/test_designation_upload.py +++ b/tests/test_designation_upload.py @@ -1,3 +1,4 @@ +from pathlib import Path from unittest.mock import Mock, patch import pytest @@ -122,3 +123,29 @@ def test_expression_evaluation_error_becomes_runtime_error( _mock_client(), report_func=lambda _: None, ) + + +@patch("uploader.app.structured.designations.upload.rawdata_batches") +@patch("uploader.app.structured.designations.upload._fetch_column_units") +def test_output_file_writes_id_designation_pairs( + mock_fetch_column_units: Mock, + mock_rawdata_batches: Mock, + tmp_path: Path, +) -> None: + mock_fetch_column_units.return_value = ({"name"}, {"name": ""}) + mock_rawdata_batches.return_value = iter( + [[{"hyperleda_internal_id": "id-1", "name": "NGC 123"}]], + ) + output_file = tmp_path / "designations.csv" + + upload_designations( + _mock_storage(), + "test_table", + "name", + 100, + _mock_client(), + output_file=str(output_file), + report_func=lambda _: None, + ) + + assert output_file.read_text().splitlines() == ["id,designation", "id-1,NGC 123"] diff --git a/tests/test_formula_evaluate.py b/tests/test_formula_evaluate.py index 286678f..c9d3f9e 100644 --- a/tests/test_formula_evaluate.py +++ b/tests/test_formula_evaluate.py @@ -79,6 +79,18 @@ def evaluate_expr(source: str, columns: dict[str, Col]) -> object: columns=_COLUMNS, result_val="M NGC 123", ), + EvalCase( + name="string_concat_with_str", + expression='col("string_col_2") + " " + str(col("float_col"))', + columns=_COLUMNS, + result_val="M 1.5", + ), + EvalCase( + name="str_on_string_column", + expression='str(col("string_col_1"))', + columns=_COLUMNS, + result_val="NGC 123", + ), EvalCase( name="function_unit", expression='col("float_col_dex")', diff --git a/uploader/app/lib/formula/namespace.py b/uploader/app/lib/formula/namespace.py index 3d0a70f..c504fe4 100644 --- a/uploader/app/lib/formula/namespace.py +++ b/uploader/app/lib/formula/namespace.py @@ -18,9 +18,22 @@ "mag": 1 * u.mag, } + +def _formula_str(value: Value) -> str | np.ndarray: + if isinstance(value, str): + return value + if isinstance(value, u.Quantity): + scalar = value.value + if isinstance(scalar, np.ndarray): + return np.asarray(scalar, dtype=str) + return str(scalar) + return np.asarray(value, dtype=str) + + FUNCTIONS: dict[str, object] = { "sin": np.sin, "cos": np.cos, + "str": _formula_str, } @@ -42,7 +55,7 @@ def expression_syntax_help() -> str: '(e.g. col("a"), e_logd25).\n' "Bare identifiers that match predefined constants use those constants.\n" "Operators: + - * / ** %.\n" - "Functions: sin(x), cos(x) (argument must be an angle).\n" + "Functions: sin(x), cos(x) (argument must be an angle), str(x).\n" "Numbers are dimensionless.\n" "String literals and + concatenation are supported.\n" 'Modulo divisors must carry units (e.g. col("pa") % (180 * deg)).\n' diff --git a/uploader/app/structured/designations/upload.py b/uploader/app/structured/designations/upload.py index f76cd0c..0c99ee7 100644 --- a/uploader/app/structured/designations/upload.py +++ b/uploader/app/structured/designations/upload.py @@ -1,3 +1,5 @@ +import csv +import pathlib from collections.abc import Callable from typing import Any @@ -182,6 +184,22 @@ def _evaluate_designation( ) from e +class _DesignationOutputWriter: + def __init__(self, path: str) -> None: + output_path = pathlib.Path(path) + output_path.parent.mkdir(parents=True, exist_ok=True) + self._file = output_path.open("w", newline="") + self._writer = csv.writer(self._file) + self._writer.writerow(["id", "designation"]) + + def write_batch(self, ids: list[str], designations: list[list[str]]) -> None: + for record_id, designation in zip(ids, designations, strict=True): + self._writer.writerow([record_id, designation[0]]) + + def close(self) -> None: + self._file.close() + + def upload_designations( storage: PgStorage, table_name: str, @@ -191,6 +209,7 @@ def upload_designations( *, write: bool = False, print_unmatched: bool = False, + output_file: str = "", report_func: Callable[[report.Event], None], ) -> int: parsed = parse(expression) @@ -208,71 +227,79 @@ def upload_designations( total_count = int(cnt[0]["cnt"]) if cnt else 0 processed_rows = 0 + output_writer = _DesignationOutputWriter(output_file) if output_file else None - for rows in rawdata_batches(storage, table_name, sorted(needed_cols), batch_size): - batch_ids: list[str] = [] - batch_names: list[list[str]] = [] - - for row in rows: - internal_id = row["hyperleda_internal_id"] - if any(row[col] is None for col in needed_cols): - unmatched += 1 - continue - name_str = _evaluate_designation(parsed, row, column_units) - if not name_str: - unmatched += 1 - continue - match_result = match(name_str) - if match_result is not None: - transformed, rule_name = match_result - rule_counts[rule_name] += 1 - else: - unmatched += 1 - transformed = name_str - if print_unmatched: - report_func(report.LogEvent(message=name_str)) - batch_ids.append(internal_id) - batch_names.append([transformed]) - - if write and batch_ids: - handle_call( - save_structured_data.sync_detailed( - client=client, - body=action_description.apply( - SaveStructuredDataRequest( - catalog="designation", - columns=["design"], - ids=batch_ids, - data=batch_names, + try: + for rows in rawdata_batches(storage, table_name, sorted(needed_cols), batch_size): + batch_ids: list[str] = [] + batch_names: list[list[str]] = [] + + for row in rows: + internal_id = row["hyperleda_internal_id"] + if any(row[col] is None for col in needed_cols): + unmatched += 1 + continue + name_str = _evaluate_designation(parsed, row, column_units) + if not name_str: + unmatched += 1 + continue + match_result = match(name_str) + if match_result is not None: + transformed, rule_name = match_result + rule_counts[rule_name] += 1 + else: + unmatched += 1 + transformed = name_str + if print_unmatched: + report_func(report.LogEvent(message=name_str)) + batch_ids.append(internal_id) + batch_names.append([transformed]) + + if write and batch_ids: + handle_call( + save_structured_data.sync_detailed( + client=client, + body=action_description.apply( + SaveStructuredDataRequest( + catalog="designation", + columns=["design"], + ids=batch_ids, + data=batch_names, + ), ), - ), + ) ) - ) - processed_rows += len(rows) - total_so_far = sum(rule_counts.values()) + unmatched - - def total_pct(n: int, t: int = total_so_far) -> float: - return (100.0 * n / t) if t else 0.0 - - log.logger.info( - "processed batch", - total=total_so_far, - matched=sum(rule_counts.values()), - matched_pct=round(total_pct(sum(rule_counts.values())), 1), - unmatched=unmatched, - unmatched_pct=round(total_pct(unmatched), 1), - ) - progress_pct = int(100 * processed_rows / total_count) if total_count else 0 - _report_batch_progress( - report_func, - rows_read=len(rows), - total_so_far=total_so_far, - matched=sum(rule_counts.values()), - unmatched=unmatched, - progress_pct=progress_pct, - rule_counts=rule_counts, - ) + if output_writer is not None and batch_ids: + output_writer.write_batch(batch_ids, batch_names) + + processed_rows += len(rows) + total_so_far = sum(rule_counts.values()) + unmatched + + def total_pct(n: int, t: int = total_so_far) -> float: + return (100.0 * n / t) if t else 0.0 + + log.logger.info( + "processed batch", + total=total_so_far, + matched=sum(rule_counts.values()), + matched_pct=round(total_pct(sum(rule_counts.values())), 1), + unmatched=unmatched, + unmatched_pct=round(total_pct(unmatched), 1), + ) + progress_pct = int(100 * processed_rows / total_count) if total_count else 0 + _report_batch_progress( + report_func, + rows_read=len(rows), + total_so_far=total_so_far, + matched=sum(rule_counts.values()), + unmatched=unmatched, + progress_pct=progress_pct, + rule_counts=rule_counts, + ) + finally: + if output_writer is not None: + output_writer.close() total = sum(rule_counts.values()) + unmatched _report_rule_distribution(report_func, rule_counts, unmatched, total) diff --git a/uploader/forms/structured_designation.py b/uploader/forms/structured_designation.py index f3f134b..1d34d0e 100644 --- a/uploader/forms/structured_designation.py +++ b/uploader/forms/structured_designation.py @@ -21,6 +21,11 @@ class StructuredDesignationAdvancedSettings(BaseModel): title="Log unmatched names", description="Append each unmatched name to the log stream.", ) + output_file: str = Field( + default="", + title="Output file", + description="If set, write id and designation pairs to this file path.", + ) class StructuredDesignationForm(BaseModel): @@ -69,5 +74,6 @@ def handle_structured_designation( client, write=f.write, print_unmatched=advanced.print_unmatched, + output_file=advanced.output_file.strip(), report_func=report_func, ) From e5796dbc589ea112ee60fb01a9db3d44567ad61e Mon Sep 17 00:00:00 2001 From: kraysent Date: Mon, 6 Jul 2026 23:12:52 +0100 Subject: [PATCH 10/10] fix int str function --- tests/test_formula_evaluate.py | 12 ++++++++++++ uploader/app/lib/formula/namespace.py | 13 ++++++++++--- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/tests/test_formula_evaluate.py b/tests/test_formula_evaluate.py index c9d3f9e..8c438ba 100644 --- a/tests/test_formula_evaluate.py +++ b/tests/test_formula_evaluate.py @@ -91,6 +91,18 @@ def evaluate_expr(source: str, columns: dict[str, Col]) -> object: columns=_COLUMNS, result_val="NGC 123", ), + EvalCase( + name="str_integer", + expression='str(col("num"))', + columns={"num": Col(495444.0)}, + result_val="495444", + ), + EvalCase( + name="str_keeps_fraction", + expression='str(col("float_col"))', + columns=_COLUMNS, + result_val="1.5", + ), EvalCase( name="function_unit", expression='col("float_col_dex")', diff --git a/uploader/app/lib/formula/namespace.py b/uploader/app/lib/formula/namespace.py index c504fe4..cc73c2a 100644 --- a/uploader/app/lib/formula/namespace.py +++ b/uploader/app/lib/formula/namespace.py @@ -19,15 +19,22 @@ } +def _scalar_to_str(value: float | int | np.number) -> str: + numeric = float(value) + if numeric.is_integer(): + return str(int(numeric)) + return str(numeric) + + def _formula_str(value: Value) -> str | np.ndarray: if isinstance(value, str): return value if isinstance(value, u.Quantity): scalar = value.value if isinstance(scalar, np.ndarray): - return np.asarray(scalar, dtype=str) - return str(scalar) - return np.asarray(value, dtype=str) + return np.asarray([_scalar_to_str(x) for x in scalar]) + return _scalar_to_str(scalar) + return np.asarray([_scalar_to_str(x) for x in value]) FUNCTIONS: dict[str, object] = {