From d9f91d2b96ef41e1ae8a0db38d7f45158b3c34d0 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 10 Jul 2026 21:33:41 +0200 Subject: [PATCH 1/2] Add FLEXMEASURES_LP_SOLVER_OPTIONS to configure solver options Solver options for the scheduling solver could previously only be set in code (a hardcoded HiGHS profile in device_scheduler). This adds a FLEXMEASURES_LP_SOLVER_OPTIONS config dict, merged over that profile so operators can tune the solver (e.g. {"mip_rel_gap": "1e-4"}) without patching FlexMeasures. When the solver is HiGHS, the options are validated against the installed HiGHS build: an unknown option name, an invalid value, or a feature the build lacks raises a clear ValueError. This matters because Pyomo's appsi_highs interface applies options without checking HiGHS' return status, so a mistake would otherwise be silently ignored (e.g. solver="hipo" is unavailable in the pip-installed highspy, yet appsi_highs solves on as if nothing were wrong). Setting threads/parallel logs a warning about HiGHS' process-global thread scheduler. Found while investigating SeitaBV/ems#172. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01LeqGFrHfGrBHAJyjdAyr3y --- documentation/changelog.rst | 1 + documentation/configuration.rst | 14 +++++ .../models/planning/linear_optimization.py | 45 ++++++++++++++ .../planning/tests/test_solver_options.py | 62 +++++++++++++++++++ flexmeasures/utils/config_defaults.py | 1 + 5 files changed, 123 insertions(+) create mode 100644 flexmeasures/data/models/planning/tests/test_solver_options.py diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 7f020f3647..3467051c43 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -21,6 +21,7 @@ New features * The flex-context can now define multiple commodities, each specifying their own prices and grid capacities [see `PR #1946 `_, `PR #2172 `_, `PR #2235 `_ and `PR #2271 `_] * CLI support for adding/editing account attributes [see `PR #2242 `_] * Extended ``GET /api/v3_0/jobs/`` with a ``result`` field containing ``unresolved`` and ``resolved`` soft state-of-charge constraint analysis (``soc-minima``/``soc-maxima`` violations or satisfied constraints, keyed by asset ID) for scheduling jobs; both arrays are empty when no SoC constraints were defined [see `PR #2072 `_] +* New ``FLEXMEASURES_LP_SOLVER_OPTIONS`` config setting to pass solver options to the scheduling solver, validated against the installed HiGHS build so that unknown or unsupported options raise instead of being silently ignored Infrastructure / Support ---------------------- diff --git a/documentation/configuration.rst b/documentation/configuration.rst index 1c79235e6a..854665446b 100644 --- a/documentation/configuration.rst +++ b/documentation/configuration.rst @@ -62,6 +62,20 @@ Note that you need to install the solver, read more at :ref:`installing-a-solver Default: ``"appsi_highs"`` +FLEXMEASURES_LP_SOLVER_OPTIONS +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Solver options passed to the scheduling solver, overriding the defaults FlexMeasures sets itself. Use this to tune the solver without patching code, for example to trade optimality for speed:: + + FLEXMEASURES_LP_SOLVER_OPTIONS = {"mip_rel_gap": "1e-4"} + +When the solver is HiGHS, FlexMeasures validates these against the installed HiGHS build and raises on an unknown option name, an invalid value, or a feature the build lacks. This matters because Pyomo's ``appsi_highs`` interface otherwise applies solver options without checking whether HiGHS accepted them, so a typo would be silently ignored. + +.. note:: HiGHS initializes its thread scheduler once per process. Setting ``threads`` or ``parallel`` therefore only affects the first solve in a worker process; later solves fail with ``global scheduler has already been initialized`` and return no schedule. FlexMeasures logs a warning if you set either. + +Default: ``{}`` + + FLEXMEASURES_HOSTS_AND_AUTH_START ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/flexmeasures/data/models/planning/linear_optimization.py b/flexmeasures/data/models/planning/linear_optimization.py index 12b33fd70c..6ad3bfc81c 100644 --- a/flexmeasures/data/models/planning/linear_optimization.py +++ b/flexmeasures/data/models/planning/linear_optimization.py @@ -33,6 +33,45 @@ infinity = float("inf") +def validate_highs_options(options: dict) -> None: + """Raise if HiGHS would refuse any of these options. + + Pyomo's appsi_highs interface applies solver options without checking HiGHS' + return status, so an unknown name, an invalid value, or a feature missing from + the installed HiGHS build is otherwise ignored without a word. That silently + turns a mis-typed option into a no-op, and a benchmark of it into a false + negative. Probing a throwaway Highs instance surfaces the rejection instead. + """ + try: + import highspy + except ImportError: + # Solver named "*highs*" but highspy absent: let the solver interface complain. + return + + probe = highspy.Highs() + probe.setOptionValue("output_flag", False) + rejected = [ + f"{name}={value!r}" + for name, value in options.items() + if probe.setOptionValue(name, value) != highspy.HighsStatus.kOk + ] + if rejected: + raise ValueError( + f"HiGHS rejected these FLEXMEASURES_LP_SOLVER_OPTIONS: {', '.join(rejected)}." + " The option name may be unknown, the value invalid, or the feature absent" + " from this HiGHS build. For example, the HiPO solver (solver='hipo') needs" + " a HiGHS built against BLAS and METIS, which the pip-installed highspy is not." + ) + + if "threads" in options or "parallel" in options: + current_app.logger.warning( + "FLEXMEASURES_LP_SOLVER_OPTIONS sets 'threads' and/or 'parallel'. HiGHS" + " initializes its thread scheduler once per process, so inside a long-lived" + " worker only the first solve honours these; later solves fail with 'global" + " scheduler has already been initialized' and yield no schedule." + ) + + def device_scheduler( # noqa C901 device_constraints: list[pd.DataFrame], ems_constraints: pd.DataFrame | list[pd.DataFrame], @@ -819,6 +858,12 @@ def cost_function(m): if current_app.config["LOGGING_LEVEL"] == "INFO": profile["output_flag"] = "false" + # Apply operator-configured options last, so they override the defaults above. + configured_options = current_app.config.get("FLEXMEASURES_LP_SOLVER_OPTIONS") or {} + if configured_options and "highs" in solver_name.lower(): + validate_highs_options(configured_options) + profile.update(configured_options) + for option_name, option_value in profile.items(): solver.options[option_name] = option_value diff --git a/flexmeasures/data/models/planning/tests/test_solver_options.py b/flexmeasures/data/models/planning/tests/test_solver_options.py new file mode 100644 index 0000000000..85738ccf1b --- /dev/null +++ b/flexmeasures/data/models/planning/tests/test_solver_options.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +import pytest + +from flexmeasures.data.models.planning.linear_optimization import validate_highs_options + +highspy = pytest.importorskip("highspy") + + +def highs_accepts(name, value) -> bool: + probe = highspy.Highs() + probe.setOptionValue("output_flag", False) + return probe.setOptionValue(name, value) == highspy.HighsStatus.kOk + + +def test_validate_highs_options_accepts_the_defaults_we_ship(): + """The profile device_scheduler applies itself must stay acceptable to HiGHS.""" + validate_highs_options( + { + "mip_rel_gap": "0", + "mip_abs_gap": "0", + "primal_feasibility_tolerance": "1e-9", + "dual_feasibility_tolerance": "1e-9", + "mip_feasibility_tolerance": "1e-9", + "output_flag": "false", + } + ) + + +def test_validate_highs_options_rejects_unknown_option_name(): + with pytest.raises(ValueError, match="no_such_option"): + validate_highs_options({"no_such_option": "1"}) + + +def test_validate_highs_options_rejects_invalid_option_value(): + with pytest.raises(ValueError, match="solver='banana'"): + validate_highs_options({"solver": "banana"}) + + +def test_validate_highs_options_reports_every_rejected_option(): + with pytest.raises(ValueError) as exc: + validate_highs_options({"no_such_option": "1", "solver": "banana"}) + assert "no_such_option" in str(exc.value) + assert "solver='banana'" in str(exc.value) + + +def test_validate_highs_options_rejects_solver_missing_from_this_build(): + """HiPO needs a HiGHS built against BLAS and METIS; the pip-installed one is not. + + Pyomo's appsi_highs would swallow this, so we must not. + """ + if highs_accepts("solver", "hipo"): + pytest.skip("this HiGHS build provides the HiPO solver") + with pytest.raises(ValueError, match="hipo"): + validate_highs_options({"solver": "hipo"}) + + +def test_validate_highs_options_warns_about_the_global_thread_scheduler(app, caplog): + """HiGHS initializes its thread scheduler once per process, so `threads` is a trap.""" + with app.app_context(): + validate_highs_options({"threads": 2}) + assert "thread scheduler" in caplog.text diff --git a/flexmeasures/utils/config_defaults.py b/flexmeasures/utils/config_defaults.py index e4d869054b..d792c051cf 100644 --- a/flexmeasures/utils/config_defaults.py +++ b/flexmeasures/utils/config_defaults.py @@ -154,6 +154,7 @@ class Config(object): "EVSE": ["one-way_evse", "two-way_evse"], } # how to group assets by asset types FLEXMEASURES_LP_SOLVER: str = "appsi_highs" + FLEXMEASURES_LP_SOLVER_OPTIONS: dict[str, str | int | float] = {} FLEXMEASURES_JOB_TTL: timedelta = timedelta(days=1) FLEXMEASURES_PLANNING_HORIZON: timedelta = timedelta(days=2) FLEXMEASURES_MAX_PLANNING_HORIZON: timedelta | int | None = ( From 32d3f10dd65509740dc745f0449fefbec0e162b6 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 10 Jul 2026 21:42:40 +0200 Subject: [PATCH 2/2] Reference PR #2283 in the changelog entry Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01LeqGFrHfGrBHAJyjdAyr3y --- documentation/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 3467051c43..c745f299ae 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -21,7 +21,7 @@ New features * The flex-context can now define multiple commodities, each specifying their own prices and grid capacities [see `PR #1946 `_, `PR #2172 `_, `PR #2235 `_ and `PR #2271 `_] * CLI support for adding/editing account attributes [see `PR #2242 `_] * Extended ``GET /api/v3_0/jobs/`` with a ``result`` field containing ``unresolved`` and ``resolved`` soft state-of-charge constraint analysis (``soc-minima``/``soc-maxima`` violations or satisfied constraints, keyed by asset ID) for scheduling jobs; both arrays are empty when no SoC constraints were defined [see `PR #2072 `_] -* New ``FLEXMEASURES_LP_SOLVER_OPTIONS`` config setting to pass solver options to the scheduling solver, validated against the installed HiGHS build so that unknown or unsupported options raise instead of being silently ignored +* New ``FLEXMEASURES_LP_SOLVER_OPTIONS`` config setting to pass solver options to the scheduling solver, validated against the installed HiGHS build so that unknown or unsupported options raise instead of being silently ignored [see `PR #2283 `_] Infrastructure / Support ----------------------