Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
82 changes: 80 additions & 2 deletions examples/tutorials/model_explainability.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@
"import warnings\n",
"from typing import cast\n",
"\n",
"from openstef_meta.presets import EnsembleForecastingWorkflowConfig, create_ensemble_forecasting_workflow\n",
"\n",
"warnings.filterwarnings(\"ignore\")\n",
"\n",
"from openstef_core.testing import configure_notebook_display, setup_notebook_logging\n",
Expand Down Expand Up @@ -174,7 +176,7 @@
},
"outputs": [],
"source": [
"from openstef_models.explainability import ExplainableForecaster\n",
"from openstef_models.explainability import ExplainableForecaster, ModelContributionsPlotter\n",
"from openstef_models.models.forecasting_model import ForecastingModel\n",
"\n",
"forecaster = cast(ForecastingModel, workflow.model).forecaster\n",
Expand Down Expand Up @@ -317,6 +319,70 @@
"fig.show()"
]
},
{
"cell_type": "markdown",
"id": "5118a84f",
"metadata": {},
"source": [
"## Ensemble model weights\n",
"\n",
"For each forecast quantile, a classifier estimates which base model is likely\n",
"to perform best. Its predicted probabilities become nonnegative model weights\n",
"that sum to one at each timestep. The top panel shows the ensemble forecast\n",
"for the selected quantile; the stacked area below shows each model's weight.\n",
"The plot defaults to P50 when no quantile is specified."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "621e0f94",
"metadata": {
"tags": [
"hide-input"
]
},
"outputs": [],
"source": [
"ensemble_workflow = create_ensemble_forecasting_workflow(\n",
" config=EnsembleForecastingWorkflowConfig(\n",
" model_id=\"example_gblinear_lgbm_ensemble\",\n",
" ensemble_type=\"learned_weights\",\n",
" base_models=[\"gblinear\", \"lgbm\"],\n",
" combiner_model=\"lgbm\",\n",
" quantiles=[Q(0.1), Q(0.5), Q(0.9)],\n",
" horizons=[LeadTime.from_string(\"PT36H\")],\n",
" mlflow_storage=None,\n",
" temperature_column=\"temperature_2m\",\n",
" relative_humidity_column=\"relative_humidity_2m\",\n",
" wind_speed_column=\"wind_speed_10m\",\n",
" radiation_column=\"shortwave_radiation\",\n",
" pressure_column=\"surface_pressure\",\n",
" )\n",
")\n",
"ensemble_result = ensemble_workflow.fit(train_dataset)\n",
"ensemble_forecast, model_weights = ensemble_workflow.model.predict_with_contributions(\n",
" predict_dataset, forecast_start=train_end\n",
")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "df7a1d98",
"metadata": {},
"outputs": [],
"source": [
"fig = ModelContributionsPlotter.plot_stacked_area(\n",
" model_weights,\n",
" forecast=ensemble_forecast,\n",
" quantile=Q(0.1),\n",
")\n",
"fig.update_layout(height=650)\n",
"fig.show()\n",
"# fig.write_html(\"ensemble_model_weights.html\", include_plotlyjs=True)"
]
},
{
"cell_type": "markdown",
"id": "dc95f759",
Expand All @@ -337,9 +403,21 @@
"notebook_metadata_filter": "-jupytext.text_representation.jupytext_version"
},
"kernelspec": {
"display_name": ".venv",
"display_name": "openstef (3.12.10)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.10"
}
},
"nbformat": 4,
Expand Down
47 changes: 45 additions & 2 deletions examples/tutorials/model_explainability.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
# format_name: percent
# format_version: '1.3'
# kernelspec:
# display_name: .venv
# display_name: openstef (3.12.10)
# language: python
# name: python3
# ---
Expand All @@ -23,6 +23,8 @@
import warnings
from typing import cast

from openstef_meta.presets import EnsembleForecastingWorkflowConfig, create_ensemble_forecasting_workflow

warnings.filterwarnings("ignore")

from openstef_core.testing import configure_notebook_display, setup_notebook_logging
Expand Down Expand Up @@ -122,7 +124,7 @@
# tiles represent more influential features.

# %% tags=["hide-input"]
from openstef_models.explainability import ExplainableForecaster
from openstef_models.explainability import ExplainableForecaster, ModelContributionsPlotter
from openstef_models.models.forecasting_model import ForecastingModel

forecaster = cast(ForecastingModel, workflow.model).forecaster
Expand Down Expand Up @@ -193,6 +195,47 @@
fig.update_layout(title="Prediction decomposition (timestep 48)", height=500)
fig.show()

# %% [markdown]
# ## Ensemble model weights
#
# For each forecast quantile, a classifier estimates which base model is likely
# to perform best. Its predicted probabilities become nonnegative model weights
# that sum to one at each timestep. The top panel shows the ensemble forecast
# for the selected quantile; the stacked area below shows each model's weight.
# The plot defaults to P50 when no quantile is specified.

# %% tags=["hide-input"]
ensemble_workflow = create_ensemble_forecasting_workflow(
config=EnsembleForecastingWorkflowConfig(
model_id="example_gblinear_lgbm_ensemble",
ensemble_type="learned_weights",
base_models=["gblinear", "lgbm"],
combiner_model="lgbm",
quantiles=[Q(0.1), Q(0.5), Q(0.9)],
horizons=[LeadTime.from_string("PT36H")],
mlflow_storage=None,
temperature_column="temperature_2m",
relative_humidity_column="relative_humidity_2m",
wind_speed_column="wind_speed_10m",
radiation_column="shortwave_radiation",
pressure_column="surface_pressure",
)
)
ensemble_result = ensemble_workflow.fit(train_dataset)
ensemble_forecast, model_weights = ensemble_workflow.model.predict_with_contributions(
predict_dataset, forecast_start=train_end
)

# %%
fig = ModelContributionsPlotter.plot_stacked_area(
model_weights,
forecast=ensemble_forecast,
quantile=Q(0.1),

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Image

@egordm in the heatmap you showed me the contributions of models at different quantiles simulteneously. That is not what this figure does, it shows the learned model weights for one quantile. It cannot replace the heatmap, but is it a useful addition? Are there tweaks I can do to make it as useful to you as the heatmap?

)
fig.update_layout(height=650)
fig.show()
# fig.write_html("ensemble_model_weights.html", include_plotlyjs=True)

# %% [markdown]
# ## Next steps
#
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -673,5 +673,41 @@ def predict_contributions(
original_data=data,
)

def predict_with_contributions(
self,
data: TimeSeriesDataset,
forecast_start: datetime | None = None,
) -> tuple[ForecastDataset, TimeSeriesDataset]:
"""Generate forecasts and per-model contributions in a single pass.

Args:
data: Input time series dataset.
forecast_start: Optional start time for forecasts.

Returns:
A tuple of ``(forecast, contributions)`` where *forecast* is the
ensemble prediction (a :class:`ForecastDataset`) and *contributions*
is the :class:`TimeSeriesDataset` returned by the configured combiner.
Its meaning depends on the combiner; for learned weights, columns
contain per-model selection weights.

Raises:
NotFittedError: If the model has not been fitted yet.
"""
if not self.is_fitted:
raise NotFittedError(self.__class__.__name__)

ensemble_predictions = self._predict_forecasters(data=data, forecast_start=forecast_start)

forecast = self._predict_transform_combiner(
ensemble_dataset=ensemble_predictions,
original_data=data,
)
contributions = self._predict_contributions_combiner(
ensemble_dataset=ensemble_predictions,
original_data=data,
)
return forecast, contributions

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

added this so that I would have both the forecast and the contributions for the two model contributions plotter panels. Does it make sense to add this here? Or would another approach be better suited?



__all__ = ["EnsembleForecastingModel", "EnsembleModelFitResult", "ModelFitResult"]
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,23 @@ def test_forecasting_model__predict__raises_error_when_not_fitted(
model.predict(data=sample_timeseries_dataset)


def test_predict_with_contributions_matches_separate_calls(
sample_timeseries_dataset: TimeSeriesDataset, model: EnsembleForecastingModel
) -> None:
"""Combined prediction matches separate forecast and contribution calls."""
model.fit(data=sample_timeseries_dataset)
forecast_start = datetime.fromisoformat("2025-01-01T12:00:00")

forecast_combined, contributions_combined = model.predict_with_contributions(
sample_timeseries_dataset, forecast_start=forecast_start
)
forecast_separate = model.predict(sample_timeseries_dataset, forecast_start=forecast_start)
contributions_separate = model.predict_contributions(sample_timeseries_dataset, forecast_start=forecast_start)

pd.testing.assert_frame_equal(forecast_combined.data, forecast_separate.data)
pd.testing.assert_frame_equal(contributions_combined.data, contributions_separate.data)


def test_forecasting_model__score__returns_metrics(
sample_timeseries_dataset: TimeSeriesDataset, model: EnsembleForecastingModel
):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,12 @@
"""

from .mixins import ContributionsMixin, ExplainableForecaster
from .plotters import ContributionsPlotter, FeatureImportancePlotter
from .plotters import ContributionsPlotter, FeatureImportancePlotter, ModelContributionsPlotter

__all__ = [
"ContributionsMixin",
"ContributionsPlotter",
"ExplainableForecaster",
"FeatureImportancePlotter",
"ModelContributionsPlotter",
]
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,10 @@

from .contributions_plotter import ContributionsPlotter
from .feature_importance_plotter import FeatureImportancePlotter
from .model_contributions_plotter import ModelContributionsPlotter

__all__ = [
"ContributionsPlotter",
"FeatureImportancePlotter",
"ModelContributionsPlotter",
]
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
# SPDX-FileCopyrightText: 2026 Contributors to the OpenSTEF project <openstef@lfenergy.org>
#
# SPDX-License-Identifier: MPL-2.0

"""Layout helpers for explainability plots."""

from __future__ import annotations

from typing import TYPE_CHECKING

import plotly.graph_objects as go
from plotly.subplots import make_subplots

if TYPE_CHECKING:
import pandas as pd


class VerticalTwoPanelPlotLayout:
"""Create vertically aligned subplots."""

@staticmethod
def make_two_panel_time_figure() -> go.Figure:
"""Create the vertically aligned subplot layout for two subplots."""
return make_subplots(rows=2, cols=1, shared_xaxes=True, row_heights=[0.2, 0.8], vertical_spacing=0.03)

@staticmethod
def add_top_panel_line_plot(
fig: go.Figure, x: pd.Index, y: pd.Series, *, name: str = "Prediction", show_legend: bool = False
) -> go.Figure:
"""Add a lineplot. Can be used to add a top panel prediction line plot."""
return fig.add_trace(
go.Scatter(
x=x, y=y, mode="lines", line={"color": "black", "width": 1.5}, name=name, showlegend=show_legend
),
row=1,
col=1,
)
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,9 @@
from typing import TYPE_CHECKING, cast

import plotly.graph_objects as go
from plotly.subplots import make_subplots

from openstef_core.datasets import TimeSeriesDataset # noqa: TC001
from openstef_models.explainability.plotters.common import VerticalTwoPanelPlotLayout

if TYPE_CHECKING:
import pandas as pd
Expand Down Expand Up @@ -69,26 +69,9 @@ def plot_heatmap(
if bias is not None:
prediction += bias

fig = make_subplots(
rows=2,
cols=1,
shared_xaxes=True,
row_heights=[0.2, 0.8],
vertical_spacing=0.03,
)
fig = VerticalTwoPanelPlotLayout.make_two_panel_time_figure()

fig.add_trace(
go.Scatter(
x=df.index,
y=prediction,
mode="lines",
name="Prediction",
line={"color": "black", "width": 1.5},
showlegend=False,
),
row=1,
col=1,
)
VerticalTwoPanelPlotLayout.add_top_panel_line_plot(fig, df.index, prediction)
fig.add_trace(heatmap, row=2, col=1)

fig.update_layout(
Expand Down
Loading
Loading