From 55ee206bc903a87d6eb206ad9b14d94ece1ade34 Mon Sep 17 00:00:00 2001 From: Eric Bariaux <375613+ebariaux@users.noreply.github.com> Date: Thu, 8 Oct 2026 14:32:41 +0200 Subject: [PATCH 1/9] Read curve type to determine how to perform the expansion/gap filling --- .../protocol/PublicationMarketDocument.java | 8 +++ .../PublicationMarketDocumentTest.groovy | 65 +++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 entsoe/src/test/groovy/org/openremote/extension/entsoe/agent/protocol/PublicationMarketDocumentTest.groovy diff --git a/entsoe/src/main/java/org/openremote/extension/entsoe/agent/protocol/PublicationMarketDocument.java b/entsoe/src/main/java/org/openremote/extension/entsoe/agent/protocol/PublicationMarketDocument.java index 9d94973..5042024 100644 --- a/entsoe/src/main/java/org/openremote/extension/entsoe/agent/protocol/PublicationMarketDocument.java +++ b/entsoe/src/main/java/org/openremote/extension/entsoe/agent/protocol/PublicationMarketDocument.java @@ -57,9 +57,17 @@ public String getEnd() { @XmlAccessorType(XmlAccessType.FIELD) public static class TimeSeries { + // ENTSO-E time series guide section 2: an omitted curve type defaults to A01. + @XmlElement(name = "curveType", namespace = NS) + private String curveType = "A01"; + @XmlElement(name = "Period", namespace = NS) private List periods; + public String getCurveType() { + return curveType; + } + public List getPeriods() { return periods; } diff --git a/entsoe/src/test/groovy/org/openremote/extension/entsoe/agent/protocol/PublicationMarketDocumentTest.groovy b/entsoe/src/test/groovy/org/openremote/extension/entsoe/agent/protocol/PublicationMarketDocumentTest.groovy new file mode 100644 index 0000000..57fae0f --- /dev/null +++ b/entsoe/src/test/groovy/org/openremote/extension/entsoe/agent/protocol/PublicationMarketDocumentTest.groovy @@ -0,0 +1,65 @@ +/* + * Copyright 2026, OpenRemote Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +package org.openremote.extension.entsoe.agent.protocol + +import jakarta.xml.bind.JAXBContext +import spock.lang.Shared +import spock.lang.Specification + +class PublicationMarketDocumentTest extends Specification { + + @Shared + JAXBContext context = JAXBContext.newInstance(PublicationMarketDocument) + + def "unmarshals curve type when #description"() { + given: + def xml = """ + + + ${curveTypeXml} + + + 2026-02-16T23:00Z + 2026-02-17T00:00Z + + PT15M + + 1 + 73.24 + + + + +""" + + when: + def document = context.createUnmarshaller().unmarshal(new StringReader(xml)) as PublicationMarketDocument + + then: + document.timeSeries.size() == 1 + document.timeSeries[0].getCurveType() == expectedCurveType + + where: + description | curveTypeXml | expectedCurveType + "A01 is explicit" | "A01" | "A01" + "A03 is explicit" | "A03" | "A03" + "curveType is absent" | "" | "A01" + "an unknown code is supplied" | "A99" | "A99" + } +} From 143409754bce9bf62eef7abfbbf4a6cd9ae05aef Mon Sep 17 00:00:00 2001 From: Eric Bariaux <375613+ebariaux@users.noreply.github.com> Date: Thu, 8 Oct 2026 14:55:24 +0200 Subject: [PATCH 2/9] Fix ENTSO-E A03 price curve expansion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Expand variable-size blocks through skipped positions to each period’s end, respecting unavailable values and the future timestamp cutoff. Add decoder tests and correct integration expectations. --- .../entsoe/agent/protocol/EntsoeProtocol.java | 58 ++++++ .../agent/protocol/EntsoeCurveTest.groovy | 197 ++++++++++++++++++ .../agent/protocol/EntsoeProtocolTest.groovy | 24 ++- 3 files changed, 268 insertions(+), 11 deletions(-) create mode 100644 entsoe/src/test/groovy/org/openremote/extension/entsoe/agent/protocol/EntsoeCurveTest.groovy diff --git a/entsoe/src/main/java/org/openremote/extension/entsoe/agent/protocol/EntsoeProtocol.java b/entsoe/src/main/java/org/openremote/extension/entsoe/agent/protocol/EntsoeProtocol.java index ebd43ed..18d8bbf 100644 --- a/entsoe/src/main/java/org/openremote/extension/entsoe/agent/protocol/EntsoeProtocol.java +++ b/entsoe/src/main/java/org/openremote/extension/entsoe/agent/protocol/EntsoeProtocol.java @@ -473,6 +473,11 @@ protected List> buildPredictedDatapoints(PublicationMarketDocu continue; } + if ("A03".equals(timeSeries.getCurveType())) { + addVariableBlockDatapoints(period, start, resolution, nowMillis, values); + continue; + } + period.getPoints().stream() .filter( point -> @@ -495,6 +500,59 @@ protected List> buildPredictedDatapoints(PublicationMarketDocu return values; } + protected void addVariableBlockDatapoints( + PublicationMarketDocument.Period period, + Instant start, + Duration resolution, + long nowMillis, + List> values) { + if (resolution.isZero() || resolution.isNegative()) { + LOG.warning("Cannot expand ENTSO-E A03 period with non-positive resolution"); + return; + } + if (period.getTimeInterval() == null || period.getTimeInterval().getEnd() == null) { + LOG.warning("Cannot expand ENTSO-E A03 period without its end time"); + return; + } + + final Instant end; + try { + end = parseEntsoeInstant(period.getTimeInterval().getEnd()); + } catch (Exception e) { + LOG.log(Level.WARNING, e, () -> "Could not parse ENTSO-E A03 period end"); + return; + } + + // Keep price-less points: they mark where the preceding price stops being available. + List points = + period.getPoints().stream() + .filter(point -> point.getPosition() != null && point.getPosition() > 0) + .sorted(Comparator.comparingInt(PublicationMarketDocument.Point::getPosition)) + .toList(); + + for (int i = 0; i < points.size(); i++) { + PublicationMarketDocument.Point point = points.get(i); + if (point.getPriceAmount() == null) { + continue; + } + Instant blockStart = start.plus(resolution.multipliedBy(point.getPosition() - 1L)); + Instant blockEnd = end; + if (i + 1 < points.size()) { + Instant nextStart = + start.plus(resolution.multipliedBy(points.get(i + 1).getPosition() - 1L)); + if (nextStart.isBefore(blockEnd)) { + blockEnd = nextStart; + } + } + for (Instant time = blockStart; time.isBefore(blockEnd); time = time.plus(resolution)) { + long timestamp = time.toEpochMilli(); + if (timestamp >= nowMillis) { + values.add(new ValueDatapoint<>(timestamp, point.getPriceAmount())); + } + } + } + } + protected Instant parseEntsoeInstant(String value) { try { return Instant.parse(value); diff --git a/entsoe/src/test/groovy/org/openremote/extension/entsoe/agent/protocol/EntsoeCurveTest.groovy b/entsoe/src/test/groovy/org/openremote/extension/entsoe/agent/protocol/EntsoeCurveTest.groovy new file mode 100644 index 0000000..f42c702 --- /dev/null +++ b/entsoe/src/test/groovy/org/openremote/extension/entsoe/agent/protocol/EntsoeCurveTest.groovy @@ -0,0 +1,197 @@ +/* + * Copyright 2026, OpenRemote Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +package org.openremote.extension.entsoe.agent.protocol + +import org.openremote.container.timer.TimerService +import spock.lang.Specification + +import java.time.Duration +import java.time.Instant + +class EntsoeCurveTest extends Specification { + + private static final Instant START = Instant.parse("2026-02-16T23:00:00Z") + private EntsoeProtocol protocol + + def setup() { + protocol = new EntsoeProtocol(new EntsoeAgent()) + setNow(START) + } + + def cleanup() { + EntsoeProtocol.client.getAndSet(null)?.close() + } + + def "A03 expands skipped positions and the final block in position order"() { + given: + def document = document("A03", period(START, START.plusSeconds(3600), "PT15M", [[3, 0], [1, -5]])) + + when: + def values = protocol.buildPredictedDatapoints(document) + + then: + values.collect { + it.timestamp + } == (0..3).collect { + START.plusSeconds(it * 900L).toEpochMilli() + } + values.collect { it.value } == [-5G, -5G, 0G, 0G] + } + + def "a single A03 point covers the full period at #resolution resolution"() { + given: + def document = document("A03", period(START, START.plusSeconds(14400), resolution, [[1, 80]])) + + when: + def values = protocol.buildPredictedDatapoints(document) + + then: + values.size() == count + values.collect { it.timestamp } == (0..${START.plusSeconds(3600)}", "") + def document = document("A03", periodXml) + + expect: + protocol.buildPredictedDatapoints(document).isEmpty() + } + + private void setNow(Instant now) { + protocol.timerService = Stub(TimerService) { + getCurrentTimeMillis() >> now.toEpochMilli() + } + } + + private PublicationMarketDocument document(String curveType, String periods) { + def curveTypeXml = curveType == null ? "" : "${curveType}" + def xml = """ + + + ${curveTypeXml} + ${periods} + + +""" + protocol.unmarshalPublicationMarketDocument(xml) + } + + private static String period(Instant start, Instant end, String resolution, List points) { + def pointXml = points.collect { point -> + "${point[0]}" + + (point[1] == null ? "" : "${point[1]}") + "" + }.join("\n") + """ + + ${start}${end} + ${resolution} + ${pointXml} + +""" + } +} diff --git a/entsoe/src/test/groovy/org/openremote/extension/entsoe/agent/protocol/EntsoeProtocolTest.groovy b/entsoe/src/test/groovy/org/openremote/extension/entsoe/agent/protocol/EntsoeProtocolTest.groovy index 3bc4b7b..82f1de4 100644 --- a/entsoe/src/test/groovy/org/openremote/extension/entsoe/agent/protocol/EntsoeProtocolTest.groovy +++ b/entsoe/src/test/groovy/org/openremote/extension/entsoe/agent/protocol/EntsoeProtocolTest.groovy @@ -398,7 +398,7 @@ class EntsoeProtocolTest extends Specification implements ManagerContainerTrait then: "predicted datapoints are written using period start and resolution" conditions.eventually { List datapoints = assetPredictedDatapointService.getDatapoints(attributeRef) - assert datapoints.size() == 4 + assert datapoints.size() == 96 def asc = datapoints.sort { it.timestamp } def start = Instant.parse(DATASET_START).toEpochMilli() @@ -413,6 +413,8 @@ class EntsoeProtocolTest extends Specification implements ManagerContainerTrait assert (asc[1].value as BigDecimal).compareTo(69.79G) == 0 assert (asc[2].value as BigDecimal).compareTo(65.84G) == 0 assert (asc[3].value as BigDecimal).compareTo(65.05G) == 0 + assert asc[95].timestamp == start + (95 * step) + assert asc.drop(3).every { (it.value as BigDecimal).compareTo(65.05G) == 0 } } cleanup: "remove created assets and mock client" @@ -481,7 +483,7 @@ class EntsoeProtocolTest extends Specification implements ManagerContainerTrait List datapoints = assetPredictedDatapointService.getDatapoints(attributeRef).sort { it.timestamp } - assert datapoints.size() == 2 + assert datapoints.size() == 94 def start = Instant.parse(DATASET_START).toEpochMilli() def step = 15 * 60 * 1000L @@ -572,8 +574,8 @@ class EntsoeProtocolTest extends Specification implements ManagerContainerTrait it.timestamp } - assert beDatapoints.size() == 4 - assert nlDatapoints.size() == 4 + assert beDatapoints.size() == 96 + assert nlDatapoints.size() == 96 assert (beDatapoints[0].value as BigDecimal).compareTo(73.24G) == 0 assert (beDatapoints[1].value as BigDecimal).compareTo(69.79G) == 0 @@ -670,8 +672,8 @@ class EntsoeProtocolTest extends Specification implements ManagerContainerTrait it.timestamp } - assert datapointsOne.size() == 4 - assert datapointsTwo.size() == 4 + assert datapointsOne.size() == 96 + assert datapointsTwo.size() == 96 assert datapointsOne.collect { [it.timestamp, it.value] } == datapointsTwo.collect { @@ -1061,7 +1063,7 @@ class EntsoeProtocolTest extends Specification implements ManagerContainerTrait List datapoints = assetPredictedDatapointService.getDatapoints(attributeRef).sort { it.timestamp } - assert datapoints.size() == 4 + assert datapoints.size() == 8 def firstPeriodStart = Instant.parse(DATASET_START).toEpochMilli() def secondPeriodStart = Instant.parse("2026-02-17T00:00:00.000Z").toEpochMilli() @@ -1069,13 +1071,13 @@ class EntsoeProtocolTest extends Specification implements ManagerContainerTrait assert datapoints[0].timestamp == firstPeriodStart assert datapoints[1].timestamp == firstPeriodStart + step - assert datapoints[2].timestamp == secondPeriodStart - assert datapoints[3].timestamp == secondPeriodStart + step + assert datapoints[4].timestamp == secondPeriodStart + assert datapoints[5].timestamp == secondPeriodStart + step assert (datapoints[0].value as BigDecimal).compareTo(51.11G) == 0 assert (datapoints[1].value as BigDecimal).compareTo(52.22G) == 0 - assert (datapoints[2].value as BigDecimal).compareTo(53.33G) == 0 - assert (datapoints[3].value as BigDecimal).compareTo(54.44G) == 0 + assert (datapoints[4].value as BigDecimal).compareTo(53.33G) == 0 + assert (datapoints[5].value as BigDecimal).compareTo(54.44G) == 0 } cleanup: "remove created assets and mock client" From ad48842ee4ed5d1f8d967a9716b5162333417f1b Mon Sep 17 00:00:00 2001 From: Eric Bariaux <375613+ebariaux@users.noreply.github.com> Date: Thu, 8 Oct 2026 15:00:42 +0200 Subject: [PATCH 3/9] Skip invalid curve types --- .../entsoe/agent/protocol/EntsoeProtocol.java | 9 +++- .../agent/protocol/EntsoeCurveTest.groovy | 48 +++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/entsoe/src/main/java/org/openremote/extension/entsoe/agent/protocol/EntsoeProtocol.java b/entsoe/src/main/java/org/openremote/extension/entsoe/agent/protocol/EntsoeProtocol.java index 18d8bbf..d2c72cf 100644 --- a/entsoe/src/main/java/org/openremote/extension/entsoe/agent/protocol/EntsoeProtocol.java +++ b/entsoe/src/main/java/org/openremote/extension/entsoe/agent/protocol/EntsoeProtocol.java @@ -444,6 +444,13 @@ protected List> buildPredictedDatapoints(PublicationMarketDocu } for (PublicationMarketDocument.TimeSeries timeSeries : document.getTimeSeries()) { + String curveType = timeSeries.getCurveType(); + if (!"A01".equals(curveType) && !"A03".equals(curveType)) { + LOG.warning( + "Skipping ENTSO-E time series with unsupported curve type: '" + curveType + "'"); + continue; + } + if (timeSeries.getPeriods() == null || timeSeries.getPeriods().isEmpty()) { continue; } @@ -473,7 +480,7 @@ protected List> buildPredictedDatapoints(PublicationMarketDocu continue; } - if ("A03".equals(timeSeries.getCurveType())) { + if ("A03".equals(curveType)) { addVariableBlockDatapoints(period, start, resolution, nowMillis, values); continue; } diff --git a/entsoe/src/test/groovy/org/openremote/extension/entsoe/agent/protocol/EntsoeCurveTest.groovy b/entsoe/src/test/groovy/org/openremote/extension/entsoe/agent/protocol/EntsoeCurveTest.groovy index f42c702..f5adf6a 100644 --- a/entsoe/src/test/groovy/org/openremote/extension/entsoe/agent/protocol/EntsoeCurveTest.groovy +++ b/entsoe/src/test/groovy/org/openremote/extension/entsoe/agent/protocol/EntsoeCurveTest.groovy @@ -162,6 +162,54 @@ class EntsoeCurveTest extends Specification { protocol.buildPredictedDatapoints(document).isEmpty() } + def "unsupported or empty curve type '#curveType' produces no predictions"() { + given: + def document = document(curveType, period(START, START.plusSeconds(3600), "PT15M", [[1, 80]])) + + expect: + protocol.buildPredictedDatapoints(document).isEmpty() + + where: + curveType << ["A02", "A04", "A05", "A99", "", " "] + } + + def "rejected series do not prevent valid series in the same document from being decoded"() { + given: + def xml = """ + + + A01 + ${period(START, START.plusSeconds(1800), "PT15M", [[1, 80], [2, 81]])} + + + A99 + ${period(START.plusSeconds(1800), START.plusSeconds(3600), "PT15M", [[1, 999]])} + + + A03 + ${period(START.plusSeconds(3600), START.plusSeconds(5400), "PT15M", [[1, 90]])} + + + + ${period(START.plusSeconds(5400), START.plusSeconds(7200), "PT15M", [[1, 999]])} + + + ${period(START.plusSeconds(7200), START.plusSeconds(9000), "PT15M", [[1, 100], [2, 101]])} + + +""" + def document = protocol.unmarshalPublicationMarketDocument(xml) + + when: + def values = protocol.buildPredictedDatapoints(document) + + then: + values.collect { it.timestamp } == [0, 900, 3600, 4500, 7200, 8100].collect { + START.plusSeconds(it).toEpochMilli() + } + values.collect { it.value } == [80G, 81G, 90G, 90G, 100G, 101G] + } + private void setNow(Instant now) { protocol.timerService = Stub(TimerService) { getCurrentTimeMillis() >> now.toEpochMilli() From 385c9c4f612a4ed97084378f29aa4d099d5c9865 Mon Sep 17 00:00:00 2001 From: Eric Bariaux <375613+ebariaux@users.noreply.github.com> Date: Thu, 8 Oct 2026 15:08:41 +0200 Subject: [PATCH 4/9] Validate ENTSO-E periods and A01 completeness Reject malformed timing and missing, duplicate, or out-of-range positions before generating predictions. Require complete A01 periods, preserve valid neighboring periods, and update regression tests. --- .../entsoe/agent/protocol/EntsoeProtocol.java | 108 +++++++------ .../agent/protocol/EntsoeCurveTest.groovy | 153 ++++++++++++++++-- .../agent/protocol/EntsoeProtocolTest.groovy | 32 +--- 3 files changed, 211 insertions(+), 82 deletions(-) diff --git a/entsoe/src/main/java/org/openremote/extension/entsoe/agent/protocol/EntsoeProtocol.java b/entsoe/src/main/java/org/openremote/extension/entsoe/agent/protocol/EntsoeProtocol.java index d2c72cf..7a4917e 100644 --- a/entsoe/src/main/java/org/openremote/extension/entsoe/agent/protocol/EntsoeProtocol.java +++ b/entsoe/src/main/java/org/openremote/extension/entsoe/agent/protocol/EntsoeProtocol.java @@ -36,9 +36,11 @@ import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Comparator; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; @@ -460,38 +462,54 @@ protected List> buildPredictedDatapoints(PublicationMarketDocu continue; } - PublicationMarketDocument.PeriodTimeInterval timeInterval = - period.getTimeInterval() != null - ? period.getTimeInterval() - : document.getPeriodTimeInterval(); + PublicationMarketDocument.PeriodTimeInterval timeInterval = period.getTimeInterval(); if (timeInterval == null || timeInterval.getStart() == null + || timeInterval.getEnd() == null || period.getResolution() == null) { + LOG.warning("Skipping ENTSO-E period without its own start, end, or resolution"); continue; } final Instant start; + final Instant end; final Duration resolution; + final long intervalCount; try { start = parseEntsoeInstant(timeInterval.getStart()); + end = parseEntsoeInstant(timeInterval.getEnd()); resolution = Duration.parse(period.getResolution()); + if (!end.isAfter(start) || resolution.isZero() || resolution.isNegative()) { + LOG.warning("Skipping ENTSO-E period with non-positive duration or resolution"); + continue; + } + Duration duration = Duration.between(start, end); + intervalCount = duration.dividedBy(resolution); + if (intervalCount == 0 || !resolution.multipliedBy(intervalCount).equals(duration)) { + LOG.warning( + "Skipping ENTSO-E period whose duration is not a multiple of its resolution"); + continue; + } } catch (Exception e) { LOG.log(Level.WARNING, e, () -> "Could not parse ENTSO-E timeseries time data"); continue; } + if (!hasValidPeriodPositions(period.getPoints(), curveType, intervalCount)) { + continue; + } + List points = + period.getPoints().stream() + .sorted(Comparator.comparingInt(PublicationMarketDocument.Point::getPosition)) + .toList(); + if ("A03".equals(curveType)) { - addVariableBlockDatapoints(period, start, resolution, nowMillis, values); + addVariableBlockDatapoints(points, start, end, resolution, nowMillis, values); continue; } - period.getPoints().stream() - .filter( - point -> - point.getPosition() != null - && point.getPosition() > 0 - && point.getPriceAmount() != null) - .sorted(Comparator.comparingInt(PublicationMarketDocument.Point::getPosition)) + points.stream() + .filter(point -> point.getPriceAmount() != null) .forEach( point -> { long timestamp = @@ -507,50 +525,50 @@ protected List> buildPredictedDatapoints(PublicationMarketDocu return values; } + protected boolean hasValidPeriodPositions( + List points, String curveType, long intervalCount) { + Set positions = new HashSet<>(); + for (PublicationMarketDocument.Point point : points) { + Integer position = point != null ? point.getPosition() : null; + if (position == null + || position < 1 + || position > intervalCount + || !positions.add(position)) { + LOG.warning("Skipping ENTSO-E period with missing, duplicate, or out-of-range positions"); + return false; + } + } + + if (!positions.contains(1)) { + LOG.warning("Skipping ENTSO-E period without position 1"); + return false; + } + // Unique positions in [1, intervalCount] cover A01 completely only when all are present. + if ("A01".equals(curveType) && positions.size() != intervalCount) { + LOG.warning("Skipping incomplete ENTSO-E A01 period"); + return false; + } + return true; + } + protected void addVariableBlockDatapoints( - PublicationMarketDocument.Period period, + List points, Instant start, + Instant end, Duration resolution, long nowMillis, List> values) { - if (resolution.isZero() || resolution.isNegative()) { - LOG.warning("Cannot expand ENTSO-E A03 period with non-positive resolution"); - return; - } - if (period.getTimeInterval() == null || period.getTimeInterval().getEnd() == null) { - LOG.warning("Cannot expand ENTSO-E A03 period without its end time"); - return; - } - - final Instant end; - try { - end = parseEntsoeInstant(period.getTimeInterval().getEnd()); - } catch (Exception e) { - LOG.log(Level.WARNING, e, () -> "Could not parse ENTSO-E A03 period end"); - return; - } - - // Keep price-less points: they mark where the preceding price stops being available. - List points = - period.getPoints().stream() - .filter(point -> point.getPosition() != null && point.getPosition() > 0) - .sorted(Comparator.comparingInt(PublicationMarketDocument.Point::getPosition)) - .toList(); - + // Validated, sorted points include price-less boundaries so prices never extend through them. for (int i = 0; i < points.size(); i++) { PublicationMarketDocument.Point point = points.get(i); if (point.getPriceAmount() == null) { continue; } Instant blockStart = start.plus(resolution.multipliedBy(point.getPosition() - 1L)); - Instant blockEnd = end; - if (i + 1 < points.size()) { - Instant nextStart = - start.plus(resolution.multipliedBy(points.get(i + 1).getPosition() - 1L)); - if (nextStart.isBefore(blockEnd)) { - blockEnd = nextStart; - } - } + Instant blockEnd = + i + 1 < points.size() + ? start.plus(resolution.multipliedBy(points.get(i + 1).getPosition() - 1L)) + : end; for (Instant time = blockStart; time.isBefore(blockEnd); time = time.plus(resolution)) { long timestamp = time.toEpochMilli(); if (timestamp >= nowMillis) { diff --git a/entsoe/src/test/groovy/org/openremote/extension/entsoe/agent/protocol/EntsoeCurveTest.groovy b/entsoe/src/test/groovy/org/openremote/extension/entsoe/agent/protocol/EntsoeCurveTest.groovy index f5adf6a..df4509c 100644 --- a/entsoe/src/test/groovy/org/openremote/extension/entsoe/agent/protocol/EntsoeCurveTest.groovy +++ b/entsoe/src/test/groovy/org/openremote/extension/entsoe/agent/protocol/EntsoeCurveTest.groovy @@ -124,9 +124,11 @@ class EntsoeCurveTest extends Specification { values.collect { it.value } == [80G, 80G, 90G] } - def "#curveType keeps fixed-size point behavior"() { + def "complete fixed-size periods support #curveType at #resolution resolution"() { given: - def document = document(curveType, period(START, START.plusSeconds(3600), "PT15M", [[1, 80], [3, 90]])) + def step = Duration.parse(resolution) + def document = document(curveType, period(START, START.plus(step.multipliedBy(4)), resolution, + [[4, 90], [2, 0], [1, -5], [3, 80]])) when: def values = protocol.buildPredictedDatapoints(document) @@ -134,32 +136,156 @@ class EntsoeCurveTest extends Specification { then: values.collect { it.timestamp - } == [START.toEpochMilli(), START.plusSeconds(1800).toEpochMilli()] - values.collect { it.value } == [80G, 90G] + } == (0..3).collect { + START.plus(step.multipliedBy(it)).toEpochMilli() + } + values.collect { it.value } == [-5G, 0G, 80G, 90G] where: - curveType << ["A01", null] + [curveType, resolution] << [["A01", null], ["PT15M", "PT30M", "PT60M"]].combinations() + } + + def "A01 filters past intervals after validating the complete period"() { + given: + setNow(START.plusSeconds(1200)) + def document = document("A01", period(START, START.plusSeconds(3600), "PT15M", + [[1, 80], [2, 81], [3, 82], [4, 83]])) + + when: + def values = protocol.buildPredictedDatapoints(document) + + then: + values.collect { it.timestamp } == [1800, 2700].collect { START.plusSeconds(it).toEpochMilli() } + values.collect { it.value } == [82G, 83G] + } + + def "incomplete fixed-size periods are rejected for #curveType and positions #positions"() { + given: + def document = document(curveType, period(START, START.plusSeconds(3600), "PT15M", + positions.collect { [it, 80] })) + + expect: + protocol.buildPredictedDatapoints(document).isEmpty() + + where: + [curveType, positions] << [["A01", null], [[2, 3, 4], [1, 3, 4], [1, 2, 3]]].combinations() + } + + def "#curveType rejects invalid positions #positions without partial output"() { + given: + def periodXml = period(START, START.plusSeconds(3600), "PT15M", positions.collect { [it, 80] }) + .replace("null", "") + def document = document(curveType, periodXml) + + expect: + protocol.buildPredictedDatapoints(document).isEmpty() + + where: + [curveType, positions] << [ + ["A01", "A03"], + [[1, 2, 2, 4], [0, 2, 3, 4], [-1, 2, 3, 4], [1, 2, 3, 5], [1, 2, 3, 2147483647], [1, 2, 3, null]] + ].combinations() + } + + def "A03 requires a block starting at position one"() { + given: + def document = document("A03", period(START, START.plusSeconds(3600), "PT15M", [[2, 80], [4, 90]])) + + expect: + protocol.buildPredictedDatapoints(document).isEmpty() } - def "A03 does not expand a non-positive resolution #resolution"() { + def "#curveType rejects invalid period timing: #description"() { given: - def document = document("A03", period(START, START.plusSeconds(3600), resolution, [[1, 80]])) + def periodXml = period(START, START.plusSeconds(endOffset), resolution, [[1, 80], [2, 81], [3, 82], [4, 83]]) + def document = document(curveType, periodXml) expect: protocol.buildPredictedDatapoints(document).isEmpty() where: - resolution << ["PT0S", "PT-15M"] + [curveType, description, endOffset, resolution] << ["A01", "A03"].collectMany { type -> + [ + [type, "end equals start", 0, "PT15M"], + [type, "end precedes start", -3600, "PT15M"], + [type, "end is off the resolution grid", 3500, "PT15M"], + [type, "period is shorter than resolution", 600, "PT15M"], + [type, "zero resolution", 3600, "PT0S"], + [type, "negative resolution", 3600, "PT-15M"], + [type, "unparseable resolution", 3600, "invalid"] + ] + } } - def "A03 does not expand a period without an end"() { + def "#curveType requires its own complete time interval and resolution: #description"() { given: - def periodXml = period(START, START.plusSeconds(3600), "PT15M", [[1, 80]]) - .replace("${START.plusSeconds(3600)}", "") - def document = document("A03", periodXml) + def periodXml = period(START, START.plusSeconds(3600), "PT15M", [[1, 80], [2, 81], [3, 82], [4, 83]]) + .replace(removedXml, replacementXml) + def header = "${START}${START.plusSeconds(3600)}" + def document = document(curveType, periodXml, header) expect: protocol.buildPredictedDatapoints(document).isEmpty() + + where: + [curveType, description, removedXml, replacementXml] << ["A01", "A03"].collectMany { type -> + [ + [ + type, + "missing interval", + "${START}${START.plusSeconds(3600)}", + "" + ], + [type, "missing start", "${START}", ""], + [type, "missing end", "${START.plusSeconds(3600)}", ""], + [type, "missing resolution", "PT15M", ""], + [type, "unparseable start", "${START}", "invalid"], + [type, "unparseable end", "${START.plusSeconds(3600)}", "invalid"] + ] + } + } + + def "invalid #curveType periods do not prevent valid periods in the same series from being decoded"() { + given: + def document = document(curveType, + period(START, START.plusSeconds(1800), "PT15M", [[1, 80], [2, 81]]) + + period(START.plusSeconds(1800), START.plusSeconds(3600), "PT15M", [[1, 999], [1, 999]]) + + period(START.plusSeconds(3600), START.plusSeconds(5400), "PT15M", [[1, 90], [2, 91]])) + + when: + def values = protocol.buildPredictedDatapoints(document) + + then: + values.collect { + it.timestamp + } == [0, 900, 3600, 4500].collect { + START.plusSeconds(it).toEpochMilli() + } + values.collect { it.value } == [80G, 81G, 90G, 91G] + + where: + curveType << ["A01", "A03"] + } + + def "#curveType supports a #hours hour period without assuming 24 hours"() { + given: + def points = curveType == "A01" ? (1..hours).collect { [it, 80] } : [[1, 80]] + def document = document(curveType, period(START, START.plusSeconds(hours * 3600L), "PT60M", points)) + + when: + def values = protocol.buildPredictedDatapoints(document) + + then: + values.size() == hours + values.collect { + it.timestamp + } == (0..${curveType}" def xml = """ + ${header} ${curveTypeXml} ${periods} diff --git a/entsoe/src/test/groovy/org/openremote/extension/entsoe/agent/protocol/EntsoeProtocolTest.groovy b/entsoe/src/test/groovy/org/openremote/extension/entsoe/agent/protocol/EntsoeProtocolTest.groovy index 82f1de4..efd315b 100644 --- a/entsoe/src/test/groovy/org/openremote/extension/entsoe/agent/protocol/EntsoeProtocolTest.groovy +++ b/entsoe/src/test/groovy/org/openremote/extension/entsoe/agent/protocol/EntsoeProtocolTest.groovy @@ -185,13 +185,13 @@ class EntsoeProtocolTest extends Specification implements ManagerContainerTrait 2026-02-16T23:00Z - 2026-02-17T23:00Z + 2026-02-17T00:00Z 2026-02-16T23:00Z - 2026-02-17T23:00Z + 2026-02-17T00:00Z PT15M @@ -259,13 +259,14 @@ class EntsoeProtocolTest extends Specification implements ManagerContainerTrait 2026-02-16T23:00Z - 2026-02-17T23:00Z + 2026-02-17T00:00Z + A01 2026-02-16T23:00Z - 2026-02-17T23:00Z + 2026-02-17T00:00Z PT15M @@ -927,7 +928,7 @@ class EntsoeProtocolTest extends Specification implements ManagerContainerTrait closeClient() } - def "ENTSO-E integration test keeps position timing when intermediate point is missing"() { + def "ENTSO-E integration test rejects an A01 period with a missing intermediate position"() { given: "the container environment is started" requestCountByZone.clear() def conditions = new PollingConditions(timeout: 10, delay: 0.2) @@ -977,25 +978,8 @@ class EntsoeProtocolTest extends Specification implements ManagerContainerTrait when: "a polling update is triggered" protocol.updateAllLinkedAttributes() - then: "only provided points are stored and their timestamps follow position offsets" - conditions.eventually { - List datapoints = assetPredictedDatapointService.getDatapoints(attributeRef).sort { - it.timestamp - } - assert datapoints.size() == 3 - - def start = Instant.parse(DATASET_START).toEpochMilli() - def step = 15 * 60 * 1000L - - assert datapoints[0].timestamp == start - assert datapoints[1].timestamp == start + step - assert datapoints[2].timestamp == start + (3 * step) - assert datapoints[2].timestamp - datapoints[1].timestamp == 2 * step - - assert (datapoints[0].value as BigDecimal).compareTo(81.11G) == 0 - assert (datapoints[1].value as BigDecimal).compareTo(82.22G) == 0 - assert (datapoints[2].value as BigDecimal).compareTo(84.44G) == 0 - } + then: "the incomplete A01 period produces no predicted datapoints" + assetPredictedDatapointService.getDatapoints(attributeRef).isEmpty() cleanup: "remove created assets and mock client" if (asset?.id) { From 848c5c5c761ad19bbdad355b5d28424b229dcc16 Mon Sep 17 00:00:00 2001 From: Eric Bariaux <375613+ebariaux@users.noreply.github.com> Date: Thu, 8 Oct 2026 15:29:26 +0200 Subject: [PATCH 5/9] Replace stale ENTSO-E prices with explicit null predictions Persist unavailable A01 intervals and A03 blocks as null values. Add decoding and persistence regression tests, and document that null predictions retain their timestamped entries. --- entsoe/README.md | 17 ++++ .../entsoe/agent/protocol/EntsoeProtocol.java | 24 +++--- .../agent/protocol/EntsoeCurveTest.groovy | 38 +++++++-- .../agent/protocol/EntsoeProtocolTest.groovy | 78 +++++++++++++++++++ 4 files changed, 138 insertions(+), 19 deletions(-) diff --git a/entsoe/README.md b/entsoe/README.md index 8ae49b0..65cb7e9 100644 --- a/entsoe/README.md +++ b/entsoe/README.md @@ -37,3 +37,20 @@ You'll for instance find that `10YNL----------L` is the code to use for The Neth Make sure to also add the `Has predicted data points` configuration to your attribute. ![](img/ENTSO-E-agent-link.png) + +### Unavailable prices and null predictions + +**An unavailable price is stored as a predicted datapoint with a timestamp and a `null` value. The datapoint entry is not deleted.** + +When a point in an otherwise valid returned period has no `price.amount`: + +- For `A01`, the agent writes `null` for that point's interval. +- For `A03`, the agent writes `null` at each resolution interval from that point up to the next point, or the period end if it is the final point. The end boundary is excluded. + +For example, if a stored prediction at `12:15` has price `80` and a later publication explicitly marks that interval unavailable, the stored prediction becomes `(12:15, null)`. If no prediction exists at that timestamp, a null-valued entry is inserted. A later publication with an available price can replace the null value. + +Reading predicted datapoints still returns these timestamped null entries. Consumers must treat `null` as **unavailable**, not as zero, and must not carry a previous price forward through those intervals. Code that expects every prediction to contain a number must handle null values explicitly. + +Only intervals whose start timestamp is at or after the current time are updated. Historical predictions are left unchanged. + +This behavior applies to explicitly present points without a price. An omitted position in `A03` continues the preceding block; it does not by itself mean unavailable. Incomplete `A01` periods are rejected. Fetch failures, no-data acknowledgements, and rejected periods do not clear existing predictions. Gaps between separate returned periods currently produce no updates, so previously stored predictions in those gaps can remain; clearing those stale values is a pending improvement. diff --git a/entsoe/src/main/java/org/openremote/extension/entsoe/agent/protocol/EntsoeProtocol.java b/entsoe/src/main/java/org/openremote/extension/entsoe/agent/protocol/EntsoeProtocol.java index 7a4917e..f8587f1 100644 --- a/entsoe/src/main/java/org/openremote/extension/entsoe/agent/protocol/EntsoeProtocol.java +++ b/entsoe/src/main/java/org/openremote/extension/entsoe/agent/protocol/EntsoeProtocol.java @@ -508,16 +508,15 @@ protected List> buildPredictedDatapoints(PublicationMarketDocu continue; } - points.stream() - .filter(point -> point.getPriceAmount() != null) - .forEach( - point -> { - long timestamp = - start.plus(resolution.multipliedBy(point.getPosition() - 1L)).toEpochMilli(); - if (timestamp >= nowMillis) { - values.add(new ValueDatapoint<>(timestamp, point.getPriceAmount())); - } - }); + // Persist explicit unavailability so a later publication can clear stale prices. + points.forEach( + point -> { + long timestamp = + start.plus(resolution.multipliedBy(point.getPosition() - 1L)).toEpochMilli(); + if (timestamp >= nowMillis) { + values.add(new ValueDatapoint<>(timestamp, point.getPriceAmount())); + } + }); } } @@ -558,12 +557,9 @@ protected void addVariableBlockDatapoints( Duration resolution, long nowMillis, List> values) { - // Validated, sorted points include price-less boundaries so prices never extend through them. + // Expand unavailable blocks as nulls too, replacing any previously stored prices. for (int i = 0; i < points.size(); i++) { PublicationMarketDocument.Point point = points.get(i); - if (point.getPriceAmount() == null) { - continue; - } Instant blockStart = start.plus(resolution.multipliedBy(point.getPosition() - 1L)); Instant blockEnd = i + 1 < points.size() diff --git a/entsoe/src/test/groovy/org/openremote/extension/entsoe/agent/protocol/EntsoeCurveTest.groovy b/entsoe/src/test/groovy/org/openremote/extension/entsoe/agent/protocol/EntsoeCurveTest.groovy index df4509c..dcc8cc6 100644 --- a/entsoe/src/test/groovy/org/openremote/extension/entsoe/agent/protocol/EntsoeCurveTest.groovy +++ b/entsoe/src/test/groovy/org/openremote/extension/entsoe/agent/protocol/EntsoeCurveTest.groovy @@ -108,9 +108,9 @@ class EntsoeCurveTest extends Specification { values.collect { it.value } == [80G, 80G, 90G, 90G] } - def "a price-less A03 point stops the preceding price block"() { + def "#curveType emits unavailable intervals for price-less points: #description"() { given: - def document = document("A03", period(START, START.plusSeconds(4500), "PT15M", [[1, 80], [3, null], [5, 90]])) + def document = document(curveType, period(START, START.plusSeconds(3600), "PT15M", points)) when: def values = protocol.buildPredictedDatapoints(document) @@ -118,10 +118,38 @@ class EntsoeCurveTest extends Specification { then: values.collect { it.timestamp - } == [0, 900, 3600].collect { - START.plusSeconds(it).toEpochMilli() + } == (0..3).collect { + START.plusSeconds(it * 900L).toEpochMilli() } - values.collect { it.value } == [80G, 80G, 90G] + values.collect { it.value } == expected + + where: + curveType | description | points | expected + "A01" | "middle" | [[1, 80], [2, null], [3, null], [4, 90]] | [80G, null, null, 90G] + null | "default curve" | [[1, 80], [2, null], [3, null], [4, 90]] | [80G, null, null, 90G] + "A01" | "whole period" | [[1, null], [2, null], [3, null], [4, null]] | [null, null, null, null] + "A03" | "middle block" | [[1, 80], [2, null], [4, 90]] | [80G, null, null, 90G] + "A03" | "first block" | [[1, null], [3, 90]] | [null, null, 90G, 90G] + "A03" | "final block" | [[1, 80], [3, null]] | [80G, 80G, null, null] + "A03" | "whole period" | [[1, null]] | [null, null, null, null] + } + + def "#curveType filters past unavailable intervals"() { + given: + setNow(START.plusSeconds(1200)) + def document = document(curveType, period(START, START.plusSeconds(3600), "PT15M", points)) + + when: + def values = protocol.buildPredictedDatapoints(document) + + then: + values.collect { it.timestamp } == [1800, 2700].collect { START.plusSeconds(it).toEpochMilli() } + values.collect { it.value } == [null, null] + + where: + curveType | points + "A01" | [[1, null], [2, null], [3, null], [4, null]] + "A03" | [[1, null]] } def "complete fixed-size periods support #curveType at #resolution resolution"() { diff --git a/entsoe/src/test/groovy/org/openremote/extension/entsoe/agent/protocol/EntsoeProtocolTest.groovy b/entsoe/src/test/groovy/org/openremote/extension/entsoe/agent/protocol/EntsoeProtocolTest.groovy index efd315b..a833985 100644 --- a/entsoe/src/test/groovy/org/openremote/extension/entsoe/agent/protocol/EntsoeProtocolTest.groovy +++ b/entsoe/src/test/groovy/org/openremote/extension/entsoe/agent/protocol/EntsoeProtocolTest.groovy @@ -23,6 +23,7 @@ import jakarta.ws.rs.client.ClientRequestContext import jakarta.ws.rs.client.ClientRequestFilter import jakarta.ws.rs.core.MediaType import jakarta.ws.rs.core.Response +import org.openremote.container.timer.TimerService import org.openremote.manager.agent.AgentService import org.openremote.manager.asset.AssetStorageService import org.openremote.manager.datapoint.AssetPredictedDatapointService @@ -786,6 +787,83 @@ class EntsoeProtocolTest extends Specification implements ManagerContainerTrait closeClient() } + def "#curveType unavailable prices replace stored predictions and can become available again"() { + given: "a real prediction store and a protocol with a fixed clock" + def container = startContainer(defaultConfig(), defaultServices()) + setPseudoClock(BEFORE_DATASET_START) + def assetStorageService = container.getService(AssetStorageService.class) + def predictionService = container.getService(AssetPredictedDatapointService.class) + def protocol = new EntsoeProtocol(new EntsoeAgent()) + protocol.timerService = container.getService(TimerService.class) + protocol.predictedDatapointService = predictionService + def attribute = new Attribute<>("energyPrice", NUMBER) + def asset = assetStorageService.merge(new ThingAsset("Unavailable Price Asset") + .setRealm(MASTER_REALM).addOrReplaceAttributes(attribute)) + def ref = new AttributeRef(asset.id, attribute.name) + def linkedAttributes = [new AbstractMap.SimpleEntry<>(ref, attribute)] + def start = Instant.parse(DATASET_START) + def timestamp = { long offset -> start.plusSeconds(offset).toEpochMilli() } + def document = { List points -> + def pointXml = points.collect { point -> + "${point[0]}" + + (point[1] == null ? "" : "${point[1]}") + "" + }.join("") + def xml = """ + + ${curveType} + ${start}${start.plusSeconds(3600)} + PT15M${pointXml} + + + """ + protocol.unmarshalPublicationMarketDocument(xml) + } + def snapshot = { + predictionService.getDatapoints(ref).sort { it.timestamp }.collect { + [it.timestamp, it.value as BigDecimal] + } + } + def initialPoints = [[1, 80], [2, 81], [3, 82], [4, 83]] + predictionService.updateValues(ref.id, ref.name, [ + new ValueDatapoint<>(timestamp(-900), 70G), + new ValueDatapoint<>(timestamp(3600), 100G) + ]) + protocol.applyPricingInformation("test-zone", document(initialPoints), linkedAttributes) + def expectedTimes = [-900, 0, 900, 1800, 2700, 3600].collect { timestamp(it) } + + expect: "the initial prices are persisted" + snapshot() == [expectedTimes, [70G, 80G, 81G, 82G, 83G, 100G]].transpose() + + when: "a subsequent publication explicitly marks middle intervals unavailable" + protocol.applyPricingInformation("test-zone", document(unavailablePoints), linkedAttributes) + + then: "null values replace stale prices, preserving values outside the period" + snapshot() == [expectedTimes, [70G, 80G, null, null, 90G, 100G]].transpose() + + when: "the whole period becomes unavailable" + protocol.applyPricingInformation("test-zone", document(allUnavailablePoints), linkedAttributes) + + then: "all its prices are cleared, including when there are no available prices to write" + snapshot() == [expectedTimes, [70G, null, null, null, null, 100G]].transpose() + + when: "prices become available again" + protocol.applyPricingInformation("test-zone", document(initialPoints), linkedAttributes) + + then: "prices replace the unavailable values" + snapshot() == [expectedTimes, [70G, 80G, 81G, 82G, 83G, 100G]].transpose() + + cleanup: + if (asset?.id) { + assetStorageService.delete([asset.id]) + } + closeClient() + + where: + curveType | unavailablePoints | allUnavailablePoints + "A01" | [[1, 80], [2, null], [3, null], [4, 90]] | [[1, null], [2, null], [3, null], [4, null]] + "A03" | [[1, 80], [2, null], [4, 90]] | [[1, null]] + } + def "ENTSO-E integration test handles no-data acknowledgement response cleanly"() { given: "the container environment is started" requestCountByZone.clear() From 040b4796f4a5f979563655ca64ef05df1ad904c7 Mon Sep 17 00:00:00 2001 From: Eric Bariaux <375613+ebariaux@users.noreply.github.com> Date: Wed, 28 Oct 2026 11:44:42 +0100 Subject: [PATCH 6/9] Fix ENTSO-E prediction handling for gaps between periods --- entsoe/README.md | 20 ++- .../entsoe/agent/protocol/EntsoeProtocol.java | 78 ++++++++- .../protocol/PublicationMarketDocument.java | 6 +- .../agent/protocol/EntsoeCurveTest.groovy | 18 +++ .../agent/protocol/EntsoeProtocolTest.groovy | 148 ++++++++++++++++++ 5 files changed, 262 insertions(+), 8 deletions(-) diff --git a/entsoe/README.md b/entsoe/README.md index 65cb7e9..86b1310 100644 --- a/entsoe/README.md +++ b/entsoe/README.md @@ -49,8 +49,24 @@ When a point in an otherwise valid returned period has no `price.amount`: For example, if a stored prediction at `12:15` has price `80` and a later publication explicitly marks that interval unavailable, the stored prediction becomes `(12:15, null)`. If no prediction exists at that timestamp, a null-valued entry is inserted. A later publication with an available price can replace the null value. -Reading predicted datapoints still returns these timestamped null entries. Consumers must treat `null` as **unavailable**, not as zero, and must not carry a previous price forward through those intervals. Code that expects every prediction to contain a number must handle null values explicitly. +Reading predicted datapoints with the `all` query retains these timestamped null entries. In JSON responses, an unavailable entry has an `x` timestamp and an omitted `y` property. LTTB downsampling omits unavailable entries entirely, so it does not preserve gap markers. Consumers must treat `null` as **unavailable**, not as zero, and must not carry a previous price forward through those intervals. Code that expects every prediction to contain a number must handle null values explicitly. Only intervals whose start timestamp is at or after the current time are updated. Historical predictions are left unchanged. -This behavior applies to explicitly present points without a price. An omitted position in `A03` continues the preceding block; it does not by itself mean unavailable. Incomplete `A01` periods are rejected. Fetch failures, no-data acknowledgements, and rejected periods do not clear existing predictions. Gaps between separate returned periods currently produce no updates, so previously stored predictions in those gaps can remain; clearing those stale values is a pending improvement. +This behavior applies to explicitly present points without a price. An omitted position in `A03` continues the preceding block; it does not by itself mean unavailable. Incomplete `A01` periods are rejected. Fetch failures, no-data acknowledgements, and rejected periods do not clear existing predictions. + +A present `price.amount` must contain a valid number. Malformed amounts (including empty or whitespace-only elements) reject the entire period before any of its predictions are updated; they do not mean unavailable. Other valid periods still update their predictions, but gap clearing is disabled for the whole response. + +### Gaps between returned periods + +When every returned series and period passes validation, the agent also sets existing future predictions in gaps between periods to `null`. These entries retain their timestamps; they are not deleted. Periods are considered together across all returned series, so an interval covered by another series is not a gap. + +A gap starts at the preceding covered period's exclusive end and stops at the next covered period's start. Since the gap has no declared resolution, the agent updates only timestamps already stored there, including timestamps off the returned periods' resolution grids. It does not create new predictions in an empty gap. Historical entries and predictions before the first or at/after the last returned period's end are left unchanged. + +If any series or period is rejected (including unsupported curve types, malformed or incomplete periods, and empty series or periods), gap clearing is skipped for the whole response. Valid periods still update their own predictions. This prevents rejected data from being mistaken for an unavailable interval. + +This gap handling does not remove old timestamps inside a covered period when its resolution changes. + +### OpenRemote compatibility + +Gap handling requires OpenRemote `1.33.0` or later, which includes JSON-null support in the core datapoint queries. The agent uses the core `AssetDatapointAllQuery` to read existing predictions in gaps, including entries already set to null by an earlier poll. No extension-specific SQL workaround is needed. diff --git a/entsoe/src/main/java/org/openremote/extension/entsoe/agent/protocol/EntsoeProtocol.java b/entsoe/src/main/java/org/openremote/extension/entsoe/agent/protocol/EntsoeProtocol.java index f8587f1..f96c76d 100644 --- a/entsoe/src/main/java/org/openremote/extension/entsoe/agent/protocol/EntsoeProtocol.java +++ b/entsoe/src/main/java/org/openremote/extension/entsoe/agent/protocol/EntsoeProtocol.java @@ -57,6 +57,7 @@ import org.openremote.model.attribute.AttributeEvent; import org.openremote.model.attribute.AttributeRef; import org.openremote.model.datapoint.ValueDatapoint; +import org.openremote.model.datapoint.query.AssetDatapointAllQuery; import org.openremote.model.syslog.SyslogCategory; public class EntsoeProtocol extends AbstractProtocol { @@ -237,7 +238,8 @@ protected void applyPricingInformation( return; } - List> predictedDatapoints = buildPredictedDatapoints(document); + DecodedPublication decoded = decodePublication(document); + List> predictedDatapoints = decoded.values(); if (predictedDatapoints.isEmpty()) { linkedAttributesForZone.forEach( entry -> @@ -255,8 +257,28 @@ protected void applyPricingInformation( AttributeRef attributeRef = entry.getKey(); Attribute attribute = entry.getValue(); LOG.fine("Updating pricing information data for attribute " + attribute.getName()); + List> updates = new ArrayList<>(predictedDatapoints); + if (!decoded.gaps().isEmpty()) { + long nowMillis = timerService.getCurrentTimeMillis(); + // Gaps have no declared resolution. Invalidate existing timestamps without inventing a + // grid. + for (TimeRange gap : decoded.gaps()) { + long from = Math.max(nowMillis, gap.start()); + if (from >= gap.end()) { + continue; + } + predictedDatapointService + .queryDatapoints( + attributeRef.getId(), + attributeRef.getName(), + new AssetDatapointAllQuery(from, gap.end() - 1)) + .stream() + .filter(point -> point.getValue() != null) + .forEach(point -> updates.add(new ValueDatapoint<>(point.getTimestamp(), null))); + } + } predictedDatapointService.updateValues( - attributeRef.getId(), attributeRef.getName(), predictedDatapoints); + attributeRef.getId(), attributeRef.getName(), updates); }); } @@ -439,10 +461,16 @@ protected EntsoeXmlMeta(String rootElement, String reasonText) { } protected List> buildPredictedDatapoints(PublicationMarketDocument document) { + return decodePublication(document).values(); + } + + private DecodedPublication decodePublication(PublicationMarketDocument document) { List> values = new ArrayList<>(); + List coverage = new ArrayList<>(); + boolean complete = true; long nowMillis = timerService.getCurrentTimeMillis(); if (document.getTimeSeries() == null || document.getTimeSeries().isEmpty()) { - return values; + return new DecodedPublication(values, List.of()); } for (PublicationMarketDocument.TimeSeries timeSeries : document.getTimeSeries()) { @@ -450,15 +478,18 @@ protected List> buildPredictedDatapoints(PublicationMarketDocu if (!"A01".equals(curveType) && !"A03".equals(curveType)) { LOG.warning( "Skipping ENTSO-E time series with unsupported curve type: '" + curveType + "'"); + complete = false; continue; } if (timeSeries.getPeriods() == null || timeSeries.getPeriods().isEmpty()) { + complete = false; continue; } for (PublicationMarketDocument.Period period : timeSeries.getPeriods()) { if (period == null || period.getPoints() == null || period.getPoints().isEmpty()) { + complete = false; continue; } @@ -468,6 +499,7 @@ protected List> buildPredictedDatapoints(PublicationMarketDocu || timeInterval.getEnd() == null || period.getResolution() == null) { LOG.warning("Skipping ENTSO-E period without its own start, end, or resolution"); + complete = false; continue; } @@ -481,6 +513,7 @@ protected List> buildPredictedDatapoints(PublicationMarketDocu resolution = Duration.parse(period.getResolution()); if (!end.isAfter(start) || resolution.isZero() || resolution.isNegative()) { LOG.warning("Skipping ENTSO-E period with non-positive duration or resolution"); + complete = false; continue; } Duration duration = Duration.between(start, end); @@ -488,16 +521,31 @@ protected List> buildPredictedDatapoints(PublicationMarketDocu if (intervalCount == 0 || !resolution.multipliedBy(intervalCount).equals(duration)) { LOG.warning( "Skipping ENTSO-E period whose duration is not a multiple of its resolution"); + complete = false; continue; } } catch (Exception e) { LOG.log(Level.WARNING, e, () -> "Could not parse ENTSO-E timeseries time data"); + complete = false; continue; } if (!hasValidPeriodPositions(period.getPoints(), curveType, intervalCount)) { + complete = false; + continue; + } + // Validate every price before recording coverage or emitting any part of the period. + // Only an absent amount means unavailable; malformed amounts reject the period. + try { + for (PublicationMarketDocument.Point point : period.getPoints()) { + point.getPriceAmount(); + } + } catch (NumberFormatException e) { + LOG.warning("Skipping ENTSO-E period with malformed price amount"); + complete = false; continue; } + coverage.add(new TimeRange(start.toEpochMilli(), end.toEpochMilli())); List points = period.getPoints().stream() .sorted(Comparator.comparingInt(PublicationMarketDocument.Point::getPosition)) @@ -521,7 +569,29 @@ protected List> buildPredictedDatapoints(PublicationMarketDocu } values.sort(Comparator.comparingLong(ValueDatapoint::getTimestamp)); - return values; + return new DecodedPublication(values, complete ? findPeriodGaps(coverage) : List.of()); + } + + private record TimeRange(long start, long end) {} + + private record DecodedPublication(List> values, List gaps) {} + + private List findPeriodGaps(List coverage) { + List gaps = new ArrayList<>(); + coverage.sort(Comparator.comparingLong(TimeRange::start)); + if (coverage.isEmpty()) { + return gaps; + } + long coveredEnd = coverage.get(0).end(); + for (int i = 1; i < coverage.size(); i++) { + TimeRange interval = coverage.get(i); + if (interval.start() > coveredEnd) { + gaps.add(new TimeRange(coveredEnd, interval.start())); + } + // Include coverage from every series, merging adjacent and overlapping periods. + coveredEnd = Math.max(coveredEnd, interval.end()); + } + return gaps; } protected boolean hasValidPeriodPositions( diff --git a/entsoe/src/main/java/org/openremote/extension/entsoe/agent/protocol/PublicationMarketDocument.java b/entsoe/src/main/java/org/openremote/extension/entsoe/agent/protocol/PublicationMarketDocument.java index 5042024..16f2971 100644 --- a/entsoe/src/main/java/org/openremote/extension/entsoe/agent/protocol/PublicationMarketDocument.java +++ b/entsoe/src/main/java/org/openremote/extension/entsoe/agent/protocol/PublicationMarketDocument.java @@ -105,14 +105,16 @@ public static class Point { private Integer position; @XmlElement(name = "price.amount", namespace = NS) - private BigDecimal priceAmount; + // Preserve malformed text: JAXB's BigDecimal binding can silently turn it into null, + // making an invalid amount indistinguishable from an explicitly missing price. + private String priceAmount; public Integer getPosition() { return position; } public BigDecimal getPriceAmount() { - return priceAmount; + return priceAmount == null ? null : new BigDecimal(priceAmount.trim()); } } diff --git a/entsoe/src/test/groovy/org/openremote/extension/entsoe/agent/protocol/EntsoeCurveTest.groovy b/entsoe/src/test/groovy/org/openremote/extension/entsoe/agent/protocol/EntsoeCurveTest.groovy index dcc8cc6..91b4c31 100644 --- a/entsoe/src/test/groovy/org/openremote/extension/entsoe/agent/protocol/EntsoeCurveTest.groovy +++ b/entsoe/src/test/groovy/org/openremote/extension/entsoe/agent/protocol/EntsoeCurveTest.groovy @@ -134,6 +134,24 @@ class EntsoeCurveTest extends Specification { "A03" | "whole period" | [[1, null]] | [null, null, null, null] } + def "#curveType rejects a period containing malformed price '#amount' without partial updates"() { + given: + def points = curveType == "A01" ? [[1, 80], [2, amount], [3, 82], [4, 90]] : [[1, 80], [2, amount], [4, 90]] + def document = document(curveType, + period(START, START.plusSeconds(3600), "PT15M", points) + + period(START.plusSeconds(3600), START.plusSeconds(4500), "PT15M", [[1, 100]])) + + when: + def values = protocol.buildPredictedDatapoints(document) + + then: "the entire malformed period is rejected while the valid period is still decoded" + values.collect { it.timestamp } == [START.plusSeconds(3600).toEpochMilli()] + values.collect { it.value } == [100G] + + where: + [curveType, amount] << [["A01", "A03"], ["broken", "", " ", "NaN"]].combinations() + } + def "#curveType filters past unavailable intervals"() { given: setNow(START.plusSeconds(1200)) diff --git a/entsoe/src/test/groovy/org/openremote/extension/entsoe/agent/protocol/EntsoeProtocolTest.groovy b/entsoe/src/test/groovy/org/openremote/extension/entsoe/agent/protocol/EntsoeProtocolTest.groovy index 79e30ad..9cdaa0f 100644 --- a/entsoe/src/test/groovy/org/openremote/extension/entsoe/agent/protocol/EntsoeProtocolTest.groovy +++ b/entsoe/src/test/groovy/org/openremote/extension/entsoe/agent/protocol/EntsoeProtocolTest.groovy @@ -33,6 +33,7 @@ import org.openremote.model.attribute.Attribute import org.openremote.model.attribute.AttributeRef import org.openremote.model.attribute.MetaItem import org.openremote.model.datapoint.ValueDatapoint +import org.openremote.model.datapoint.query.AssetDatapointAllQuery import org.openremote.test.ManagerContainerTrait import spock.lang.Ignore import spock.lang.Issue @@ -864,6 +865,153 @@ class EntsoeProtocolTest extends Specification implements ManagerContainerTrait "A03" | [[1, 80], [2, null], [4, 90]] | [[1, null]] } + def "core numeric prediction query preserves null, zero, and negative prices"() { + given: + def container = startContainer(defaultConfig(), defaultServices()) + setPseudoClock(BEFORE_DATASET_START) + def assetStorageService = container.getService(AssetStorageService.class) + def predictionService = container.getService(AssetPredictedDatapointService.class) + def asset = assetStorageService.merge(new ThingAsset("Null Query Asset") + .setRealm(MASTER_REALM).addOrReplaceAttributes(new Attribute<>("energyPrice", NUMBER))) + def start = Instant.parse(DATASET_START).toEpochMilli() + def values = [-5G, 0G, null, 90G] + def timestamps = (0..3).collect { start + it * 900_000L } + predictionService.updateValues(asset.id, "energyPrice", (0..3).collect { + new ValueDatapoint<>(timestamps[it], values[it]) + }) + + when: + def result = predictionService.queryDatapoints(asset.id, "energyPrice", + new AssetDatapointAllQuery(timestamps.first(), timestamps.last())).sort { + it.timestamp + } + + then: + result.collect { it.timestamp } == timestamps + result.collect { it.value as BigDecimal } == values + + cleanup: + if (asset?.id) { + assetStorageService.delete([asset.id]) + } + } + + def "#curveType updates stored predictions in period gaps: #scenario"() { + given: "stored predictions including timestamps off the returned resolution grids" + def container = startContainer(defaultConfig(), defaultServices()) + def start = Instant.parse(DATASET_START) + setPseudoClock(start.plusSeconds(nowOffset).toString()) + def assetStorageService = container.getService(AssetStorageService.class) + def predictionService = container.getService(AssetPredictedDatapointService.class) + def protocol = new EntsoeProtocol(new EntsoeAgent()) + protocol.timerService = Stub(TimerService) { + getCurrentTimeMillis() >> start.plusSeconds(nowOffset).toEpochMilli() + } + protocol.predictedDatapointService = predictionService + def attribute = new Attribute<>("energyPrice", NUMBER) + def asset = assetStorageService.merge(new ThingAsset("Period Gap Asset") + .setRealm(MASTER_REALM).addOrReplaceAttributes(attribute)) + def ref = new AttributeRef(asset.id, attribute.name) + def timestamp = { long offset -> start.plusSeconds(offset).toEpochMilli() } + def offsets = [-900, 0, 900, 1200, 1800, 2700, 3600, 4500] + if (scenario == "no stored predictions in gap") { + offsets = offsets.findAll { it < 900 || it >= 2700 } + } + predictionService.updateValues(ref.id, ref.name, offsets.collect { + new ValueDatapoint<>(timestamp(it), 80G) + }) + def period = { long from, long to, String resolution, int price -> + """ + ${start.plusSeconds(from)}${start.plusSeconds(to)} + ${resolution}1${price} + """ + } + def periods = [period(0, 900, "PT15M", 90), period(2700, 4500, "PT30M", 100)] + if (scenario == "reversed periods") { + periods = periods.reverse() + } + def extraSeries = "" + switch (scenario) { + case "another series covers the gap": + extraSeries = "A03${period(900, 2700, "PT30M", 95)}" + break + case "incomplete A01 period": + extraSeries = "A01${period(900, 2700, "PT15M", 999)}" + break + case "malformed price": + periods.add(period(900, 2700, "PT15M", 999).replace("", + "2broken")) + break + case "invalid timing": + periods.add(period(900, 1800, "invalid", 999)) + break + case "empty period": + periods.add("") + break + case "unsupported series": + extraSeries = "A99${period(900, 2700, "PT30M", 999)}" + break + case "empty series": + extraSeries = "A03" + break + } + def xml = """ + + ${curveType}${periods.join("")}${extraSeries} + + """ + def expected = offsets.collectEntries { [(timestamp(it)): 80G] } + if (nowOffset <= 0) { + expected[timestamp(0)] = 90G + } + expected[timestamp(2700)] = 100G + if (scenario == "another series covers the gap") { + expected[timestamp(900)] = 95G + } + clearedOffsets.each { expected[timestamp(it)] = null } + + when: "the publication is applied to the prediction store" + protocol.applyPricingInformation("test-zone", protocol.unmarshalPublicationMarketDocument(xml), + [new AbstractMap.SimpleEntry<>(ref, attribute)]) + + then: "only future predictions in confirmed gaps become null and all entries remain" + def stored = predictionService.getDatapoints(ref) + stored.size() == offsets.size() + stored.collectEntries { [(it.timestamp): it.value as BigDecimal] } == expected + + when: "the same publication is applied again, including already-null gap entries" + protocol.applyPricingInformation("test-zone", protocol.unmarshalPublicationMarketDocument(xml), + [new AbstractMap.SimpleEntry<>(ref, attribute)]) + + then: "the stored predictions remain unchanged" + predictionService.getDatapoints(ref).collectEntries { + [(it.timestamp): it.value as BigDecimal] + } == expected + + cleanup: + if (asset?.id) { + assetStorageService.delete([asset.id]) + } + closeClient() + + where: + [curveType, scenario, nowOffset, clearedOffsets] << ["A01", "A03"].collectMany { type -> + [ + [type, "separate periods", -900, [900, 1200, 1800]], + [type, "reversed periods", -900, [900, 1200, 1800]], + [type, "partly historical gap", 1200, [1200, 1800]], + [type, "another series covers the gap", -900, []], + [type, "no stored predictions in gap", -900, []], + [type, "incomplete A01 period", -900, []], + [type, "malformed price", -900, []], + [type, "invalid timing", -900, []], + [type, "empty period", -900, []], + [type, "unsupported series", -900, []], + [type, "empty series", -900, []] + ] + } + } + def "ENTSO-E integration test handles no-data acknowledgement response cleanly"() { given: "the container environment is started" requestCountByZone.clear() From 24705aba5180bcb9590d1f80d624365927f09a54 Mon Sep 17 00:00:00 2001 From: Eric Bariaux <375613+ebariaux@users.noreply.github.com> Date: Wed, 28 Oct 2026 11:55:50 +0100 Subject: [PATCH 7/9] Fix EMS optimisation for unavailable tariff forecasts Exclude windows containing null tariffs while preserving timestamp alignment in both EMS implementations. Add regression coverage and document the ENTSO-E consumer migration requirements. --- .../optimisationMethods/EmsOptimisation.md | 6 + .../optimisationMethods/EmsOptimisation.java | 26 +++-- .../EmsOptimisationBeta.java | 26 +++-- .../EmsTariffNullTest.groovy | 104 ++++++++++++++++++ entsoe/README.md | 2 + entsoe/RELEASE_NOTES.txt | 39 +++++++ 6 files changed, 185 insertions(+), 18 deletions(-) create mode 100644 ems/src/test/groovy/org/openremote/extension/ems/manager/optimisationMethods/EmsTariffNullTest.groovy create mode 100644 entsoe/RELEASE_NOTES.txt diff --git a/ems/docs/optimisationMethods/EmsOptimisation.md b/ems/docs/optimisationMethods/EmsOptimisation.md index 2a73a37..b9f8428 100644 --- a/ems/docs/optimisationMethods/EmsOptimisation.md +++ b/ems/docs/optimisationMethods/EmsOptimisation.md @@ -1 +1,7 @@ Work in progress + +## Unavailable tariff forecasts + +Both `EmsOptimisation` and `EmsOptimisationBeta` treat null tariff values as unavailable. A moving-average window is eligible for tariff-based charging or discharging only when all its prices are available (the default window contains eight datapoints). Windows containing nulls are excluded; available windows before or after them can still be selected at their original timestamps. + +Import and export forecasts are evaluated independently. If no complete import window exists for a day, no tariff-based charging zone is selected for it; the same applies to export windows and discharging. If neither has a complete window, tariff-based zone selection produces no zones for that day. Other optimisation logic continues. Missing prices are not replaced with zero, interpolated, or carried forward. diff --git a/ems/src/main/java/org/openremote/extension/ems/manager/optimisationMethods/EmsOptimisation.java b/ems/src/main/java/org/openremote/extension/ems/manager/optimisationMethods/EmsOptimisation.java index c3310c5..ce30169 100644 --- a/ems/src/main/java/org/openremote/extension/ems/manager/optimisationMethods/EmsOptimisation.java +++ b/ems/src/main/java/org/openremote/extension/ems/manager/optimisationMethods/EmsOptimisation.java @@ -2154,6 +2154,7 @@ private Map calculateTariffChargeAndDischargeZones( Map tariffImportDailyMinimumMap = IntStream.range(0, tariffImportMovingAverage.size()) + .filter(i -> tariffImportMovingAverage.get(i).getValue() != null) .mapToObj( i -> { ValueDatapoint dp = tariffImportMovingAverage.get(i); @@ -2167,6 +2168,7 @@ private Map calculateTariffChargeAndDischargeZones( Map tariffExportDailyMinimumMap = IntStream.range(0, tariffExportMovingAverage.size()) + .filter(i -> tariffExportMovingAverage.get(i).getValue() != null) .mapToObj( i -> { ValueDatapoint dp = tariffExportMovingAverage.get(i); @@ -2397,24 +2399,30 @@ public List> movingAverage(List> dataPoints, } double sum = 0.0; + int unavailableCount = 0; for (int i = 0; i < dataPoints.size(); i++) { Double value = (Double) dataPoints.get(i).getValue(); - sum += value; + if (value == null) { + unavailableCount++; + } else { + sum += value; + } if (i >= window) { Double valueOld = (Double) dataPoints.get(i - window).getValue(); - sum -= valueOld; + if (valueOld == null) { + unavailableCount--; + } else { + sum -= valueOld; + } } if (i >= window - 1) { - double avg = sum / window; - - ValueDatapoint original = dataPoints.get(i); - - ValueDatapoint averaged = - new ValueDatapoint<>(original.getTimestamp(), round(avg, 7)); - result.add(averaged); + // Keep unavailable windows in the result so indices still map to the input tariffs. + // A partial window must not be used to select a charge or discharge zone. + Double avg = unavailableCount == 0 ? round(sum / window, 7) : null; + result.add(new ValueDatapoint<>(dataPoints.get(i).getTimestamp(), avg)); } } diff --git a/ems/src/main/java/org/openremote/extension/ems/manager/optimisationMethods/EmsOptimisationBeta.java b/ems/src/main/java/org/openremote/extension/ems/manager/optimisationMethods/EmsOptimisationBeta.java index 783c960..6f46d9d 100644 --- a/ems/src/main/java/org/openremote/extension/ems/manager/optimisationMethods/EmsOptimisationBeta.java +++ b/ems/src/main/java/org/openremote/extension/ems/manager/optimisationMethods/EmsOptimisationBeta.java @@ -2204,6 +2204,7 @@ private Map calculateTariffChargeAndDischargeZones( Map tariffImportDailyMinimumMap = IntStream.range(0, tariffImportMovingAverage.size()) + .filter(i -> tariffImportMovingAverage.get(i).getValue() != null) .mapToObj( i -> { ValueDatapoint dp = tariffImportMovingAverage.get(i); @@ -2217,6 +2218,7 @@ private Map calculateTariffChargeAndDischargeZones( Map tariffExportDailyMinimumMap = IntStream.range(0, tariffExportMovingAverage.size()) + .filter(i -> tariffExportMovingAverage.get(i).getValue() != null) .mapToObj( i -> { ValueDatapoint dp = tariffExportMovingAverage.get(i); @@ -2447,24 +2449,30 @@ public List> movingAverage(List> dataPoints, } double sum = 0.0; + int unavailableCount = 0; for (int i = 0; i < dataPoints.size(); i++) { Double value = (Double) dataPoints.get(i).getValue(); - sum += value; + if (value == null) { + unavailableCount++; + } else { + sum += value; + } if (i >= window) { Double valueOld = (Double) dataPoints.get(i - window).getValue(); - sum -= valueOld; + if (valueOld == null) { + unavailableCount--; + } else { + sum -= valueOld; + } } if (i >= window - 1) { - double avg = sum / window; - - ValueDatapoint original = dataPoints.get(i); - - ValueDatapoint averaged = - new ValueDatapoint<>(original.getTimestamp(), round(avg, 7)); - result.add(averaged); + // Keep unavailable windows in the result so indices still map to the input tariffs. + // A partial window must not be used to select a charge or discharge zone. + Double avg = unavailableCount == 0 ? round(sum / window, 7) : null; + result.add(new ValueDatapoint<>(dataPoints.get(i).getTimestamp(), avg)); } } diff --git a/ems/src/test/groovy/org/openremote/extension/ems/manager/optimisationMethods/EmsTariffNullTest.groovy b/ems/src/test/groovy/org/openremote/extension/ems/manager/optimisationMethods/EmsTariffNullTest.groovy new file mode 100644 index 0000000..36376d8 --- /dev/null +++ b/ems/src/test/groovy/org/openremote/extension/ems/manager/optimisationMethods/EmsTariffNullTest.groovy @@ -0,0 +1,104 @@ +/* + * Copyright 2026, OpenRemote Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +package org.openremote.extension.ems.manager.optimisationMethods + +import org.openremote.model.datapoint.ValueDatapoint +import spock.lang.Specification + +import java.time.LocalDate +import java.time.ZoneId + +class EmsTariffNullTest extends Specification { + + private static final long START = LocalDate.of(2026, 10, 28).atStartOfDay(ZoneId.systemDefault()).toInstant().toEpochMilli() + + def "#implementation.simpleName moving average preserves unavailable windows: #scenario"() { + given: + def optimisation = implementation.getDeclaredConstructor().newInstance() + def input = datapoints(values) + + when: + def result = optimisation.movingAverage(input, window) + + then: + result.collect { it.value } == expected + result.collect { it.timestamp } == input.drop(window - 1).collect { it.timestamp } + + where: + [implementation, scenario, values, window, expected] << [EmsOptimisation, EmsOptimisationBeta].collectMany { type -> + [ + [type, "negative and zero prices", [-4, 0, 4, 8], 2, [-2d, 2d, 6d]], + [type, "first unavailable", [null, 2, 4, 6], 2, [null, 3d, 5d]], + [type, "middle unavailable", [2, 4, null, 6, 8], 2, [3d, null, null, 7d]], + [type, "last unavailable", [2, 4, 6, null], 2, [3d, 5d, null]], + [type, "consecutive unavailable", [null, null, 2, 4], 2, [null, null, 3d]], + [type, "all unavailable", [null, null, null], 2, [null, null]], + [type, "single point windows", [2, null, 4], 1, [2d, null, 4d]], + [type, "default eight point window", [1, 2, 3, null, 5, 6, 7, 8], 8, [null]], + [type, "rounding", [0, 0, 1], 3, [0.3333333d]] + ] + } + } + + def "#implementation.simpleName tariff zones use only complete windows: #scenario"() { + given: + def optimisation = implementation.getDeclaredConstructor().newInstance() + def imports = datapoints(importValues) + def exports = datapoints(exportValues) + def expected = [:] + chargeIndices.each { expected[START + it * 900_000L] = 1 } + dischargeIndices.each { expected[START + it * 900_000L] = -1 } + + expect: + optimisation.calculateTariffChargeAndDischargeZones(imports, exports, 8) == expected + + where: + [implementation, scenario, importValues, exportValues, chargeIndices, dischargeIndices] << + [EmsOptimisation, EmsOptimisationBeta].collectMany { type -> + [ + [ + type, + "valid windows after an import gap", + [null] + [-5] * 8 + [100] * 8, + [100] * 9 + [0] * 8, + (1..8), + (9..16) + ], + [ + type, + "valid windows after an export gap", + [100] * 9 + [0] * 8, + [null] + [-5] * 8 + [100] * 8, + (9..16), + (1..8) + ], + [type, "both unavailable", [null] * 8, [null] * 8, [], []], + [type, "import unavailable", [null] * 8, [10] * 8, [], (0..7)], + [type, "export unavailable", [10] * 8, [null] * 8, (0..7), []], + [type, "no complete window", [10] * 4 + [null] + [10] * 3, [null] * 8, [], []] + ] + } + } + + private static List> datapoints(List values) { + values.withIndex().collect { value, index -> + new ValueDatapoint<>(START + index * 900_000L, value == null ? null : value.doubleValue()) + } + } +} diff --git a/entsoe/README.md b/entsoe/README.md index 86b1310..439d99b 100644 --- a/entsoe/README.md +++ b/entsoe/README.md @@ -40,6 +40,8 @@ Make sure to also add the `Has predicted data points` configuration to your attr ### Unavailable prices and null predictions +**Upgrade note:** Review all forecast consumers for null handling before deploying this change, including custom rules and external integrations. See the [release notes and migration guidance](RELEASE_NOTES.txt). + **An unavailable price is stored as a predicted datapoint with a timestamp and a `null` value. The datapoint entry is not deleted.** When a point in an otherwise valid returned period has no `price.amount`: diff --git a/entsoe/RELEASE_NOTES.txt b/entsoe/RELEASE_NOTES.txt new file mode 100644 index 0000000..0d69263 --- /dev/null +++ b/entsoe/RELEASE_NOTES.txt @@ -0,0 +1,39 @@ +Unreleased — ENTSO-E gap handling and EMS tariff availability + +BREAKING BEHAVIOUR: predicted prices can contain timestamped null values + +ENTSO-E now writes null predictions for explicitly unavailable prices and clears +existing future predictions in validated gaps between returned periods to null. +These entries retain their timestamps; they are not deleted. This can replace a +previously available price. A later publication can restore a numeric price. + +This affects every consumer of these forecasts, including custom rules, EMS or +other extensions, external applications, and integrations outside this repository. +Consumers that assume every datapoint is numeric may fail during unboxing, +arithmetic, comparisons, aggregation, or interpolation. Updating OpenRemote core +alone does not make these consumers null-safe. Review them before deploying this +ENTSO-E update. + +Required consumer adaptations: +- Treat null as unavailable. Do not interpret it as zero or carry a previous price + through an explicitly unavailable interval. +- Preserve time alignment when handling gaps. Dropping null entries and then + treating the remaining entries as consecutive can bridge unavailable intervals + or assign decisions to the wrong timestamps. +- Define how unavailable intervals affect calculations. EMS now excludes any + tariff averaging window containing a null and continues with complete windows. + Import and export windows are evaluated independently; if neither has a complete + window for a day, there are no tariff-based zones for that day. Other optimisation + logic continues. Both EmsOptimisation and EmsOptimisationBeta include this fix. +- Handle absent JSON values as well as explicit nulls: the core 'all' query retains + the timestamp as 'x' but omits 'y' for an unavailable value. LTTB omits unavailable + points entirely and must not be used to detect or preserve all gap markers. +- Test unavailable values at the beginning, middle, and end of a forecast, all-null + forecasts, repeated polling, and recovery when prices become available again. + +Scope and compatibility: +- OpenRemote 1.33.0 or later is required for the core JSON-null query support. +- Malformed price amounts reject the entire affected period and disable gap + clearing for the response. They are not interpreted as unavailable prices. +- The EMS fix covers tariff averaging and zone selection in this repository. It + does not establish that custom rules or consumers in other projects are safe. From eb03136b6ceb4af70af90b1981b1f0073753f573 Mon Sep 17 00:00:00 2001 From: Eric Bariaux <375613+ebariaux@users.noreply.github.com> Date: Wed, 28 Oct 2026 14:29:40 +0100 Subject: [PATCH 8/9] Reenable ENTSO-E integration tests for period handling Restore mid-period filtering and multiple-period coverage, and verify every predicted timestamp and price including final A03 blocks. --- .../agent/protocol/EntsoeProtocolTest.groovy | 30 +++++++++---------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/entsoe/src/test/groovy/org/openremote/extension/entsoe/agent/protocol/EntsoeProtocolTest.groovy b/entsoe/src/test/groovy/org/openremote/extension/entsoe/agent/protocol/EntsoeProtocolTest.groovy index 9cdaa0f..111720e 100644 --- a/entsoe/src/test/groovy/org/openremote/extension/entsoe/agent/protocol/EntsoeProtocolTest.groovy +++ b/entsoe/src/test/groovy/org/openremote/extension/entsoe/agent/protocol/EntsoeProtocolTest.groovy @@ -35,7 +35,6 @@ import org.openremote.model.attribute.MetaItem import org.openremote.model.datapoint.ValueDatapoint import org.openremote.model.datapoint.query.AssetDatapointAllQuery import org.openremote.test.ManagerContainerTrait -import spock.lang.Ignore import spock.lang.Issue import spock.lang.Shared import spock.lang.Specification @@ -430,7 +429,6 @@ class EntsoeProtocolTest extends Specification implements ManagerContainerTrait closeClient() } - @Ignore def "ENTSO-E integration test filters out points in the past when clock is mid-period"() { given: "the container environment is started with clock in the middle of the dataset period" requestCountByZone.clear() @@ -491,10 +489,11 @@ class EntsoeProtocolTest extends Specification implements ManagerContainerTrait def start = Instant.parse(DATASET_START).toEpochMilli() def step = 15 * 60 * 1000L - assert datapoints[0].timestamp == start + (2 * step) - assert datapoints[1].timestamp == start + (3 * step) - assert (datapoints[0].value as BigDecimal).compareTo(65.84G) == 0 - assert (datapoints[1].value as BigDecimal).compareTo(65.05G) == 0 + // The current interval started before now; the forecast stops at the exclusive period end. + assert datapoints.collect { it.timestamp } == (2..<96).collect { start + it * step } + assert datapoints.collect { + it.value as BigDecimal + } == [65.84G] + Collections.nCopies(93, 65.05G) } cleanup: "remove created assets and mock client" @@ -1217,7 +1216,6 @@ class EntsoeProtocolTest extends Specification implements ManagerContainerTrait closeClient() } - @Ignore def "ENTSO-E integration test supports multiple periods in a single timeseries"() { given: "the container environment is started" requestCountByZone.clear() @@ -1279,15 +1277,15 @@ class EntsoeProtocolTest extends Specification implements ManagerContainerTrait def secondPeriodStart = Instant.parse("2026-02-17T00:00:00.000Z").toEpochMilli() def step = 15 * 60 * 1000L - assert datapoints[0].timestamp == firstPeriodStart - assert datapoints[1].timestamp == firstPeriodStart + step - assert datapoints[4].timestamp == secondPeriodStart - assert datapoints[5].timestamp == secondPeriodStart + step - - assert (datapoints[0].value as BigDecimal).compareTo(51.11G) == 0 - assert (datapoints[1].value as BigDecimal).compareTo(52.22G) == 0 - assert (datapoints[4].value as BigDecimal).compareTo(53.33G) == 0 - assert (datapoints[5].value as BigDecimal).compareTo(54.44G) == 0 + assert datapoints.collect { it.timestamp } == + (0..<4).collect { + firstPeriodStart + it * step + } + (0..<4).collect { + secondPeriodStart + it * step + } + // Each final A03 block extends to its own exclusive period end, then positions restart. + assert datapoints.collect { it.value as BigDecimal } == + [51.11G, 52.22G, 52.22G, 52.22G, 53.33G, 54.44G, 54.44G, 54.44G] } cleanup: "remove created assets and mock client" From d34fd677343717659e8fffaf5484bd360e7c19dc Mon Sep 17 00:00:00 2001 From: Eric Bariaux <375613+ebariaux@users.noreply.github.com> Date: Fri, 30 Oct 2026 09:54:52 +0100 Subject: [PATCH 9/9] Upgrade Spotless to 8.10.4 --- deployment/build.gradle | 12 +++---- .../ems/rules/EmsDemoSimulationRules.groovy | 4 +-- .../gopacs/GOPACSHandlerHttpTest.groovy | 14 ++++---- .../energy/EnergyOptimisationAssetTest.groovy | 8 ++--- .../energy/ForecastSolarServiceTest.groovy | 2 +- .../energy/ForecastWindServiceTest.groovy | 2 +- .../agent/protocol/EntsoeCurveTest.groovy | 2 +- .../agent/protocol/EntsoeProtocolTest.groovy | 32 +++++++++---------- gradle.properties | 2 +- .../manager/HawkbitFirmwareServiceTest.groovy | 4 +-- project.gradle | 10 +++--- settings.gradle | 6 ++-- spotless.gradle | 6 ++-- 13 files changed, 52 insertions(+), 52 deletions(-) diff --git a/deployment/build.gradle b/deployment/build.gradle index 4b699d9..7bdccd9 100644 --- a/deployment/build.gradle +++ b/deployment/build.gradle @@ -59,24 +59,24 @@ def extensionProjects = rootProject.subprojects.findAll { def configureRuntimeAttributes = { configuration -> configuration.attributes { attribute( - Usage.USAGE_ATTRIBUTE, - objects.named(Usage, Usage.JAVA_RUNTIME) + Usage.USAGE_ATTRIBUTE, + objects.named(Usage, Usage.JAVA_RUNTIME) ) attribute( Category.CATEGORY_ATTRIBUTE, objects.named(Category, Category.LIBRARY) - ) + ) attribute( LibraryElements.LIBRARY_ELEMENTS_ATTRIBUTE, objects.named(LibraryElements, LibraryElements.JAR) - ) + ) attribute( TargetJvmEnvironment.TARGET_JVM_ENVIRONMENT_ATTRIBUTE, objects.named( TargetJvmEnvironment, TargetJvmEnvironment.STANDARD_JVM ) - ) + ) } } @@ -113,7 +113,7 @@ tasks.register("distributeExtensions", DistributeExtensions) { managerFiles.from(configurations.managerRuntime) destinationDirectory.set( layout.buildDirectory.dir("extensions") - ) + ) } tasks.register("installDist") { diff --git a/ems/src/main/resources/ems/rules/EmsDemoSimulationRules.groovy b/ems/src/main/resources/ems/rules/EmsDemoSimulationRules.groovy index 38314c8..27aa4db 100644 --- a/ems/src/main/resources/ems/rules/EmsDemoSimulationRules.groovy +++ b/ems/src/main/resources/ems/rules/EmsDemoSimulationRules.groovy @@ -243,8 +243,8 @@ rules.add() .then({ facts -> // Get power set-point for each battery def powerSetpointBatteries = facts - .matchAssetState(new AssetQuery().parents(energyOptimisationAssetId).attributeNames("powerSetpoint")) - .toList() + .matchAssetState(new AssetQuery().parents(energyOptimisationAssetId).attributeNames("powerSetpoint")) + .toList() String[] attributeNames = ["energyCapacity", "energyLevel", "energyLevelPercentage"] diff --git a/ems/src/test/groovy/org/openremote/extension/ems/manager/gopacs/GOPACSHandlerHttpTest.groovy b/ems/src/test/groovy/org/openremote/extension/ems/manager/gopacs/GOPACSHandlerHttpTest.groovy index 456a02a..9f0cfee 100644 --- a/ems/src/test/groovy/org/openremote/extension/ems/manager/gopacs/GOPACSHandlerHttpTest.groovy +++ b/ems/src/test/groovy/org/openremote/extension/ems/manager/gopacs/GOPACSHandlerHttpTest.groovy @@ -190,7 +190,7 @@ class GOPACSHandlerHttpTest extends Specification implements ManagerContainerTra and: "verifying the signature fetched an OAuth2 token and resolved the DSO via the address book" conditions.eventually { wireMock.verify(postRequestedFor(urlPathEqualTo(OAUTH_PATH)) - .withRequestBody(containing("grant_type=client_credentials"))) + .withRequestBody(containing("grant_type=client_credentials"))) wireMock.verify(getRequestedFor(urlPathEqualTo(ADDRESS_BOOK_PATH)) .withHeader("Authorization", equalTo("Bearer " + ACCESS_TOKEN))) } @@ -198,8 +198,8 @@ class GOPACSHandlerHttpTest extends Specification implements ManagerContainerTra and: "the FlexRequestResponse and FlexOffer are actually POSTed to the broker as signed XML with a bearer token" conditions.eventually { wireMock.verify(postRequestedFor(urlPathEqualTo(BROKER_PATH)) - .withHeader("Authorization", equalTo("Bearer " + ACCESS_TOKEN)) - .withRequestBody(containing("SignedMessage"))) + .withHeader("Authorization", equalTo("Bearer " + ACCESS_TOKEN)) + .withRequestBody(containing("SignedMessage"))) def payloads = decodedBrokerPayloads() assert payloads.any { it.contains("FlexRequestResponse") } assert payloads.any { it.contains("FlexOffer") } @@ -220,8 +220,8 @@ class GOPACSHandlerHttpTest extends Specification implements ManagerContainerTra and: "a FlexOrderResponse is POSTed to the broker as signed XML with a bearer token" conditions.eventually { wireMock.verify(postRequestedFor(urlPathEqualTo(BROKER_PATH)) - .withHeader("Authorization", equalTo("Bearer " + ACCESS_TOKEN)) - .withRequestBody(containing("SignedMessage"))) + .withHeader("Authorization", equalTo("Bearer " + ACCESS_TOKEN)) + .withRequestBody(containing("SignedMessage"))) assert decodedBrokerPayloads().any { it.contains("FlexOrderResponse") } } } @@ -240,8 +240,8 @@ class GOPACSHandlerHttpTest extends Specification implements ManagerContainerTra and: "a rejected FlexRequestResponse with a reason is delivered to the broker as signed XML" conditions.eventually { wireMock.verify(postRequestedFor(urlPathEqualTo(BROKER_PATH)) - .withHeader("Authorization", equalTo("Bearer " + ACCESS_TOKEN)) - .withRequestBody(containing("SignedMessage"))) + .withHeader("Authorization", equalTo("Bearer " + ACCESS_TOKEN)) + .withRequestBody(containing("SignedMessage"))) assert decodedBrokerPayloads().any { it.contains("FlexRequestResponse") && it.contains('Result="Rejected"') && it.contains("RejectionReason=") } diff --git a/energy/src/test/groovy/org/openremote/extension/energy/EnergyOptimisationAssetTest.groovy b/energy/src/test/groovy/org/openremote/extension/energy/EnergyOptimisationAssetTest.groovy index 0a24db0..e094a60 100644 --- a/energy/src/test/groovy/org/openremote/extension/energy/EnergyOptimisationAssetTest.groovy +++ b/energy/src/test/groovy/org/openremote/extension/energy/EnergyOptimisationAssetTest.groovy @@ -159,7 +159,7 @@ class EnergyOptimisationAssetTest extends Specification implements ManagerContai AssetDatapointIntervalQuery.Formula.AVG, true ) - ) + ) assert setpoints.size() == 7 assert setpoints[0].value == 0d @@ -191,7 +191,7 @@ class EnergyOptimisationAssetTest extends Specification implements ManagerContai AssetDatapointIntervalQuery.Formula.AVG, true ) - ) + ) assert setpoints.size() == 7 assert setpoints[0].value == 0d @@ -223,7 +223,7 @@ class EnergyOptimisationAssetTest extends Specification implements ManagerContai AssetDatapointIntervalQuery.Formula.AVG, true ) - ) + ) assert setpoints.size() == 7 assert setpoints[0].value == 0d @@ -335,7 +335,7 @@ class EnergyOptimisationAssetTest extends Specification implements ManagerContai AssetDatapointIntervalQuery.Formula.AVG, true ) - ) + ) assert setpoints.size() == 7 assert setpoints[0].value == 10d diff --git a/energy/src/test/groovy/org/openremote/extension/energy/ForecastSolarServiceTest.groovy b/energy/src/test/groovy/org/openremote/extension/energy/ForecastSolarServiceTest.groovy index 67bca10..e43dbe9 100644 --- a/energy/src/test/groovy/org/openremote/extension/energy/ForecastSolarServiceTest.groovy +++ b/energy/src/test/groovy/org/openremote/extension/energy/ForecastSolarServiceTest.groovy @@ -139,7 +139,7 @@ class ForecastSolarServiceTest extends Specification implements ManagerContainer def responseBody = ValueUtil.JSON.readValue(content, ForecastSolarService.EstimateResponse.class) requestContext.abortWith( Response.ok(responseBody, MediaType.APPLICATION_JSON_TYPE).build() - ) + ) return } diff --git a/energy/src/test/groovy/org/openremote/extension/energy/ForecastWindServiceTest.groovy b/energy/src/test/groovy/org/openremote/extension/energy/ForecastWindServiceTest.groovy index f48e576..9ab29db 100644 --- a/energy/src/test/groovy/org/openremote/extension/energy/ForecastWindServiceTest.groovy +++ b/energy/src/test/groovy/org/openremote/extension/energy/ForecastWindServiceTest.groovy @@ -650,7 +650,7 @@ class ForecastWindServiceTest extends Specification implements ManagerContainerT def responseBody = ValueUtil.JSON.readValue(content, ForecastWindService.WeatherForecastResponseModel.class) requestContext.abortWith( Response.ok(responseBody, MediaType.APPLICATION_JSON_TYPE).build() - ) + ) return } diff --git a/entsoe/src/test/groovy/org/openremote/extension/entsoe/agent/protocol/EntsoeCurveTest.groovy b/entsoe/src/test/groovy/org/openremote/extension/entsoe/agent/protocol/EntsoeCurveTest.groovy index 91b4c31..7dff292 100644 --- a/entsoe/src/test/groovy/org/openremote/extension/entsoe/agent/protocol/EntsoeCurveTest.groovy +++ b/entsoe/src/test/groovy/org/openremote/extension/entsoe/agent/protocol/EntsoeCurveTest.groovy @@ -405,7 +405,7 @@ class EntsoeCurveTest extends Specification { private static String period(Instant start, Instant end, String resolution, List points) { def pointXml = points.collect { point -> "${point[0]}" + - (point[1] == null ? "" : "${point[1]}") + "" + (point[1] == null ? "" : "${point[1]}") + "" }.join("\n") """ diff --git a/entsoe/src/test/groovy/org/openremote/extension/entsoe/agent/protocol/EntsoeProtocolTest.groovy b/entsoe/src/test/groovy/org/openremote/extension/entsoe/agent/protocol/EntsoeProtocolTest.groovy index 111720e..9c6f989 100644 --- a/entsoe/src/test/groovy/org/openremote/extension/entsoe/agent/protocol/EntsoeProtocolTest.groovy +++ b/entsoe/src/test/groovy/org/openremote/extension/entsoe/agent/protocol/EntsoeProtocolTest.groovy @@ -383,7 +383,7 @@ class EntsoeProtocolTest extends Specification implements ManagerContainerTrait .addOrReplaceAttributes( new Attribute<>("energyPrice", NUMBER) .addOrReplaceMeta(new MetaItem<>(AGENT_LINK, entsoeLink)) - ) + ) asset = assetStorageService.merge(asset) def attributeRef = new AttributeRef(asset.id, "energyPrice") @@ -462,7 +462,7 @@ class EntsoeProtocolTest extends Specification implements ManagerContainerTrait .addOrReplaceAttributes( new Attribute<>("energyPrice", NUMBER) .addOrReplaceMeta(new MetaItem<>(AGENT_LINK, entsoeLink)) - ) + ) asset = assetStorageService.merge(asset) def attributeRef = new AttributeRef(asset.id, "energyPrice") @@ -551,7 +551,7 @@ class EntsoeProtocolTest extends Specification implements ManagerContainerTrait .addOrReplaceMeta(new MetaItem<>(AGENT_LINK, beLink)), new Attribute<>("energyPriceNl", NUMBER) .addOrReplaceMeta(new MetaItem<>(AGENT_LINK, nlLink)) - ) + ) asset = assetStorageService.merge(asset) def beRef = new AttributeRef(asset.id, "energyPriceBe") @@ -646,7 +646,7 @@ class EntsoeProtocolTest extends Specification implements ManagerContainerTrait .addOrReplaceMeta(new MetaItem<>(AGENT_LINK, link1)), new Attribute<>("energyPriceTwo", NUMBER) .addOrReplaceMeta(new MetaItem<>(AGENT_LINK, link2)) - ) + ) asset = assetStorageService.merge(asset) def refOne = new AttributeRef(asset.id, "energyPriceOne") @@ -736,7 +736,7 @@ class EntsoeProtocolTest extends Specification implements ManagerContainerTrait .addOrReplaceAttributes( new Attribute<>("energyPrice", NUMBER) .addOrReplaceMeta(new MetaItem<>(AGENT_LINK, errLink)) - ) + ) asset = assetStorageService.merge(asset) def attributeRef = new AttributeRef(asset.id, "energyPrice") @@ -806,7 +806,7 @@ class EntsoeProtocolTest extends Specification implements ManagerContainerTrait def document = { List points -> def pointXml = points.collect { point -> "${point[0]}" + - (point[1] == null ? "" : "${point[1]}") + "" + (point[1] == null ? "" : "${point[1]}") + "" }.join("") def xml = """ @@ -939,7 +939,7 @@ class EntsoeProtocolTest extends Specification implements ManagerContainerTrait break case "malformed price": periods.add(period(900, 2700, "PT15M", 999).replace("", - "2broken")) + "2broken")) break case "invalid timing": periods.add(period(900, 1800, "invalid", 999)) @@ -1052,7 +1052,7 @@ class EntsoeProtocolTest extends Specification implements ManagerContainerTrait .addOrReplaceAttributes( new Attribute<>("energyPrice", NUMBER) .addOrReplaceMeta(new MetaItem<>(AGENT_LINK, noDataLink)) - ) + ) asset = assetStorageService.merge(asset) def attributeRef = new AttributeRef(asset.id, "energyPrice") @@ -1123,7 +1123,7 @@ class EntsoeProtocolTest extends Specification implements ManagerContainerTrait .addOrReplaceAttributes( new Attribute<>("energyPrice", NUMBER) .addOrReplaceMeta(new MetaItem<>(AGENT_LINK, unsafeXmlLink)) - ) + ) asset = assetStorageService.merge(asset) def attributeRef = new AttributeRef(asset.id, "energyPrice") @@ -1186,7 +1186,7 @@ class EntsoeProtocolTest extends Specification implements ManagerContainerTrait .addOrReplaceAttributes( new Attribute<>("energyPrice", NUMBER) .addOrReplaceMeta(new MetaItem<>(AGENT_LINK, gapLink)) - ) + ) asset = assetStorageService.merge(asset) def attributeRef = new AttributeRef(asset.id, "energyPrice") @@ -1249,7 +1249,7 @@ class EntsoeProtocolTest extends Specification implements ManagerContainerTrait .addOrReplaceAttributes( new Attribute<>("energyPrice", NUMBER) .addOrReplaceMeta(new MetaItem<>(AGENT_LINK, multiPeriodLink)) - ) + ) asset = assetStorageService.merge(asset) def attributeRef = new AttributeRef(asset.id, "energyPrice") @@ -1362,7 +1362,7 @@ class EntsoeProtocolTest extends Specification implements ManagerContainerTrait .addOrReplaceAttributes( new Attribute<>("energyPrice", NUMBER) .addOrReplaceMeta(new MetaItem<>(AGENT_LINK, link)) - ) + ) asset = assetStorageService.merge(asset) def attributeRef = new AttributeRef(asset.id, "energyPrice") @@ -1384,8 +1384,8 @@ class EntsoeProtocolTest extends Specification implements ManagerContainerTrait conditions.eventually { assert requestLog.any { it.zone == "10YBE----------2" && - it.periodStart == periodFormatter.format(fixedNow) && - it.periodEnd == periodFormatter.format(fixedNow.plus(1, java.time.temporal.ChronoUnit.DAYS)) + it.periodStart == periodFormatter.format(fixedNow) && + it.periodEnd == periodFormatter.format(fixedNow.plus(1, java.time.temporal.ChronoUnit.DAYS)) } } @@ -1399,8 +1399,8 @@ class EntsoeProtocolTest extends Specification implements ManagerContainerTrait conditions.eventually { assert requestLog.any { it.zone == "10YBE----------2" && - it.periodStart == periodFormatter.format(nextNow) && - it.periodEnd == periodFormatter.format(nextNow.plus(1, java.time.temporal.ChronoUnit.DAYS)) + it.periodStart == periodFormatter.format(nextNow) && + it.periodEnd == periodFormatter.format(nextNow.plus(1, java.time.temporal.ChronoUnit.DAYS)) } } diff --git a/gradle.properties b/gradle.properties index 8852092..835b954 100644 --- a/gradle.properties +++ b/gradle.properties @@ -15,4 +15,4 @@ releasesRepoUrl=https://ossrh-staging-api.central.sonatype.com/service/local/ projectName = openremote-extensions ideaExtPluginVersion = 1.4.1 -spotlessVersion = 8.10.3 +spotlessVersion = 8.10.4 diff --git a/hawkbit/src/test/groovy/org/openremote/extension/hawkbit/manager/HawkbitFirmwareServiceTest.groovy b/hawkbit/src/test/groovy/org/openremote/extension/hawkbit/manager/HawkbitFirmwareServiceTest.groovy index e28813f..3904609 100644 --- a/hawkbit/src/test/groovy/org/openremote/extension/hawkbit/manager/HawkbitFirmwareServiceTest.groovy +++ b/hawkbit/src/test/groovy/org/openremote/extension/hawkbit/manager/HawkbitFirmwareServiceTest.groovy @@ -275,8 +275,8 @@ class HawkbitFirmwareServiceTest extends Specification implements ManagerContain then: "the token is forwarded to hawkBit" 1 * service.targets.create({ TargetCreateRequest[] targets -> targets.length == 1 && - targets[0].controllerId() == CONTROLLER_ID && - targets[0].securityToken() == "custom-token" + targets[0].controllerId() == CONTROLLER_ID && + targets[0].securityToken() == "custom-token" }) >> Response.ok(new Target[]{ createdTarget }).build() diff --git a/project.gradle b/project.gradle index 78dba9c..d42845a 100644 --- a/project.gradle +++ b/project.gradle @@ -63,11 +63,11 @@ subprojects { tasks.configureEach { task -> if (task.name in [ - 'findSonatypeStagingRepository', - 'closeSonatypeStagingRepository', - 'releaseSonatypeStagingRepository', - 'closeAndReleaseSonatypeStagingRepository' - ]) { + 'findSonatypeStagingRepository', + 'closeSonatypeStagingRepository', + 'releaseSonatypeStagingRepository', + 'closeAndReleaseSonatypeStagingRepository' + ]) { // See: https://github.com/gradle-nexus/publish-plugin/issues/221 task.notCompatibleWithConfigurationCache("Gradle Nexus Publish plugin task keeps Gradle Task/Project state") } diff --git a/settings.gradle b/settings.gradle index aac2465..b39d7ea 100644 --- a/settings.gradle +++ b/settings.gradle @@ -35,7 +35,7 @@ fileTree( .filter { !new File(it.parentFile, ".buildignore").exists() } .each { include it.parentFile.canonicalPath - .replace(rootDir.canonicalPath, "") - .replace("\\", ":") - .replace("/", ":") + .replace(rootDir.canonicalPath, "") + .replace("\\", ":") + .replace("/", ":") } diff --git a/spotless.gradle b/spotless.gradle index 0658da0..cac19d2 100644 --- a/spotless.gradle +++ b/spotless.gradle @@ -24,9 +24,9 @@ String prettierVersion = devDependencies.prettier def readIgnorePatterns = { File ignoreFile -> List patterns = ignoreFile - .readLines('UTF-8') - .collect { it.trim() } - .findAll { it && !it.startsWith('#') } + .readLines('UTF-8') + .collect { it.trim() } + .findAll { it && !it.startsWith('#') } List unsupported = patterns.findAll { it.startsWith('!') || it.startsWith('/') || it.contains('\\') }