From 36d5d85dacfcd9e8e4ac01fe5bdd735dc6816759 Mon Sep 17 00:00:00 2001 From: Oleg <18466855+EvaTheSalmon@users.noreply.github.com> Date: Mon, 19 Oct 2026 19:44:02 +0300 Subject: [PATCH] fix(chart): draw a live-edge break as a gap A break that opened at the live edge drew as a held line until the next history read covered the span: the poll counted a q = 32 row for its debug line only and emitted it as an ordinary sample. The realtime seam now carries an absence. A Sample with a null Value is a break for that pen at that moment; TrendCoordinator keeps it in the batch, and ChartRealtimeApplier appends a NaN column there at every layer instead of folding it into the last column. RealtimePoll turns a q = 32 row into its value followed by a break 1 ms later, the smallest step that lands on a distinct axis position. The later OPC UA slices write breaks on session loss, on a Bad status and ahead of a pen's first live sample through the same null sample. Closes the backlog item "A break that opens at the live edge draws as a held line". --- .../SemiPlot.Core/Trends/RealtimeBatch.cs | 6 +- SemiPlot/SemiPlot.Core/Trends/Sample.cs | 3 +- .../RealtimePoll.cs | 39 +- .../Journeys/LiveEdgeArchiveJourneyTests.cs | 48 +- .../RealtimePollReadTests.cs | 69 ++- .../UI/Bridge/FakeDataProvider.cs | 65 ++- .../UI/Bridge/TrendCoordinatorTests.cs | 23 + .../UI/Chart/ChartRealtimeApplierTests.cs | 95 ++++ .../UI/Chart/TrendChartViewModelTests.cs | 60 +++ .../UI/Chart/TrendPenStateTests.cs | 19 + .../SemiPlot.UI/Bridge/TrendCoordinator.cs | 2 +- .../SemiPlot.UI/Chart/ChartRealtimeApplier.cs | 16 +- docs/architecture/bench.md | 4 + docs/architecture/charting.md | 51 ++- docs/architecture/data-integration.md | 25 +- docs/architecture/testing-strategy.md | 5 +- docs/architecture/trend-feature-spec.md | 4 +- docs/architecture/trend-interaction.md | 23 +- docs/plans/backlog.md | 7 - .../completed/20261019-live-edge-break.md | 423 ++++++++++++++++++ 20 files changed, 893 insertions(+), 94 deletions(-) create mode 100644 SemiPlot/SemiPlot.Tests.Unit/UI/Chart/ChartRealtimeApplierTests.cs create mode 100644 docs/plans/completed/20261019-live-edge-break.md diff --git a/SemiPlot/SemiPlot.Core/Trends/RealtimeBatch.cs b/SemiPlot/SemiPlot.Core/Trends/RealtimeBatch.cs index 102501e..ff4f3fb 100644 --- a/SemiPlot/SemiPlot.Core/Trends/RealtimeBatch.cs +++ b/SemiPlot/SemiPlot.Core/Trends/RealtimeBatch.cs @@ -1,13 +1,13 @@ namespace SemiPlot.Core.Trends; /// -/// One pen's share of a , on that pen's own timestamps and nothing else; -/// double, not double?, because a null here would draw a break the archive never recorded. +/// One pen's share of a , on that pen's own timestamps only. A null is a break the +/// provider reported; a pen carries no entry, null or real, at a union timestamp it did not report. /// public sealed record PenRealtimeValues( int PenId, IReadOnlyList TimestampsUtc, - IReadOnlyList Values); + IReadOnlyList Values); /// /// One buffer window of the live edge. is the ascending union of every pen's own diff --git a/SemiPlot/SemiPlot.Core/Trends/Sample.cs b/SemiPlot/SemiPlot.Core/Trends/Sample.cs index 2c8bdda..044a3dd 100644 --- a/SemiPlot/SemiPlot.Core/Trends/Sample.cs +++ b/SemiPlot/SemiPlot.Core/Trends/Sample.cs @@ -1,3 +1,4 @@ namespace SemiPlot.Core.Trends; -public sealed record Sample(int PenId, DateTime TimestampUtc, double Value); +/// A null is a break for the pen from this moment until its next sample. +public sealed record Sample(int PenId, DateTime TimestampUtc, double? Value); diff --git a/SemiPlot/SemiPlot.DataSource.Postgres/RealtimePoll.cs b/SemiPlot/SemiPlot.DataSource.Postgres/RealtimePoll.cs index 317db87..a612c5a 100644 --- a/SemiPlot/SemiPlot.DataSource.Postgres/RealtimePoll.cs +++ b/SemiPlot/SemiPlot.DataSource.Postgres/RealtimePoll.cs @@ -31,14 +31,16 @@ internal sealed class RealtimePoll( int[] penIds, ILogger logger) { - // q = 32 marks the last sample before a break; read for the debug line only - // (docs/architecture/scada-archive.md, Quality and gaps). + // docs/architecture/scada-archive.md#quality-and-gaps private const int LastBeforeBreakQuality = 32; // Must stay far below the connection string's backstop: three stalled ticks must raise the fault // inside half a minute. private const int TickCommandTimeoutSeconds = 10; + // docs/architecture/data-integration.md#realtime + private static readonly TimeSpan _breakAnchorStep = TimeSpan.FromMilliseconds(1); + private static readonly IReadOnlyList _noSamples = []; private int _consecutiveFailures; @@ -150,11 +152,14 @@ private async Task> ReadSamplesAsync( var newest = lastSeen; var rowCount = 0; var droppedCount = 0; - var breakMarkCount = 0; + var breakCount = 0; while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) { + var penId = reader.GetInt32(0); var archiveLocal = reader.GetDateTime(1); + var timestampUtc = timeConverter.ToUtc(archiveLocal); + var isBreakMark = reader.GetInt32(3) == LastBeforeBreakQuality; rowCount++; @@ -163,33 +168,33 @@ private async Task> ReadSamplesAsync( newest = archiveLocal; } - if (reader.GetInt32(3) == LastBeforeBreakQuality) - { - breakMarkCount++; - } - - // Sample.Value is non-nullable; GetDouble would throw and count as a connection failure, so the - // row is dropped and lastSeen still advances. + // docs/architecture/data-integration.md#realtime if (reader.IsDBNull(2)) { droppedCount++; - - continue; + } + else + { + samples.Add(new Sample(penId, timestampUtc, reader.GetDouble(2))); } - samples.Add(new Sample(reader.GetInt32(0), timeConverter.ToUtc(archiveLocal), reader.GetDouble(2))); + if (isBreakMark) + { + samples.Add(new Sample(penId, timestampUtc + _breakAnchorStep, null)); + breakCount++; + } } Advance(newest); logger.LogDebug( - "The realtime poll read {RowCount} rows past {LastSeen:O}: {SampleCount} samples, {DroppedCount} " - + "dropped for a null value, {BreakMarkCount} marking the last sample before a break.", + "The realtime poll read {RowCount} rows past {LastSeen:O}: {ValueCount} value samples, {DroppedCount} " + + "rows dropped for a null value, {BreakCount} breaks.", rowCount, lastSeen, - samples.Count, + rowCount - droppedCount, droppedCount, - breakMarkCount); + breakCount); return samples; } diff --git a/SemiPlot/SemiPlot.Tests.Integration/Journeys/LiveEdgeArchiveJourneyTests.cs b/SemiPlot/SemiPlot.Tests.Integration/Journeys/LiveEdgeArchiveJourneyTests.cs index 39cccce..9b78397 100644 --- a/SemiPlot/SemiPlot.Tests.Integration/Journeys/LiveEdgeArchiveJourneyTests.cs +++ b/SemiPlot/SemiPlot.Tests.Integration/Journeys/LiveEdgeArchiveJourneyTests.cs @@ -86,6 +86,44 @@ public async Task ARowWrittenAfterStartupReachesTheChartOnceAndMovesItsLiveEdge( chart.Pens.Should().AllSatisfy(pen => pen.CurrentValue.Should().Be(ValueFor(pen.Pen.PenId, tick: 2))); } + [AvaloniaFact] + public async Task ARowMarkingTheLastSampleBeforeABreakReachesTheChartAsItsValueAndThenABreak() + { + await using var stand = await OpenStandAsync(); + var pen = stand.Catalogue[0]; + var seededLastLocal = ArchiveProviderFactory.TimeConverter.ToArchiveLocal(stand.Extent.LastUtc); + var appendLocal = seededLastLocal.AddSeconds(1); + var appendUtc = ArchiveProviderFactory.TimeConverter.ToUtc(appendLocal); + var value = ValueFor(pen.PenId, tick: 1); + var delivery = stand.Batches.NextBatch(); + + await AppendAsync(appendLocal, [pen], tick: 1, ArchiveRow.LastBeforeBreakQuality); + await delivery.WaitAsync(GateBound, TestContext.Current.CancellationToken); + + // The batch is what the poll delivered, which no history read can race. + var batch = stand.Batches.Batches.Should().ContainSingle().Subject; + var breakUtc = appendUtc.AddMilliseconds(1); + + batch.Timestamps.Should().Equal([appendUtc, breakUtc]); + var penValues = batch.Pens.Should().ContainSingle().Subject; + penValues.PenId.Should().Be(pen.PenId); + penValues.Values.Should().Equal([value, null]); + stand.Chart.Navigation.To.Should().Be(breakUtc, "the break is the batch's last timestamp"); + + var state = stand.Chart.Pens.Should().ContainSingle(candidate => candidate.Pen.PenId == pen.PenId).Subject; + var columns = state.Line.SnapshotColumns(); + + // A landed history read may add a NaN ahead of the break: + // docs/architecture/charting.md#per-pen-plottable-envelopeline + var lastRealColumn = columns.Last(column => !double.IsNaN(column.Center)); + + lastRealColumn.X.Should().Be(LocalTimeAxis.ToAxis(appendUtc)); + lastRealColumn.Center.Should().Be(value); + columns[^1].X.Should().Be(LocalTimeAxis.ToAxis(breakUtc)); + double.IsNaN(columns[^1].Center).Should().BeTrue("the row marks the last sample before a break"); + state.CurrentValue.Should().Be(value); + } + // The archive is per-variable and change-based with a deadband, so one poll tick spans as many distinct // timestamps as there are variables; a pen with no sample at a timestamp is left alone rather than // appended as null, which would draw a break the archive never recorded. @@ -120,7 +158,7 @@ public async Task RowsOnAVariableOfTheirOwnReachTheChartWithoutBreakingAnyPen() }); } - // Only the raw layer appends a realtime sample as a point of its own; a coarse layer folds it + // Only the raw layer appends a realtime value as a point of its own; a coarse layer folds a value // into the last column instead, and the assertions read the appended point. private async Task OpenStandAsync() { @@ -192,7 +230,11 @@ private async Task OpenStandAsync() // One row per variable, written through the appending path ArchiveWriter takes for a follow run: the // archive already carries the template's rows, so the seeded refusal has to be stood down. The day // partition the rows fall into already exists, and CreateStatement's IF NOT EXISTS passes through it. - private async Task AppendAsync(DateTime archiveLocal, IReadOnlyList pens, int tick) + private async Task AppendAsync( + DateTime archiveLocal, + IReadOnlyList pens, + int tick, + int quality = ArchiveRow.OrdinaryQuality) { var rows = pens .Select(pen => new ArchiveRow( @@ -200,7 +242,7 @@ private async Task AppendAsync(DateTime archiveLocal, IReadOnlyList pens, i ArchiveRow.RawLayer, archiveLocal, ValueFor(pen.PenId, tick), - ArchiveRow.OrdinaryQuality)) + quality)) .ToArray(); var written = await Writer() diff --git a/SemiPlot/SemiPlot.Tests.Integration/RealtimePollReadTests.cs b/SemiPlot/SemiPlot.Tests.Integration/RealtimePollReadTests.cs index f7377f3..3a94868 100644 --- a/SemiPlot/SemiPlot.Tests.Integration/RealtimePollReadTests.cs +++ b/SemiPlot/SemiPlot.Tests.Integration/RealtimePollReadTests.cs @@ -4,6 +4,7 @@ using SemiPlot.Core.Data; using SemiPlot.Core.Data.Errors; +using SemiPlot.Core.Trends; using SemiPlot.Tools.ArchiveSeeder; using Xunit; @@ -144,23 +145,20 @@ await AppendAsync( await poll.ReadOnceAsync(cancellationToken); - var rowCounts = logs.Collector.GetSnapshot() - .Select(entry => entry.GetStructuredStateValue("RowCount")) - .OfType(); - - rowCounts.Should().Equal("1", "0"); + PollCounts(logs).Should().Equal(("1", "0", "1", "0"), ("0", "0", "0", "0")); } - // The archive's break mark carries a real value (docs/architecture/scada-archive.md, Quality and gaps). [Fact] - public async Task ARowMarkingTheLastSampleBeforeABreakEmitsAnOrdinarySample() + public async Task ARowMarkingTheLastSampleBeforeABreakEmitsItsValueAndThenABreak() { var cancellationToken = TestContext.Current.CancellationToken; var appended = SeededLast.AddSeconds(1); + var appendedUtc = ArchiveProviderFactory.TimeConverter.ToUtc(appended); using var services = ArchiveProviderFactory.Build(Database.PlotConnectionString); + using var logs = new FakeLoggerProvider(); - var poll = NewPoll(services); + var poll = NewPoll(services, logs.CreateLogger("RealtimePoll")); await poll.ReadOnceAsync(cancellationToken); await AppendAsync( @@ -171,10 +169,46 @@ await AppendAsync( ArchiveRow.LastBeforeBreakQuality, cancellationToken: cancellationToken); - var sample = (await poll.ReadOnceAsync(cancellationToken)).Samples.Should().ContainSingle().Which; + var tick = await poll.ReadOnceAsync(cancellationToken); - sample.Value.Should().Be(11.25); - sample.TimestampUtc.Should().Be(ArchiveProviderFactory.TimeConverter.ToUtc(appended)); + tick.Samples.Should().Equal( + new Sample(SubscribedPenIds[0], appendedUtc, 11.25), + new Sample(SubscribedPenIds[0], appendedUtc.AddMilliseconds(1), null)); + PollCounts(logs).Should().Equal(("1", "1", "0", "1")); + } + + // docs/architecture/data-integration.md#realtime + [Fact] + public async Task ARowMarkingTheLastSampleBeforeABreakWithANullValueEmitsOnlyTheBreak() + { + var cancellationToken = TestContext.Current.CancellationToken; + var appended = SeededLast.AddSeconds(1); + + using var services = ArchiveProviderFactory.Build(Database.PlotConnectionString); + using var logs = new FakeLoggerProvider(); + + var poll = NewPoll(services, logs.CreateLogger("RealtimePoll")); + + await poll.ReadOnceAsync(cancellationToken); + await AppendAsync( + Database, + SubscribedPenIds[0], + appended, + null, + ArchiveRow.LastBeforeBreakQuality, + cancellationToken: cancellationToken); + + var tick = await poll.ReadOnceAsync(cancellationToken); + + tick.Samples.Should().Equal(new Sample( + SubscribedPenIds[0], + ArchiveProviderFactory.TimeConverter.ToUtc(appended).AddMilliseconds(1), + null)); + tick.StateChange.Should().BeNull(); + + await poll.ReadOnceAsync(cancellationToken); + + PollCounts(logs).Should().Equal(("1", "0", "1", "1"), ("0", "0", "0", "0")); } // Renamed rather than dropped: the container is shared. The tick after recovery must report nothing. @@ -261,6 +295,19 @@ private Task RestoreTrendsAsync(CancellationToken cancellationToken) cancellationToken); } + // The baseline entry carries no RowCount. + private static IEnumerable<(string? Rows, string? Values, string? Dropped, string? Breaks)> PollCounts( + FakeLoggerProvider logs) + { + return logs.Collector.GetSnapshot() + .Where(entry => entry.GetStructuredStateValue("RowCount") is not null) + .Select(entry => ( + entry.GetStructuredStateValue("RowCount"), + entry.GetStructuredStateValue("ValueCount"), + entry.GetStructuredStateValue("DroppedCount"), + entry.GetStructuredStateValue("BreakCount"))); + } + // The unsubscribed variable's newest row sits a minute after the subscribed ones, so a baseline read // over the whole archive rather than over the requested identifiers answers a different instant and fails. private static IReadOnlyList SeededRowsWithAnUnsubscribedVariable() diff --git a/SemiPlot/SemiPlot.Tests.Unit/UI/Bridge/FakeDataProvider.cs b/SemiPlot/SemiPlot.Tests.Unit/UI/Bridge/FakeDataProvider.cs index 03c1e3b..766a933 100644 --- a/SemiPlot/SemiPlot.Tests.Unit/UI/Bridge/FakeDataProvider.cs +++ b/SemiPlot/SemiPlot.Tests.Unit/UI/Bridge/FakeDataProvider.cs @@ -4,6 +4,8 @@ using System.Reactive.Linq; using System.Reactive.Subjects; +using AwesomeAssertions; + using FluentResults; using SemiPlot.Core.Data; @@ -36,6 +38,8 @@ internal sealed class FakeDataProvider( private readonly Lock _historyQueriesGate = new(); private readonly List _historyQueries = []; + private int? _penToBreakOnNextTick; + public DateTime ArchiveFirstUtc { get; set; } = new(2025, 12, 25, 0, 0, 0, DateTimeKind.Utc); public DateTime ArchiveLastUtc { get; set; } = new(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc); @@ -181,6 +185,35 @@ public void ReportConnectionState(ArchiveConnectionState state) _connectionFaults.OnNext(state); } + /// The next tick follows the pen's value with a break 1 ms later, as the poll's q = 32 row. + public void BreakRealtimePenOnce(int penId) + { + _penToBreakOnNextTick = penId; + } + + /// The pen's one break across the batches, and the value its tick carried 1 ms ahead of it. + public static (double Value, DateTime ValueUtc, DateTime BreakUtc) TheBreakIn( + IEnumerable batches, + int penId) + { + var samples = batches + .SelectMany(batch => batch.Pens) + .Where(values => values.PenId == penId) + .SelectMany(values => values.TimestampsUtc.Zip(values.Values)) + .ToList(); + var breakIndex = samples.FindIndex(sample => sample.Second is null); + + samples.Count(sample => sample.Second is null).Should().Be(1); + breakIndex.Should().BePositive(); + + var (valueUtc, value) = samples[breakIndex - 1]; + var breakUtc = samples[breakIndex].First; + + breakUtc.Should().Be(valueUtc.AddMilliseconds(1)); + + return (value.GetValueOrDefault(), valueUtc, breakUtc); + } + public IObservable> Subscribe(IReadOnlyList penIds) { return Observable @@ -211,14 +244,30 @@ private IObservable> BuildLiveEdge(IReadOnlyList penI return Observable .Interval(_realtimeInterval, _scheduler) .Select(tick => PoisonRealtimeWindow - ? (IReadOnlyList)new PoisonedSampleList() - : [.. subscribed - .Select((id, index) => new Sample( - id, - RealtimeEpoch - + TimeSpan.FromTicks(_realtimeInterval.Ticks * (tick + 1)) - + (StaggerRealtimeTimestamps ? TimeSpan.FromMilliseconds(index) : TimeSpan.Zero), - id + tick))]); + ? new PoisonedSampleList() + : TickSamples(subscribed, tick)); + } + + private IReadOnlyList TickSamples(IReadOnlyList subscribed, long tick) + { + var samples = new List(subscribed.Count + 1); + + for (var index = 0; index < subscribed.Count; index++) + { + var id = subscribed[index]; + var timestampUtc = RealtimeEpoch + + TimeSpan.FromTicks(_realtimeInterval.Ticks * (tick + 1)) + + (StaggerRealtimeTimestamps ? TimeSpan.FromMilliseconds(index) : TimeSpan.Zero); + samples.Add(new Sample(id, timestampUtc, id + tick)); + + if (id == _penToBreakOnNextTick) + { + samples.Add(new Sample(id, timestampUtc.AddMilliseconds(1), null)); + _penToBreakOnNextTick = null; + } + } + + return samples; } public Task>> QueryPensAsync() diff --git a/SemiPlot/SemiPlot.Tests.Unit/UI/Bridge/TrendCoordinatorTests.cs b/SemiPlot/SemiPlot.Tests.Unit/UI/Bridge/TrendCoordinatorTests.cs index b8ee8e5..e198a5c 100644 --- a/SemiPlot/SemiPlot.Tests.Unit/UI/Bridge/TrendCoordinatorTests.cs +++ b/SemiPlot/SemiPlot.Tests.Unit/UI/Bridge/TrendCoordinatorTests.cs @@ -69,6 +69,29 @@ public void RealtimeBatch_KeepsEachPenOnItsOwnTimestamps() batch.Pens.SelectMany(values => values.TimestampsUtc).Should().OnlyHaveUniqueItems(); } + [Fact] + public void ABreakTheProviderReports_SurvivesTheBufferWindowOnItsOwnTimestamp() + { + var (coordinator, scheduler, provider) = CreateCoordinator(realtimeInterval: ChartTestBuilder.RealtimeInterval); + using var ownedCoordinator = coordinator; + var batches = new List(); + using var subscription = coordinator.RealtimeBatches.Subscribe(batches.Add); + + provider.BreakRealtimePenOnce(FakeDataProvider.FirstPen.PenId); + scheduler.AdvanceBy(TrendCoordinator.BatchWindow.Ticks); + + var batch = batches.Single(); + var (_, _, breakUtc) = FakeDataProvider.TheBreakIn([batch], FakeDataProvider.FirstPen.PenId); + batch.Timestamps.Should().Contain(breakUtc); + batch.Pens.Should().OnlyContain(values => values.Values.Count == values.TimestampsUtc.Count); + batch.Pens.Single(values => values.PenId == FakeDataProvider.SecondPen.PenId).Values.Should().NotContainNulls(); + + scheduler.AdvanceBy(TrendCoordinator.BatchWindow.Ticks); + + batches.Should().HaveCount(2); + batches[1].Pens.Should().AllSatisfy(values => values.Values.Should().NotContainNulls()); + } + [Fact] public void RealtimeBatches_WithAnEmptyCatalog_EmitNothing() { diff --git a/SemiPlot/SemiPlot.Tests.Unit/UI/Chart/ChartRealtimeApplierTests.cs b/SemiPlot/SemiPlot.Tests.Unit/UI/Chart/ChartRealtimeApplierTests.cs new file mode 100644 index 0000000..dd996fa --- /dev/null +++ b/SemiPlot/SemiPlot.Tests.Unit/UI/Chart/ChartRealtimeApplierTests.cs @@ -0,0 +1,95 @@ +using AwesomeAssertions; + +using SemiPlot.Core.Trends; +using SemiPlot.Tests.Unit.UI.Bridge; +using SemiPlot.UI.Chart; + +using Xunit; + +namespace SemiPlot.Tests.Unit.UI.Chart; + +[Trait("Component", "UI")] +[Trait("Area", "Chart")] +[Trait("Category", "Unit")] +public sealed class ChartRealtimeApplierTests +{ + private const double SampledValue = 7.0; + private static readonly DateTime _start = ChartTestBuilder.From; + + [Fact] + public void AtRaw_AValueAndThenABreak_AppendARealColumnAndThenANaNColumn() + { + var state = NewState(); + + Apply(state, foldIntoColumn: false, (_start, SampledValue), (_start.AddMilliseconds(1), null)); + + var columns = state.Line.SnapshotColumns(); + columns.Should().HaveCount(2); + columns[0].Center.Should().Be(SampledValue); + double.IsNaN(columns[1].Center).Should().BeTrue(); + columns[1].X.Should().Be(LocalTimeAxis.ToAxis(_start.AddMilliseconds(1))); + state.CurrentValue.Should().Be(SampledValue); + } + + [Fact] + public void AtACoarseLayer_TheValueFoldsAndTheBreakAppendsPastTheFoldedColumn() + { + var state = NewState(); + state.LoadHistory(new PenHistoryEnvelope(1, [_start], [1.0], [5.0], [2.0]), _start); + + Apply( + state, + foldIntoColumn: true, + (_start.AddSeconds(1), SampledValue), + (_start.AddSeconds(1).AddMilliseconds(1), null)); + + var columns = state.Line.SnapshotColumns(); + columns.Should().HaveCount(2); + columns[0].Min.Should().Be(1.0); + columns[0].Max.Should().Be(SampledValue); + columns[0].Center.Should().Be(SampledValue); + double.IsNaN(columns[1].Center).Should().BeTrue(); + columns[1].X.Should().Be(LocalTimeAxis.ToAxis(_start.AddSeconds(1).AddMilliseconds(1))); + state.CurrentValue.Should().Be(SampledValue); + } + + // docs/architecture/charting.md#per-pen-plottable-envelopeline + [Fact] + public void AtACoarseLayer_OnAPenWithNoColumn_TheBreakAppendsAndTheNextValueOpensAColumnPastIt() + { + const double NextValue = 9.0; + var state = NewState(); + + Apply( + state, + foldIntoColumn: true, + (_start, SampledValue), + (_start.AddMilliseconds(1), null), + (_start.AddSeconds(1), NextValue)); + + var columns = state.Line.SnapshotColumns(); + columns.Should().HaveCount(2); + double.IsNaN(columns[0].Center).Should().BeTrue(); + columns[1].Center.Should().Be(NextValue); + columns[1].X.Should().Be(LocalTimeAxis.ToAxis(_start.AddSeconds(1))); + state.CurrentValue.Should().Be(NextValue); + } + + private static void Apply(TrendPenState state, bool foldIntoColumn, params (DateTime Utc, double? Value)[] samples) + { + var timestamps = samples.Select(sample => sample.Utc).ToArray(); + var batch = new RealtimeBatch( + timestamps, + [new PenRealtimeValues(state.Pen.PenId, timestamps, [.. samples.Select(sample => sample.Value)])]); + var applier = new ChartRealtimeApplier( + new Dictionary { [state.Pen.PenId] = state }, + new ChartNavigationController()); + + applier.Apply(batch, foldIntoColumn); + } + + private static TrendPenState NewState() + { + return new TrendPenState(FakeDataProvider.FirstPen, new EnvelopeLine()); + } +} diff --git a/SemiPlot/SemiPlot.Tests.Unit/UI/Chart/TrendChartViewModelTests.cs b/SemiPlot/SemiPlot.Tests.Unit/UI/Chart/TrendChartViewModelTests.cs index 65d4461..7f384b3 100644 --- a/SemiPlot/SemiPlot.Tests.Unit/UI/Chart/TrendChartViewModelTests.cs +++ b/SemiPlot/SemiPlot.Tests.Unit/UI/Chart/TrendChartViewModelTests.cs @@ -1036,6 +1036,66 @@ await ReleaseAndAwait( provider.HistoryQueryCount.Should().Be(queriesBefore, "the pan stays inside the coarse band"); } + [AvaloniaFact] + public async Task ABreakAtACoarseLayerAppendsANaNColumnAndTheNextFoldOpensAColumnPastIt() + { + var (viewModel, scheduler, coordinator, provider) = CreateViewModel(RealtimeInterval); + var state = viewModel.AddPen(FakeDataProvider.FirstPen); + viewModel.Navigation.TrackDataExtents( + FakeDataProvider.RealtimeEpoch.AddDays(-30.0), FakeDataProvider.RealtimeEpoch); + await ZoomOutAndIssueTheCoarseRead(viewModel, scheduler, provider); + await ReleaseAndAwait( + viewModel, 1, () => provider.HistoryGate.SetResult(EnvelopeEndingAtTheRealtimeEpoch())); + LandTheMarginReadFromTheReleasedGate(scheduler); + var columnsBefore = state.Line.SnapshotColumns(); + var foldedColumn = columnsBefore[^1]; + foldedColumn.X.Should().Be(LocalTimeAxis.ToAxis(FakeDataProvider.RealtimeEpoch)); + var batches = new List(); + using var subscription = coordinator.RealtimeBatches.Subscribe(batches.Add); + + provider.BreakRealtimePenOnce(FakeDataProvider.FirstPen.PenId); + scheduler.AdvanceBy(TrendCoordinator.BatchWindow.Ticks * 2); + + var (_, _, breakUtc) = FakeDataProvider.TheBreakIn(batches, FakeDataProvider.FirstPen.PenId); + var columns = state.Line.SnapshotColumns(); + columns.Should().HaveCount(columnsBefore.Count + 2, "the folds after the break land in one column"); + var widened = columns[columnsBefore.Count - 1]; + widened.X.Should().Be(foldedColumn.X); + double.IsNaN(widened.Center).Should().BeFalse(); + widened.Max.Should().BeGreaterThan(foldedColumn.Max, "the value ahead of the break folds into it"); + double.IsNaN(columns[^2].Center).Should().BeTrue(); + columns[^2].X.Should().Be(LocalTimeAxis.ToAxis(breakUtc)); + double.IsNaN(columns[^1].Center).Should().BeFalse(); + columns[^1].X.Should().BeGreaterThan(columns[^2].X); + state.CurrentValue.Should().Be(columns[^1].Center); + viewModel.Navigation.To.Should().Be(batches[^1].Timestamps[^1]); + } + + [AvaloniaFact] + public async Task ABreakAtRawAppendsANaNColumnOneMillisecondPastItsValue() + { + var (viewModel, scheduler, coordinator, provider) = CreateViewModel(RealtimeInterval); + var state = viewModel.AddPen(FakeDataProvider.FirstPen); + await LoadRawEndingAtTheRealtimeEpoch(viewModel, scheduler, provider); + state.Line.SnapshotColumns().Should().NotContain(column => double.IsNaN(column.Center)); + var batches = new List(); + using var subscription = coordinator.RealtimeBatches.Subscribe(batches.Add); + + provider.BreakRealtimePenOnce(FakeDataProvider.FirstPen.PenId); + scheduler.AdvanceBy(TrendCoordinator.BatchWindow.Ticks * 2); + + var (value, valueUtc, breakUtc) = FakeDataProvider.TheBreakIn(batches, FakeDataProvider.FirstPen.PenId); + var columns = state.Line.SnapshotColumns(); + var breakIndex = columns.ToList().FindIndex(column => double.IsNaN(column.Center)); + columns.Count(column => double.IsNaN(column.Center)).Should().Be(1); + breakIndex.Should().BePositive(); + columns[breakIndex].X.Should().Be(LocalTimeAxis.ToAxis(breakUtc)); + columns[breakIndex - 1].X.Should().Be(LocalTimeAxis.ToAxis(valueUtc)); + columns[breakIndex - 1].Center.Should().Be(value); + columns.Should().HaveCountGreaterThan(breakIndex + 1, "the tick after the break resumes the line"); + state.CurrentValue.Should().Be(columns[^1].Center); + } + [AvaloniaFact] public async Task AnAppendOverRawColumnsKeepsTheRawBand() { diff --git a/SemiPlot/SemiPlot.Tests.Unit/UI/Chart/TrendPenStateTests.cs b/SemiPlot/SemiPlot.Tests.Unit/UI/Chart/TrendPenStateTests.cs index c3141eb..6b1b10d 100644 --- a/SemiPlot/SemiPlot.Tests.Unit/UI/Chart/TrendPenStateTests.cs +++ b/SemiPlot/SemiPlot.Tests.Unit/UI/Chart/TrendPenStateTests.cs @@ -64,6 +64,25 @@ public void Realtime_NullSample_AppendsNaNGap() double.IsNaN(columns[0].Center).Should().BeTrue(); } + [Fact] + public void Realtime_BreakOneTickAfterTheValue_IsRejectedAndOneMillisecondAfterIt_AddsANaNColumn() + { + var state = NewState(); + state.AppendRealtime(_start, 5.0); + + state.AppendRealtime(_start.AddTicks(1), value: null); + + state.Line.SnapshotColumns().Should().ContainSingle("the axis keeps milliseconds only"); + + state.AppendRealtime(_start.AddMilliseconds(1.0), value: null); + + var columns = state.Line.SnapshotColumns(); + columns.Should().HaveCount(2); + columns[0].Center.Should().Be(5.0); + double.IsNaN(columns[1].Center).Should().BeTrue(); + state.CurrentValue.Should().Be(5.0); + } + [Fact] public void History_LoadsColumnsCarryingMinAndMax() { diff --git a/SemiPlot/SemiPlot.UI/Bridge/TrendCoordinator.cs b/SemiPlot/SemiPlot.UI/Bridge/TrendCoordinator.cs index d801d7b..add6edc 100644 --- a/SemiPlot/SemiPlot.UI/Bridge/TrendCoordinator.cs +++ b/SemiPlot/SemiPlot.UI/Bridge/TrendCoordinator.cs @@ -160,7 +160,7 @@ private static RealtimeBatch BuildRealtimeBatch(IList> win return new RealtimeBatch(timestamps, pens); } - // A pen carries only the samples it has; a filler null would draw a break the archive never recorded. + // A pen carries only its own samples, no filler at another pen's timestamp. private static PenRealtimeValues BuildPenValues(IGrouping penSamples) { var ordered = penSamples.OrderBy(sample => sample.TimestampUtc).ToArray(); diff --git a/SemiPlot/SemiPlot.UI/Chart/ChartRealtimeApplier.cs b/SemiPlot/SemiPlot.UI/Chart/ChartRealtimeApplier.cs index 669ef0e..677cc82 100644 --- a/SemiPlot/SemiPlot.UI/Chart/ChartRealtimeApplier.cs +++ b/SemiPlot/SemiPlot.UI/Chart/ChartRealtimeApplier.cs @@ -25,17 +25,23 @@ private void ApplyBatch(RealtimeBatch batch, bool foldIntoColumn) continue; } - // The pen's own timestamps, never the batch's union: a timestamp this pen did not sample is - // not its gap, and appending it here would draw a break the archive never recorded. + // The pen's own timestamps, never the batch's union: a timestamp this pen did not report is not its gap. for (var index = 0; index < penValues.Values.Count; index++) { - if (foldIntoColumn) + var timestampUtc = penValues.TimestampsUtc[index]; + var value = penValues.Values[index]; + + if (value is null) + { + state.AppendRealtime(timestampUtc, null); + } + else if (foldIntoColumn) { - state.FoldRealtime(penValues.TimestampsUtc[index], penValues.Values[index]); + state.FoldRealtime(timestampUtc, value.Value); } else { - state.AppendRealtime(penValues.TimestampsUtc[index], penValues.Values[index]); + state.AppendRealtime(timestampUtc, value); } } } diff --git a/docs/architecture/bench.md b/docs/architecture/bench.md index d8bf5c4..5443015 100644 --- a/docs/architecture/bench.md +++ b/docs/architecture/bench.md @@ -91,6 +91,10 @@ dotnet run --project SemiPlot/SemiPlot.Tools.ArchiveSeeder/SemiPlot.Tools.Archiv Every tick appends the raw rows of the window since the previous tick, thins them into the coarse layers and prints both counts; `Ctrl+C` stops the loop where it waits, never inside an append. +To see a live break on the stand, insert one raw row for a pen in view as `scada_writer`, with `l = 0`, +`q = 32` and `t` past the newest row: the line ends at its value and resumes at the writer's next row +(`data-integration.md#realtime`). + - **The window is `after < t <= to`.** `LiveTailGenerator.Generate(options, after, to)` takes the archive's edge on the first tick and the previous tick's own instant after that. Both are instants the archive already accounts for, so consecutive windows partition the lattice: a restart writes no diff --git a/docs/architecture/charting.md b/docs/architecture/charting.md index 3d8b2b8..8527db6 100644 --- a/docs/architecture/charting.md +++ b/docs/architecture/charting.md @@ -61,10 +61,12 @@ choice and keep the flag, so a thin line stays thin on the pen that takes the sl except after a catalogue read that leaves no pen: `ApplyCatalogue` then clears the flag, so the first pen a later read seats starts thin, as in a new window. -**Gaps via NaN.** A column carrying `NaN` in `Min`/`Max` is the envelope's gap marker (and what -`TrendPenState` encodes a null append as): `Build` breaks the path there and the next real column opens a -new segment with a `MoveTo`. Every NaN a chart draws comes from the history path: `RealtimeBatch` carries -`double` values, so the live edge has no null to encode. +**Gaps via NaN.** A column carrying `NaN` in `Min`/`Max` is the envelope's gap marker: `Build` breaks the +path there and the next real column opens a new segment with a `MoveTo`. A `NaN` column comes from one of +two routes. The history path writes one after a bucket whose `breaks` flag a `q = 32` marker or a null +`v` sets (`data-integration.md#quality-and-gaps`). The live edge writes one for a break the provider +reported: `TrendPenState.AppendRealtime` encodes a null value as a `NaN` column and keeps `CurrentValue` at +the last real value. **Per-pen stepping.** The line carries the Core `PenLineStyle` — `Stepped` for discrete/digital tags, `Interpolated` for analog. `Chart/TrendPenState` sets it and the colour from the pen through @@ -109,14 +111,20 @@ meets the edit throws "Collection was modified" on the render thread, the crash plottable's column list, which every render re-reads, so appends are live and nothing is re-set. The realtime tail appends one degenerate column (`Min == Max == Center == value`) at the live edge. At coarse layers (minute/hour) a realtime sample does **not** append; it folds into the current decimation column -(`FoldRealtime` widens that column's `Min`/`Max` and moves its `Center`). A gap column takes no fold: -when the last column is a `NaN` gap, as after a history envelope that ends on nulls, the sample opens a -column at its own timestamp past the gap, and the samples after it fold into that one. The gap still draws -as a break, and the line and `CurrentValue` resume at the first value instead of waiting for the next read. -A sample at or before the gap's X is rejected, like any append at or before the last X. A pen with no -column at all folds nothing. Cursor and legend read the `Center` channel consistently across the seam. The fold follows the layer in view, not the layer the columns -were read at: until a coarse read lands, the columns are still Raw, and the fold over them opens the history -gate so the return to Raw reads the folded span again +(`FoldRealtime` widens that column's `Min`/`Max` and moves its `Center`). A live break is always appended, +at every layer: `ChartRealtimeApplier` hands a null value to `AppendRealtime` ahead of the fold-or-append +choice, because `Math.Min`/`Math.Max` with a `NaN` would turn the folded column into a gap and erase its +extremes. A gap column takes no fold: when the last column is a `NaN` gap, as after a history envelope +that ends on nulls or after a live break, the sample opens a column at its own timestamp past the gap, and +the samples after it fold into that one. The gap still draws as a break, and the line and `CurrentValue` +resume at the first value instead of waiting for the next read. A sample at or before the gap's X is +rejected, like any append at or before the last X. A pen with no column folds no value, but a break still +appends its `NaN` column, and the folds after it open a live column past it before any history read lands; +the next read replaces that span up to its new last column, and the live columns past it, the break and +the column after it among them where the last bucket sits before the break, stay under the kept-tail rule +below. Cursor and legend read the `Center` channel consistently across the seam. The fold follows the +layer in view, not the layer the columns were read at: until a coarse read lands, the columns are still +Raw, and the fold over them opens the history gate so the return to Raw reads the folded span again (`data-integration.md#what-one-history-query-covers`). A history apply keeps the live tail past its snapshot. `EnvelopeLine.ReplaceColumns` takes the new @@ -128,7 +136,11 @@ except where `EnvelopeSplice.Over` laid a margin-less read over the envelope in (`data-integration.md#what-one-history-query-covers`). A pan into the past asks for a range that ends before the follow phase's tail, so the tail goes and the band just read keeps the whole buffer. `TrendPenState.ClearHistory` is a replace with no columns, so a -requested pen absent from a result keeps its tail too. `LoadHistory` and `ClearHistory` both set +requested pen absent from a result keeps its tail too. A live break column is a live append and follows the +same rule. A read that ends at a `q = 32` marker writes its anchor `NaN` one tick past the marker, which +maps to the marker's own X (`data-integration.md#quality-and-gaps`); the live value at that X is not after +the new last column and goes, and the live break 1 ms past it stays. The pen then holds two adjacent `NaN` +columns, and `Build` draws one break across them. `LoadHistory` and `ClearHistory` both set `CurrentValue` from the merged columns. An append at or before the last kept column is rejected, as before any other last column; at a coarse layer the next fold widens the last kept column (`data-integration.md#what-one-history-query-covers`). Only appended columns survive: a sample folded into @@ -462,7 +474,9 @@ models, backed by renderer-agnostic models in `SemiPlot.Core`. Responsibilities: caller. - `Chart/ChartRealtimeApplier` — the append-vs-fold rule per layer for incoming `RealtimeBatch`es. It walks each `PenRealtimeValues` on that pen's own timestamps, never on the batch's union, and - hands the union's last timestamp to `ChartNavigationController.OnLiveEdge`. + hands the union's last timestamp to `ChartNavigationController.OnLiveEdge`. A null value is a break: + it goes to `TrendPenState.AppendRealtime` ahead of the fold-or-append choice, so it appends a `NaN` + column at every layer and never folds. - `Chart/ChartCursorReader` / `ChartDeltaCursorReader` — view-side state wrapping the Core cursor models, resolving the visible / active pens (`ChartDeltaCursorReader.FormatReadout` formats Δt/Δy). - `Chart/ChartHoverReadout` — pure static `BuildContent`: builds the readout string (local timestamp + @@ -767,13 +781,14 @@ view model: for run last. The minimap band is a second reader outside the debouncer: one pen at 250 columns every 2-60 s (`trend-interaction.md#archive-overview-minimap`). - **Realtime:** `IObservable` — an ascending union timeline the live edge advances - from, plus one `PenRealtimeValues` per pen carrying that pen's **own** timestamps and `double` + from, plus one `PenRealtimeValues` per pen carrying that pen's **own** timestamps and `double?` values; buffered on the data scheduler and observed on the UI scheduler. The values are per pen rather than a column over the union because the archive is per-variable and change-based with a deadband, so one buffer window routinely spans timestamps only one pen sampled. A pen carries no - entry at a timestamp it did not sample, and a consumer leaves it alone there. The type is `double` - and not `double?`: `Sample` carries a non-nullable value, so the live edge has no representation - for a break, and the gap a chart draws is the history path's reconstruction from `q = 32`. The pen + entry, null or real, at a timestamp it did not report, and a consumer leaves it alone there. A null + is a break the provider reported, a `Sample` whose `Value` is null: the pen has no data from that + moment until its next sample, and the chart appends a `NaN` column there. `RealtimePoll` writes one + 1 ms after a `q = 32` row (`data-integration.md#realtime`). The pen set the stream carries follows the catalogue: `TrendCoordinator.SetPens` switches the provider subscription and `RealtimeBatches` stays one published stream (`data-integration.md#realtime`). - **Archive extent:** `QueryArchiveExtentAsync()` returns an `ArchiveExtent(FirstUtc, LastUtc)` — diff --git a/docs/architecture/data-integration.md b/docs/architecture/data-integration.md index 81318f4..810d6e7 100644 --- a/docs/architecture/data-integration.md +++ b/docs/architecture/data-integration.md @@ -67,7 +67,7 @@ public interface IDataProvider | Type | Shape | Notes | | --- | --- | --- | | `Pen` | `PenId`, `Name`, `Groups`, `Color`, `Unit`, `Format`, `EnabledOnStart`, `ScaleMinOnStart`, `ScaleMaxOnStart`, `LogScaleOnStart`, `LineStyle` | `PenId` is the archive's `trends.id`. Equality compares `Groups` element by element, ordinal, so two reads of one stored pen are equal; `LogScaleOnStart` takes part in it, so a pen whose flag alone changed is revised. | -| `Sample` | `PenId`, `TimestampUtc`, `Value` | Realtime element. Timestamps are UTC by the time they leave the provider. | +| `Sample` | `PenId`, `TimestampUtc`, `Value` | Realtime element. Timestamps are UTC by the time they leave the provider. A null `Value` is a break the provider reported: the pen has no data from that moment until its next sample. | | `PenHistoryEnvelope` | parallel `Timestamps` / `Min` / `Max` / `Center`, strictly ascending, `NaN` marks a gap | One per pen per history query. | | `ArchiveExtent` | `FirstUtc`, `LastUtc`, `IsEmpty` | The span of the configured variables, consumed by the minimap (`TM-4`). `ArchiveExtent.Empty` is the no-span form. | | `AggregationLayer` | `Raw`, `Minute`, `Hour` | Maps one-to-one onto the archive's `l` column. | @@ -913,7 +913,7 @@ The provider maps the archive's quality marks onto the envelope's gap representa | Archive | Envelope | | --- | --- | | `q = 0` | ordinary point | -| `q = 32` (last sample before a break) | point kept, then a `NaN` anchor inserted after it | +| `q = 32` (last sample before a break) | point kept, then a `NaN` anchor inserted after it; at the live edge, a break 1 ms later ("Realtime") | | `q = 16` (first sample after a break) | point kept; the line resumes here | | absence of rows without a preceding `q = 32` | no anchor — the value simply did not change | @@ -929,7 +929,8 @@ takes no branch: the next bucket resumes the line. A seed row carries its own `q `32` opens the window inside a gap. The coarse layers hold the markers too (`scada-archive.md#layers`), so a break splits a `Minute` or `Hour` bucket as it splits a `Raw` one. No decimator runs on the client. `RealArchiveGapTests` drives the fold with rows lifted from the customer dump, one row per -bucket, including a 4 min 18 s absence carrying no marker. +bucket, including a 4 min 18 s absence carrying no marker. This route and its one tick are the history +read's; the poll maps the same mark to a break 1 ms past the row ("Realtime"). A null `v` sets the same flag, because a bucket is the finest resolution a read has: `min(v)`, `max(v)` and the newest-value aggregate all skip a null, so without the flag a bucket holding data @@ -979,10 +980,20 @@ catalogue. `Dispose` completes the subject. - **The sequence never completes and never faults.** A query error logs, drops that tick's rows and leaves the observable running. - **No timestamp is emitted at or before the last one already delivered** (`DA-7`). -- **A row whose `v` is null is dropped, and `lastSeen` still advances past it.** `Sample.Value` is - non-nullable. -- **A `q = 32` row opens no gap here.** `Sample` carries no null to rebuild one with; a break at the - live edge draws as a held line until the next history read covers it. +- **A row whose `v` is null is dropped, and `lastSeen` still advances past it.** The history path flags + such a bucket as a gap instead ("Quality and gaps"), so a null run the poll reads draws as a held line + until the next history read covers it. `v` was never null in the measured archive + (`scada-archive.md#quality-and-gaps`), so the poll writes a live break for a `q = 32` row only. +- **A `q = 32` row emits its value and then a break 1 ms later.** The break is a `Sample` with a null + `Value` at the row's UTC timestamp plus `RealtimePoll._breakAnchorStep`, added on the UTC side as + `BucketedRowFold` adds its tick. A `q = 32` row whose `v` is null emits the break alone. The step is + 1 ms because the chart's X axis keeps milliseconds only (`LocalTimeAxis.ToAxis` is `ToOADate`): one + tick, the history path's anchor, maps to the marker's own X, and `EnvelopeLine` rejects an append at or + before the last X (`DA-7`), so the break would be dropped. The archive's `t` is `timestamp(3)` and a + change pair is about 100 ms apart (`scada-archive.md#write-behavior`), so no real row lands on the + break's millisecond. `RealtimePollReadTests` pins both row shapes, and + `TrendPenStateTests.Realtime_BreakOneTickAfterTheValue_IsRejectedAndOneMillisecondAfterIt_AddsANaNColumn` + pins the step. ### The connection state the poll reports diff --git a/docs/architecture/testing-strategy.md b/docs/architecture/testing-strategy.md index 21e81ab..6e48430 100644 --- a/docs/architecture/testing-strategy.md +++ b/docs/architecture/testing-strategy.md @@ -44,6 +44,7 @@ and built from the commit under test. Its value is diagnosis. | The shipped configuration set | `SemiPlot.Tests.Unit/DeliveredConfigurationTests.cs` over `ConfigFiles/**`, linked into the output directory by `SemiPlot.Tests.Unit.csproj` | | The live theme's pipeline, its watcher and `App`'s wiring of both | `SemiPlot.Tests.Unit/UI/Settings/AppSectionWatcherTests.cs`, `UI/Startup/AppConfigurationTests.cs`; the wall-clock exception below | | A pen's buffer and its derived state | `SemiPlot.Tests.Unit/UI/Chart/TrendPenStateTests.cs` | +| The realtime append-or-fold rule per layer, and the live break | `SemiPlot.Tests.Unit/UI/Chart/ChartRealtimeApplierTests.cs` | Shared helpers sit beside the tests that use them, one home each: `ResultText` (a failed result read in an assertion), `TemporaryDirectory`, `FixedClock`, `HeadlessInput` (the four gestures), `ViewProbe` (find, measure, @@ -215,13 +216,13 @@ at the far end. Its value is an existence proof that parts which each pass their connect. Its cost is that a failure names nothing, so they stay few and thin — the seams carry the coverage, and the journeys only prove the chain is closed. -`SemiPlot.Tests.Integration/Journeys/` holds them — three tests in two classes over a container-backed +`SemiPlot.Tests.Integration/Journeys/` holds them — four tests in two classes over a container-backed archive, each closing a chain whose halves are already proved apart: | Test | Chain it closes | The seam tests it joins | | --- | --- | --- | | `BreakRenderArchiveJourneyTests` | a seeded break reaches the canvas: archive → `AddPostgresData` → `TrendCoordinator` → `TrendChartViewModel` → the pixels the rasteriser leaves blank | `PostgresHistoryReadTests` counts the fold's NaN anchor; `ChartGapRenderTests` measures the blank pixels a NaN column leaves | -| `LiveEdgeArchiveJourneyTests` | two: a row appended while the application runs reaches the chart's live edge and reaches it once; and rows on a variable of its own reach the chart without breaking a pen that has no sample at that timestamp | `RealtimeSubscriptionTests` asserts the first rule over the provider alone; `TrendCoordinatorTests` and `TrendChartViewModelTests` cover the per-variable batch shape above it | +| `LiveEdgeArchiveJourneyTests` | three: a row appended while the application runs reaches the chart's live edge and reaches it once; rows on a variable of its own reach the chart without breaking a pen that has no sample at that timestamp; and a `q = 32` row written while the application runs reaches the chart as its value and then a `NaN` column 1 ms past it, while `CurrentValue` keeps the value | `RealtimeSubscriptionTests` asserts the first rule over the provider alone; `TrendCoordinatorTests` and `TrendChartViewModelTests` cover the per-variable batch shape above it; `RealtimePollReadTests` asserts the value-then-break pair, and `ChartRealtimeApplierTests` and `TrendPenStateTests` cover the append-never-fold rule and the 1 ms step | The composed path adds what those seam tests cannot see: the coordinator's buffering and folding, the chart view model's applier, the navigation controller. Each can lose a sample or replay one without diff --git a/docs/architecture/trend-feature-spec.md b/docs/architecture/trend-feature-spec.md index 91295b0..3b65032 100644 --- a/docs/architecture/trend-feature-spec.md +++ b/docs/architecture/trend-feature-spec.md @@ -199,7 +199,7 @@ through `TrendChartViewModel.SetActivePen`; the picked line is drawn 2 px wide a **Note.** Closes the most real correctness risk (b)7. ### DA-8 — Quality and gaps (SHOULD) -**Definition.** The archive marks a break with `q = 32` on the last sample before it and `q = 16` on the first sample after it; absence of rows without a preceding `q = 32` means the value did not change and the line continues horizontally (`scada-archive.md`). The provider maps a break to a NaN anchor in the envelope along the whole path from PostgreSQL — from the archive's own `q` column, never from a synthesised marker. Any other quality code is drawn as an ordinary point; no such code appeared in either measured archive (`scada-archive.md#quality-and-gaps`). +**Definition.** The archive marks a break with `q = 32` on the last sample before it and `q = 16` on the first sample after it; absence of rows without a preceding `q = 32` means the value did not change and the line continues horizontally (`scada-archive.md`). The provider maps a break to a NaN anchor in the envelope along the whole path from PostgreSQL — from the archive's own `q` column, never from a synthesised marker. At the live edge the poll emits the marker's value and then a null sample 1 ms later, which the chart appends as a NaN column (`data-integration.md#realtime`). Any other quality code is drawn as an ordinary point; no such code appeared in either measured archive (`scada-archive.md#quality-and-gaps`). **Acceptance.** DB: a marked break produces a line break, while a long run without rows produces a continuous horizontal line; the readout inside a break is "no data". **Note.** Closes gap (b)10. @@ -220,7 +220,7 @@ through `TrendChartViewModel.SetActivePen`; the picked line is drawn 2 px wide a **Acceptance.** Core/test: `TrendNavigationModel` auto-detaches sticky when the live edge leaves the window; jump-to-now restores it. UI: in realtime the window rides the data, a manual shift stops the follow. ### RT-3 — Folding realtime on coarse layers (SHOULD) -**Definition.** On coarse aggregation layers realtime folds into the last column (`FoldRealtime` widens Min/Max) rather than drawing raw points over the aggregate. +**Definition.** On coarse aggregation layers realtime folds into the last column (`FoldRealtime` widens Min/Max) rather than drawing raw points over the aggregate. A live break (a null value) is never folded: it is appended as a NaN column at every layer (`charting.md`, Realtime append / live-edge join). **Acceptance.** Core/test: feeding a point on a coarse layer widens the Min/Max of the last column without adding a new raw point. ### RT-4 — Update/render parameters (NICE) diff --git a/docs/architecture/trend-interaction.md b/docs/architecture/trend-interaction.md index dde9258..af7fac2 100644 --- a/docs/architecture/trend-interaction.md +++ b/docs/architecture/trend-interaction.md @@ -67,16 +67,20 @@ operator interaction. - **Decimation envelope contract:** history record per pen = ascending `X[]` + `Min[]` + `Max[]` + center `Y[]`; realtime stays single-value `double?[]` (null = gap); rendered as one `EnvelopeLine` polyline (see Renderer). - *As-built note:* realtime is `double`, not `double?`: `PenRealtimeValues` carries each pen's own - timestamps and values, and the live edge has no null, so every gap the chart draws comes from the - history path (charting.md, "Gaps via NaN"). + *As-built note:* realtime is `double?` on each pen's own timestamps, not on a shared column: + `PenRealtimeValues` carries no entry at a timestamp the pen did not report, and a null is a break the + provider reported. The poll writes one 1 ms after a `q = 32` row (data-integration.md#realtime); every + other gap the chart draws comes from the history path (charting.md, "Gaps via NaN"). - **Δ cursors:** Δy is reported only for the **active/selected pen** (pens share X but have independent Y scales, so a global Δy is meaningless). - **Bad quality → gap:** OPC bad-quality is mapped to `null` at the `IDataProvider` boundary, reusing the null = gap path; a distinct value-present-but-bad-quality flag is deferred. - *As-built note:* nothing maps to `null`. The archive marks the last sample before a break with - `q = 32`, the server closes the bucket there, and `BucketedRowFold` writes a `NaN` column one tick after - it. `q = 16` takes no branch: the next bucket resumes the line (data-integration.md#quality-and-gaps). + *As-built note:* no OPC quality exists yet, so no bad-quality status maps to `null`. The archive marks + the last sample before a break with `q = 32`, and that mark maps to a break. On the history path the + server closes the bucket there, and `BucketedRowFold` writes a `NaN` column one tick after it + (data-integration.md#quality-and-gaps). At the live edge the poll emits the row's value and then a null + `Sample` 1 ms after it, which the chart appends as a `NaN` column (data-integration.md#realtime). + `q = 16` takes no branch: the next bucket or sample resumes the line. - **Real-time return:** "jump to real-time" re-attaches sticky immediately; now-marker at the **right edge** (centering is at most a transient transition animation). - **Many-axes management:** **single active Y axis + per-pen autoscale** (legacy SCADA model); @@ -612,11 +616,12 @@ and the render budget (§RT-4). The as-built rationale and mechanics: The column count carries **its own deadband** on the same grounds: one quantisation step doubles or halves every ceiling, so a pixel of jitter across a boundary must not move it. - **Gaps come from the break markers, not from empty spans** (§DA-5). The archive writes a row only - when a value changes, so a span with no row is no gap by itself. The provider anchors a `NaN` column + when a value changes, so a span with no row is no gap by itself. A history read anchors a `NaN` column by one route at every layer: the server closes a bucket at a `q = 32` marker, and `BucketedRowFold` writes the `NaN` column one tick after that bucket, so the line breaks there instead of bridging the - missing span with a straight line (see `data-integration.md`, Quality and gaps). No decimator runs - on the client. + missing span with a straight line (see `data-integration.md`, Quality and gaps). At the live edge the + poll emits a `q = 32` row's value and then a null sample 1 ms later, which the chart appends as a `NaN` + column at every layer (`data-integration.md#realtime`). No decimator runs on the client. - **Use a min/max-per-pixel envelope, not plain sampling.** Plain decimation aliases away spikes (AVEVA warns of exactly this). Retain min AND max per pixel column so spikes survive (M4: min/max/first/last per column → visually lossless; MinMaxLTTB for speed; Power Chart "MinMax"). diff --git a/docs/plans/backlog.md b/docs/plans/backlog.md index ebeec4c..910bd1c 100644 --- a/docs/plans/backlog.md +++ b/docs/plans/backlog.md @@ -13,13 +13,6 @@ each is a scoped future task. missing is a real archive: the decimation and extent paths have been exercised against the seeded bench and against a customer dump, not against production volumes on a running tool. -- **A break that opens at the live edge draws as a held line.** `Sample` carries no null channel, so - the realtime poll emits a `q = 32` row as an ordinary sample and cannot reconstruct the gap the way - `BucketedRowFold` does from a bucket's breaks flag. The line holds at the last value until the next - history read covers that span and redraws it as a break. Closing it means giving the realtime seam a - way to carry an absence — a nullable sample, or a separate break signal — which is a change to - `IDataProvider` and to every consumer of `RealtimeBatch`, so it is its own task. - - **Two settings saves can lose one of them.** `SettingsSave.Save` reads a section at stage time and promotes its copy later; a save from another process that replaces the same file in between is overwritten (`overview.md#the-settings-window`). With the live theme the loss shows: every window diff --git a/docs/plans/completed/20261019-live-edge-break.md b/docs/plans/completed/20261019-live-edge-break.md new file mode 100644 index 0000000..05114f2 --- /dev/null +++ b/docs/plans/completed/20261019-live-edge-break.md @@ -0,0 +1,423 @@ +# Live edge break + +## Overview + +- The realtime seam carries an absence: a `Sample` with a null `Value` is a break for that pen at that + moment. `TrendCoordinator` keeps it in the batch, and the chart appends a `NaN` column there at every + layer. +- The production writer is the existing poll: a `q = 32` row becomes its value followed by a break 1 ms + later. This closes the backlog item "A break that opens at the live edge draws as a held line" + (`docs/plans/backlog.md:16-21`). Today the line holds at the last value until the next history read + covers the span. +- Later slices write breaks on OPC session loss, on a `Bad` status and ahead of a pen's first live sample. + A break that carries its own timestamp and no value serves all three. None of them is built here. + +## Context (from discovery) + +- Roadmap: docs/plans/roadmaps/20261019-opc-live-edge-roadmap.md — slice live-edge-break +- Verified at `38de80f`. +- Vocabulary: + - `Sample(int PenId, DateTime TimestampUtc, double Value)` (`SemiPlot/SemiPlot.Core/Trends/Sample.cs:3`). + - `PenRealtimeValues.Values` is `IReadOnlyList`, and its summary states that a null "would draw + a break the archive never recorded" (`SemiPlot/SemiPlot.Core/Trends/RealtimeBatch.cs:3-10`). + - `IDataProvider.Subscribe` returns `IObservable>` + (`SemiPlot/SemiPlot.Core/Data/IDataProvider.cs:10`). +- Producer `RealtimePoll`: + - Counts `q = 32` rows for the debug line only (`SemiPlot/SemiPlot.DataSource.Postgres/RealtimePoll.cs:34-36`, + `:166-169`). + - Drops a null-`v` row (`:171-178`) and emits every other row as an ordinary `Sample` (`:180`). + - The debug line (`:185-192`) carries the structured placeholder `{RowCount}`, the number of rows read. + `RealtimePollReadTests.cs:148` and `RealtimeSubscriptionTests.cs:120` select entries by it. +- Hub `TrendCoordinator`: + - `BuildRealtimeBatch` takes the ascending, distinct union of timestamps (`SemiPlot/SemiPlot.UI/Bridge/TrendCoordinator.cs:149-161`). + - `BuildPenValues` orders each pen's samples by timestamp and copies `Value` with `Array.ConvertAll` + (`:163-172`). Under a `double?` `Value` it infers `double?[]` and compiles unchanged. + - The buffer window is 100 ms (`TrendCoordinator.BatchWindow`, `:14`). The unit tests run the fake at + `ChartTestBuilder.RealtimeInterval` = 10 ms (`ChartTestBuilder.cs:27`), so one window holds about ten ticks. +- Chart: + - `ChartRealtimeApplier.ApplyBatch` folds at coarse layers and appends at `Raw`, on each pen's own + timestamps (`SemiPlot/SemiPlot.UI/Chart/ChartRealtimeApplier.cs:19-42`). + - The fold flag comes from `TrendChartViewModel.ApplyRealtimeBatch` (`SemiPlot/SemiPlot.UI/Chart/TrendChartViewModel.cs:797`). + - `TrendPenState.AppendRealtime(DateTime, double?)` already appends `NaN` for a null and keeps + `CurrentValue` (`SemiPlot/SemiPlot.UI/Chart/TrendPenState.cs:77-83`). Only `Realtime_NullSample_AppendsNaNGap` + calls it with a null (`SemiPlot/SemiPlot.Tests.Unit/UI/Chart/TrendPenStateTests.cs:56-65`). + - `FoldRealtime(DateTime, double)` takes no null (`TrendPenState.cs:86-92`). +- `EnvelopeLine` invariants this slice keeps: + - `AppendAfterLastColumn` rejects `X <= last X` (DA-7), accepts any column on an empty list, and trims + 10 000 columns past the 100 000 cap (`SemiPlot/SemiPlot.UI/Chart/EnvelopeLine.cs:14-15`, `:271-286`). + - `FoldIntoLastColumn` returns false on an empty list, appends past a gap, and otherwise takes + `Math.Min`/`Math.Max` over the last column (`:226-252`). So a `NaN` sample folded into a real column turns + it into `NaN`. + - `ReplaceColumns` keeps the live columns past the new history end and drops those past the requested + end (`:195-214`). It reads no `Min` or `Max`, so a `NaN` column follows the same rule as any live column. + - `EnvelopePath.Build` starts a new segment after a `NaN` column (`SemiPlot/SemiPlot.UI/Chart/EnvelopePath.cs:40-52`). +- Axis resolution: `LocalTimeAxis.ToAxis` is `ToLocalTime().ToOADate()` (`SemiPlot/SemiPlot.UI/Chart/LocalTimeAxis.cs:8-13`). + Verified 2026-10-19 in PowerShell: `ToOADate` maps `t`, `t + 1 tick` and `t + 9999 ticks` to the same + double, and `t + 1 ms` to a different one. +- The history path anchors its `NaN` one tick after a break-closing bucket + (`SemiPlot/SemiPlot.DataSource.Postgres/BucketedRowFold.cs:51-58`). That works only because + `ReplaceColumns` inserts without an X check. This slice does not touch the history path. +- Consumers of `Sample` / `RealtimeBatch` / `PenRealtimeValues`: + - Production: `RealtimePoll`, `PostgresDataProvider.Subscribe` (`SemiPlot/SemiPlot.DataSource.Postgres/PostgresDataProvider.cs:94-98`), + `TrendCoordinator`, `ChartRealtimeApplier`, `TrendChartViewModel`. + - Tests: `FakeDataProvider.BuildLiveEdge` (`SemiPlot/SemiPlot.Tests.Unit/UI/Bridge/FakeDataProvider.cs:202-222`), + `TrendCoordinatorTests`, `TrendChartViewModelTests`, `TrendChartCatalogueTests`, + `LiveEdgeArchiveJourneyTests`, `RealtimeSubscriptionTests`, `RealtimeSubscriptionDisposalTests`. + - No test constructs `PenRealtimeValues` or `RealtimeBatch`. The test reads of `Value` and `Values` + (`RealtimePollReadTests.cs:72,176`, `LiveEdgeArchiveJourneyTests.cs:238`) compile under `double?`. + - `ChartHistoryRequestDebouncer.cs:56` uses the Rx `Sample` operator, not the record. +- Test seams: + - `RealtimeArchive.AppendAsync(database, penId, archiveLocal, double? value, int quality, ...)` + (`SemiPlot/SemiPlot.Tests.Integration/RealtimeArchive.cs:82-88`). `RealtimePollReadTests` imports it with + `using static`. `LiveEdgeArchiveJourneyTests` does not, and it has a private `AppendAsync` of its own + (`Journeys/LiveEdgeArchiveJourneyTests.cs:195`) that writes `OrdinaryQuality` for every pen. + - `ArchiveRow.LastBeforeBreakQuality = 32` (`SemiPlot/SemiPlot.Tools.ArchiveSeeder/ArchiveRow.cs:11`). + - The current pin of the defect is `RealtimePollReadTests.ARowMarkingTheLastSampleBeforeABreakEmitsAnOrdinarySample` + (`SemiPlot/SemiPlot.Tests.Integration/RealtimePollReadTests.cs:154-178`). It asserts a single ordinary sample. + - A coarse history column at the fake's live timestamps: `TrendChartViewModelTests.AFoldIntoACoarseEnvelopeKeepsThePrefetchedBand` + (`TrendChartViewModelTests.cs:1014-1037`) lands `EnvelopeEndingAtTheRealtimeEpoch()` (`:2659-2670`) at a + coarse layer. `Coordinator_CoarseLayerRealtime_FoldsInsteadOfGrowingColumns` (`:975-990`) lands no history, + so its line is empty. `CreateViewModel`'s default window (June 2026) lies after every fake live + timestamp (`FakeDataProvider.RealtimeEpoch` = 2026-01-01). +- The demo writer writes no live break (`SemiPlot/SemiPlot.Tools.ArchiveSeeder/LiveTailGenerator.cs:27-28`). + The AppHost stand therefore shows a live break only after a hand-inserted `q = 32` row. + +## Development Approach + +- **testing approach**: Regular (code first, then tests in the same task). Task 3 runs its new tests once + before the poll change to confirm that they fail. +- Complete each task fully before moving to the next. Make small, focused changes. +- Every task includes new or updated tests for the code it changes. +- All tests pass before the next task starts. +- Update this plan file when scope changes during implementation. +- `dotnet build SemiPlot.slnx` must hold at zero warnings (`TreatWarningsAsErrors`). `dotnet format + SemiPlot.slnx --verify-no-changes` and `dotnet terse` must pass over the touched `.cs` files. + +## Testing Strategy + +- Unit tests (`SemiPlot.Tests.Unit`, no container): + - `TrendCoordinatorTests`: the break survives the buffer window and the batch build. + - New `ChartRealtimeApplierTests`: the append-never-fold rule, at `Raw`, at a coarse layer, and on an + empty coarse line. + - `TrendPenStateTests`: the axis resolution behind the 1 ms step. + - `TrendChartViewModelTests`: the coarse-layer path end to end over `FakeDataProvider`. +- Integration tests (`SemiPlot.Tests.Integration`, container required): + - `RealtimePollReadTests`: a `q = 32` row produces its value and a break. + - `LiveEdgeArchiveJourneyTests`: the break reaches the chart's column list. +- Every new test class carries the `Component`, `Area` and `Category` traits. AwesomeAssertions throughout. +- No UI e2e suite exists beyond the headless tests above. + +## Acceptance Evidence + +- Defect at HEAD: `RealtimePollReadTests.ARowMarkingTheLastSampleBeforeABreakEmitsAnOrdinarySample` passes. + It asserts that a `q = 32` row yields exactly one sample (`RealtimePollReadTests.cs:174`). No column of + `NaN` reaches the live edge. +- Guard: after Task 1 and before the poll change, the renamed poll test and the new journey test fail + (Task 3, first step). +- After the change, these commands pass: + +```powershell +dotnet test SemiPlot/SemiPlot.Tests.Unit/SemiPlot.Tests.Unit.csproj --filter "FullyQualifiedName~TrendCoordinatorTests|FullyQualifiedName~ChartRealtimeApplierTests|FullyQualifiedName~TrendPenStateTests|FullyQualifiedName~TrendChartViewModelTests" +dotnet test SemiPlot/SemiPlot.Tests.Integration/SemiPlot.Tests.Integration.csproj --filter "FullyQualifiedName~RealtimePollReadTests|FullyQualifiedName~LiveEdgeArchiveJourneyTests|FullyQualifiedName~RealtimeSubscriptionTests" +``` + +- What the integration run must show: + - `RealtimePollReadTests.ARowMarkingTheLastSampleBeforeABreakEmitsItsValueAndThenABreak` reads exactly + two samples for the `q = 32` row: `(pen, ToUtc(t), 11.25)` and `(pen, ToUtc(t) + 1 ms, null)`, in that order. + - The new journey test finds, in that pen's `SnapshotColumns()`, a real column at `ToAxis(ToUtc(t))` + followed by a `NaN` column at `ToAxis(ToUtc(t) + 1 ms)`, and `CurrentValue` equals the row's value. + - ⚠️ A history read that lands before the snapshot puts its anchor `NaN` at `ToAxis(ToUtc(t))` between the + two, so the journey asserts the last real column and the last column rather than the last two (Task 4). +- Integration runs need a container runtime. ASSUMPTION: Docker is available on the executing machine. + The CI `linux` leg runs the integration project in any case. + +## Progress Tracking + +- Mark completed items with `[x]` immediately when done. +- Add newly discovered tasks with ➕ prefix. +- Document issues or blockers with ⚠️ prefix. +- Keep the plan in sync with the work done. + +## Solution Overview + +- **A break is a sample without a value.** `Sample.Value` becomes `double?`, and null means "no data from + this moment until the pen's next sample". `PenRealtimeValues.Values` becomes `IReadOnlyList`. + The later OPC writers have no value to attach a flag to: a session loss, and the break ahead of a pen's + first live sample. A break with a timestamp of its own serves them. The grey "no archive data" kind + belongs to slice live-hold-to-edge and is not added here. +- **The poll stamps the break 1 ms after the marker.** + - The archive's `t` is `timestamp(3)`, and the axis keeps milliseconds only (`LocalTimeAxis.cs:12`, verified + above). So 1 ms is the smallest step that both sides tell apart. + - At `+1 tick`, the history path's anchor, the break maps to the marker's X. `AppendAfterLastColumn` + then rejects it, and the defect stays with every poll-level test green. + - DA-7 in `AppendAfterLastColumn` stays unchanged. + - A real row at exactly the marker's archive-local millisecond plus one would share the break's + timestamp, and the chart would reject it. The archive writes none there: a change pair is about 100 ms + apart (`docs/architecture/charting.md:73-75`), and a project stop and start are seconds apart. +- **A break is always appended, never folded.** `ChartRealtimeApplier` sends a null to + `TrendPenState.AppendRealtime(ts, null)` ahead of the append-or-fold choice. The rule lives in that one + place, and `FoldRealtime` keeps its non-null `double`. + - At a coarse layer the folded column keeps its bucket X, which lies before the break. So the `NaN` + column lands past it, and the next sample's fold opens a new column past the gap + (`EnvelopeLine.cs:237-241`), as it does today after a history gap. + - `CurrentValue` stays the last real value, because `AppendRealtime` sets it only when `value.HasValue` + (`TrendPenState.cs:79-82`). + - On a coarse line with no column yet (a pen seated before its first history read lands), the break + still appends its `NaN` column, and the folds after it open a live column past it. Today such a line + folds nothing until history lands. The plan accepts this: the next history read replaces the span under + `ReplaceColumns`' kept-tail rule, and an exception for the empty line would be a second rule in the + applier. `charting.md` states it (Task 5), and `ChartRealtimeApplierTests` pins it (Task 1). +- **The coordinator needs no new rule and no code change.** Its union and per-pen ordering already sort the + break after its value, and `Array.ConvertAll` yields `double?[]`. The break's timestamp joins the union, + so `OnLiveEdge` advances 1 ms past the marker. +- **Asymmetry kept on purpose.** A null-`v` row is still dropped by the poll. The history path flags + such a bucket as a gap (`docs/architecture/data-integration.md:934-941`), and `v` was never null in the + measured archive (`docs/architecture/scada-archive.md:245`). Only a `q = 32` row writes a live break in + this slice. A `q = 32` row writes its break even when its `v` is null; only the value sample is skipped. + That row shape was never measured either; it is pinned because it is the natural branch of the code. + +## Technical Details + +- `Sample.cs`: `public sealed record Sample(int PenId, DateTime TimestampUtc, double? Value);` with a one-line + summary: a null value is a break for the pen from this moment. +- `RealtimeBatch.cs`: `PenRealtimeValues(int PenId, IReadOnlyList TimestampsUtc, IReadOnlyList Values)`. + The summary states: a null is a break the provider reported; a pen carries no entry, null or real, at + a union timestamp it did not report. +- `TrendCoordinator.BuildPenValues`: no code change. The comment at `:163` is reworded: no filler at another + pen's timestamp; a null comes only from the provider. +- `ChartRealtimeApplier.ApplyBatch`, per index: + - `value is null` → `state.AppendRealtime(ts, null)`. + - else `foldIntoColumn` → `state.FoldRealtime(ts, value.Value)`. + - else `state.AppendRealtime(ts, value)`. + - The comment at `:28-29` is reworded the same way. +- `RealtimePoll.ReadSamplesAsync`, per row: + - `isBreakMark = reader.GetInt32(3) == LastBeforeBreakQuality`. + - When `v` is not null, add `Sample(id, utc, v)`. + - When `isBreakMark`, add `Sample(id, utc + BreakAnchorStep, null)`. The step is added on the UTC side, + as `BucketedRowFold` adds its tick. + - A named constant holds the step: `BreakAnchorStep = TimeSpan.FromMilliseconds(1)`, with a one-line + comment pointing at `docs/architecture/data-integration.md#realtime`. + - ⚠️ As built the step is `private static readonly TimeSpan _breakAnchorStep` (`RealtimePoll.cs:42`): a + `TimeSpan` cannot be `const`, and `.editorconfig` requires `_camelCase` for private static readonly + fields. `data-integration.md#realtime` cites that name; `BreakAnchorStep` below means this field. + - The comment at `:34-35` ("read for the debug line only") and the one at `:171-172` are rewritten. + - The debug line keeps the `{RowCount}` placeholder with its name and meaning (rows read), because two + tests select by it. Its other counts name what the tick emitted: value samples, rows dropped for a null + value, and breaks. +- `FakeDataProvider`: a one-shot knob, `BreakRealtimePenOnce(int penId)`. It stores the pen in a private + nullable field. The next tick emits that pen's value and then `Sample(id, t + 1 ms, null)`, as the poll + does, and clears the field; later ticks are ordinary. The existing knobs (`StaggerRealtimeTimestamps`, + `PoisonRealtimeWindow`) stay as they are. + +## What Goes Where + +- **Implementation Steps**: the code, tests and documentation in this repository. +- **Post-Completion**: the manual check on the AppHost stand, and the remaining roadmap slices. + +## Implementation Steps + +### Task 1: Carry a break from `Sample` through the batch to the applier + +**Files:** +- Modify: `SemiPlot/SemiPlot.Core/Trends/Sample.cs` +- Modify: `SemiPlot/SemiPlot.Core/Trends/RealtimeBatch.cs` +- Modify: `SemiPlot/SemiPlot.UI/Bridge/TrendCoordinator.cs` +- Modify: `SemiPlot/SemiPlot.UI/Chart/ChartRealtimeApplier.cs` +- Modify: `SemiPlot/SemiPlot.Tests.Unit/UI/Bridge/FakeDataProvider.cs` +- Modify: `SemiPlot/SemiPlot.Tests.Unit/UI/Bridge/TrendCoordinatorTests.cs` +- Create: `SemiPlot/SemiPlot.Tests.Unit/UI/Chart/ChartRealtimeApplierTests.cs` + +- [x] Make `Sample.Value` `double?` and `PenRealtimeValues.Values` `IReadOnlyList`, and rewrite both + summaries (Technical Details). Confirm that the build fails at exactly `ChartRealtimeApplier.cs:34`: + `FoldRealtime` takes a `double`. That is the compiler guard. `TrendCoordinator.cs:171` + (`ConvertAll` now yields `double?[]`), `ChartRealtimeApplier.cs:38` (`AppendRealtime` takes `double?`), + `RealtimePoll.cs:180` and `FakeDataProvider.cs:216` (both convert to `double?`) still compile. +- [x] Reword the comment at `TrendCoordinator.cs:163`. No code there changes. +- [x] Put the null branch in `ChartRealtimeApplier.ApplyBatch` ahead of the fold-or-append choice, and + reword its comment. +- [x] Add `BreakRealtimePenOnce` to `FakeDataProvider` (Technical Details). +- [x] Test in `TrendCoordinatorTests`: call `BreakRealtimePenOnce(FirstPen.PenId)`, then advance one buffer + window. The first pen's `PenRealtimeValues` holds exactly one null. Its timestamp is 1 ms after the + entry before it, and that entry is a real value. `batch.Timestamps` holds the null's timestamp, and + `Values.Count == TimestampsUtc.Count` still holds. The second pen carries no null. A later window + carries no null for either pen. +- [x] Test in `ChartRealtimeApplierTests` (`Component=UI`, `Area=Chart`, `Category=Unit`, plain `[Fact]`, as + `TrendPenStateTests` builds `TrendPenState` under `[Fact]`), built over `new ChartNavigationController()` + and `TrendPenState` instances: + - At `foldIntoColumn: false`, a `[value, null]` pair on `[t, t + 1 ms]` adds a real column and then a + `NaN` column. + - At `foldIntoColumn: true`, after a history column, the value folds into it, and the null adds a + `NaN` column past it. The last real column keeps its `Min`/`Max`, not `NaN`. + - At `foldIntoColumn: true` on a pen with no column, `[value, null, value2]` on `[t, t + 1 ms, t + 1 s]` + leaves two columns: `NaN`, then a real column at `t + 1 s`. `CurrentValue` equals `value2`. This + pins the accepted empty-line behaviour (Solution Overview). + - In the first two cases `CurrentValue` equals the value. +- [x] Run `dotnet build SemiPlot.slnx` and `dotnet test SemiPlot/SemiPlot.Tests.Unit/SemiPlot.Tests.Unit.csproj`. + Both must pass before Task 2. + +### Task 2: Pin the break column through the chart + +**Files:** +- Modify: `SemiPlot/SemiPlot.Tests.Unit/UI/Chart/TrendPenStateTests.cs` +- Modify: `SemiPlot/SemiPlot.Tests.Unit/UI/Chart/TrendChartViewModelTests.cs` + +No production change is expected in this task. A red `TrendPenStateTests` case means the axis resolution +differs from the Context; a red view-model test after a correct applier test means the test's history setup +is wrong. Fix neither by changing `EnvelopeLine`. + +- [x] `TrendPenStateTests`: after `AppendRealtime(t, 5.0)`, `AppendRealtime(t + 1 tick, null)` is rejected + (one column) and `AppendRealtime(t + 1 ms, null)` adds a `NaN` column. `CurrentValue` stays `5.0`. + This pins the reason for `BreakAnchorStep` without a container. +- [x] `TrendChartViewModelTests`: model the setup on `AFoldIntoACoarseEnvelopeKeepsThePrefetchedBand` + (`TrendChartViewModelTests.cs:1014-1037`): `AddPen`, `TrackDataExtents`, `ZoomOutAndIssueTheCoarseRead`, + `ReleaseAndAwait(... EnvelopeEndingAtTheRealtimeEpoch())` and `LandTheMarginReadFromTheReleasedGate`. + Do not use the default June 2026 window: history there lies past every fake live timestamp, and DA-7 + rejects the break. Then call `BreakRealtimePenOnce(FirstPen.PenId)` and advance one buffer window. + The pen's columns grow by exactly two: the column at `RealtimeEpoch` is widened and not `NaN`, a `NaN` + column follows it, and a real column follows that. `CurrentValue` equals the last column's `Center`. +- [x] Run `dotnet test SemiPlot/SemiPlot.Tests.Unit/SemiPlot.Tests.Unit.csproj`. It must pass before Task 3. + +### Task 3: Make the poll write a break after a `q = 32` row + +**Files:** +- Modify: `SemiPlot/SemiPlot.DataSource.Postgres/RealtimePoll.cs` +- Modify: `SemiPlot/SemiPlot.Tests.Integration/RealtimePollReadTests.cs` +- Modify: `SemiPlot/SemiPlot.Tests.Integration/Journeys/LiveEdgeArchiveJourneyTests.cs` + +- [x] Write the tests first and confirm the guard: + - Rename `ARowMarkingTheLastSampleBeforeABreakEmitsAnOrdinarySample` to + `ARowMarkingTheLastSampleBeforeABreakEmitsItsValueAndThenABreak`. It asserts exactly two samples in + order: `11.25` at `ToUtc(appended)`, and `null` at `ToUtc(appended) + 1 ms`. + - Add `ARowMarkingTheLastSampleBeforeABreakWithANullValueEmitsOnlyTheBreak`: a `q = 32` row with `v` + null yields one sample, `null` at `ToUtc(appended) + 1 ms`. + - Add a journey to `LiveEdgeArchiveJourneyTests`, next to + `ARowWrittenAfterStartupReachesTheChartOnceAndMovesItsLiveEdge` (`:33`) and using its stand. Take the + batch gate, then write one row with the qualified call + `RealtimeArchive.AppendAsync(Database, pen.PenId, seededLastLocal.AddSeconds(1), value, ArchiveRow.LastBeforeBreakQuality, TestContext.Current.CancellationToken)`; + the class's own `AppendAsync` hides the unqualified name. Its partition step is `IF NOT EXISTS`, so the + template clone accepts it (`RealtimeArchive.cs:94-96`). Await the batch. The pen's last two columns + must be a real column at `LocalTimeAxis.ToAxis(ToUtc(t))` and a `NaN` column at + `LocalTimeAxis.ToAxis(ToUtc(t) + 1 ms)`. `CurrentValue` must equal the row's value. + - Run the integration filter below. The three new or renamed tests must fail: one sample for the + `q = 32` row, none for the null-`v` row, and no `NaN` column on the chart. +- [x] Change `ReadSamplesAsync` as given in Technical Details: add `BreakAnchorStep`, emit the break + after the value, emit it even when `v` is null, rewrite the two comments and the debug line, keep + `{RowCount}`. +- [x] Keep `ARowCarryingANullValueEmitsNoSampleReportsNoFaultAndIsNotReadAgain` (`:122-152`) unchanged + and green. It pins the null-`v` drop of a `q = 0` row, and it reads `{RowCount}`. + `RowsOnAVariableOfTheirOwnReachTheChartWithoutBreakingAnyPen` (`LiveEdgeArchiveJourneyTests.cs:93-121`) + must stay green: its rows are `q = 0`. +- [x] Run `dotnet test SemiPlot/SemiPlot.Tests.Integration/SemiPlot.Tests.Integration.csproj --filter + "FullyQualifiedName~RealtimePollReadTests|FullyQualifiedName~LiveEdgeArchiveJourneyTests|FullyQualifiedName~RealtimeSubscriptionTests"` + and the unit project. Both must pass before Task 4. + +### Task 4: Verify acceptance criteria + +- [x] Verify the Overview: a `q = 32` row reaches the chart as its value plus a `NaN` column at `Raw` + and at a coarse layer, and `CurrentValue` stays the last real value. Raw: the journey test (real poll) + and `ChartRealtimeApplierTests`; coarse: `ChartRealtimeApplierTests` and `TrendChartViewModelTests`. + ➕ The journey test failed 2 of 3 isolated runs: a history read landing before the snapshot puts its + anchor `NaN` at the marker's X. The test now asserts the last real column and the last column; 6 of 6 + isolated runs pass. +- [x] Verify the scope guard: `git diff origin/master --name-only` and `git status --porcelain --untracked-files=all` + list only the files named in the Files blocks of Tasks 1, 2, 3 and 5 and this plan. No file of the + history path (`BucketedRowFold.cs`, `ArchiveStatements.cs`, `EnvelopeLine.cs`, `EnvelopeSplice.cs`, + `ChartHistoryPlanner.cs`, `TrendPenState.cs`) appears. `git diff origin/master -- SemiPlot | grep -i -c "opc"` + prints `0`. (The untracked roadmap file belongs to the controller.) +- [x] Run `dotnet build SemiPlot.slnx` (0 warnings) and `dotnet format SemiPlot.slnx --verify-no-changes`. + Only NU1902 (MessagePack, `SemiPlot.AppHost`, present on master) remains. +- [x] Run `dotnet test SemiPlot.slnx`. Unit 2284/2284, Integration 195/195 at Task 4; after the review + fixes (one headless Raw test added) Unit 2285/2285, Integration 195/195. +- [x] Run `dotnet terse` over every touched `.cs` file. It must exit 0. + +### Task 5: [Final] Update documentation + +**Files:** +- Modify: `docs/architecture/charting.md` +- Modify: `docs/architecture/trend-interaction.md` +- Modify: `docs/architecture/data-integration.md` +- Modify: `docs/plans/backlog.md` + +- [x] `charting.md:64-67` ("Gaps via NaN"): a `NaN` comes from the history path or from a live break, + which `TrendPenState` appends for a null value. +- [x] `charting.md:108-119` (realtime append and live-edge join): a live break is always appended, at + every layer, and the fold after it opens a column past the gap. Replace "A pen with no column at all + folds nothing" (`:116-117`): a pen with no column folds no value, but a break appends its `NaN` column, + and the folds after it open a live column before any history read. +- [x] `charting.md:122-128` (a history apply keeps the live tail): a live break column follows the + kept-tail rule like any live column. After a read that ends at the marker, the pen can hold two + adjacent `NaN` columns, the history anchor at the marker's X and the live break 1 ms past it; + `Build` draws one break. +- [x] `charting.md:463-465` (the `ChartRealtimeApplier` entry): the null branch ahead of the fold-or-append choice. +- [x] `charting.md:769-778` (data contract, realtime): `double?` values, null = a break the provider + reported, and still no entry at a timestamp the pen did not report. +- [x] `trend-interaction.md:70-72` and `:77-79`: rewrite both as-built notes. Realtime is `double?`, and + a `q = 32` row maps to a live break 1 ms after the marker. +- [x] `data-integration.md:70` (the `Sample` row of the DTO table): a null `Value` is a break the provider + reported. +- [x] `data-integration.md:979-985`: + - Replace the `q = 32` bullet: the poll emits the value and then a break 1 ms later. Give the reason + for 1 ms: the axis keeps milliseconds, and DA-7 rejects an equal X. + - Reword the null-`v` bullet: the drop stays, the history path differs, and the reason is no longer + that `Sample.Value` is non-nullable. +- [x] `backlog.md:16-21`: remove the item "A break that opens at the live edge draws as a held line". +- [x] ➕ Review fixes: `docs/architecture/trend-feature-spec.md` (DA-8, RT-3), `docs/architecture/testing-strategy.md` + (the unit area table, the journey table) and `docs/architecture/bench.md` (a live break on the stand) name + the live break as well; none was in this task's Files block. + +## Post-Completion + +**Manual verification:** + +1. Start the demo stand: `dotnet run --project SemiPlot/SemiPlot.AppHost`. +2. With a pen in view at `Raw` and sticky, insert one row for that pen through `psql` as `scada_writer`: + `l = 0`, `q = 32`, `t` one second past the newest row. +3. The pen's line ends at that row's value and draws no segment to the demo writer's next row. + The line resumes at the next row. +4. Zoom out to a `Minute` layer and repeat. The break shows, and the line resumes in a column past it. +5. The legend's current value shows the marker's value, not a dash, until the next row arrives. + +**Remaining slices** (docs/plans/roadmaps/20261019-opc-live-edge-roadmap.md): + +- archive-state-from-catalogue: the archive connection state from the catalogue reads, bounded, + connection-class faults only. +- live-source-port: a Core live-source port, with the poll as its first implementation, behaviour-preserving. +- opc-demo-server: a Simple-Scada-shaped OPC UA server in the seeder's demo writer, `--archive-lag`, + package line 2.0.1. +- opc-live-source: the OPC UA live edge, the poll deleted, closes #116. +- opc-settings: the OPC block in the settings window. +- live-hold-to-edge: the grey no-archive-data break and the held line to the edge. + +**Executed by exec:** + +- branch: live-edge-break + +## Verify it yourself + +1. The defect on master. At `38de80f`, `RealtimePollReadTests.ARowMarkingTheLastSampleBeforeABreakEmitsAnOrdinarySample` + passes: a `q = 32` row yields one sample and no break. +2. The guard. On `live-edge-break`, put the master poll back and run the poll and journey filter: 4 of 11 + fail (checked 2026-10-19). They are the two `ARowMarkingTheLastSampleBeforeABreak...` poll tests, the break + journey, and `ARowCarryingANullValueEmitsNoSampleReportsNoFaultAndIsNotReadAgain`, which now reads the new + debug counts. The poll change and its tests landed together in `43c5868`, so no commit on the branch + carries the red state by itself. + + ```powershell + git show master:SemiPlot/SemiPlot.DataSource.Postgres/RealtimePoll.cs > SemiPlot/SemiPlot.DataSource.Postgres/RealtimePoll.cs + dotnet test SemiPlot/SemiPlot.Tests.Integration/SemiPlot.Tests.Integration.csproj --filter "FullyQualifiedName~RealtimePollReadTests|FullyQualifiedName~LiveEdgeArchiveJourneyTests" + git checkout -- SemiPlot/SemiPlot.DataSource.Postgres/RealtimePoll.cs + ``` + +3. The fix. On `live-edge-break` both Acceptance Evidence commands pass (container required for the second): + - the poll test reads `(pen, ToUtc(t), 11.25)` then `(pen, ToUtc(t) + 1 ms, null)`, and the debug entry + counts one row, one value, no drop, one break; + - the journey sees one batch `[t, t + 1 ms]` with values `[value, null]`, `Navigation.To` at `t + 1 ms`, + the last real column at `ToAxis(t)`, the last column `NaN` at `ToAxis(t + 1 ms)`, and `CurrentValue` equal + to the value. +4. Without a container, `TrendChartViewModelTests.ABreakAtRawAppendsANaNColumnOneMillisecondPastItsValue` and + `ABreakAtACoarseLayerAppendsANaNColumnAndTheNextFoldOpensAColumnPastIt` pin the same column shape at `Raw` + and at a coarse layer over `FakeDataProvider.BreakRealtimePenOnce`. +5. On screen: the Manual verification steps above. The demo writer plants no break, so the `q = 32` row is + inserted by hand (`docs/architecture/bench.md`, The demo writer). Before the fix the line holds at the + marker's value and joins the writer's next row; after it the line ends at the marker and resumes at the next + row, and the legend keeps the marker's value.