From 43d0df4cb27a9b757cfaad3fab92ffe9fd480ed2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Schultz=20Madsen?= Date: Thu, 17 Sep 2026 16:48:57 +0200 Subject: [PATCH 1/2] test(ci): shard every test class, and guard against silent omission `dotnet test` in this repo is fanned out across a matrix of shards, each a `--filter` naming the classes it owns, duplicated in dotnet-core-pr.yml and dotnet-core-master.yml. A class named in no filter is never executed: vstest reports nothing, the shard is green, the gate job is green, and the PR merges. The failure mode is total silence. 14 test classes were in that state. They are added to shards here, one class per shard chosen by cost: the three fixtures deriving from TestBaseSetup each start a MariaDB Testcontainer, so they go one apiece into light shards and nothing is added to shard `c`, which is already DB-dense (11 of its 12 classes derive TestBaseSetup). Root-cause evidence: 5 of the 14 arrived in a single commit, 861ea89e "test(timeplanning): comprehensive coverage for non-5-min stamp calculations", which merged green having executed none of its own tests. That is the shape of the problem - the omission is invisible at review time and stays invisible afterwards. CorruptedPauseIdRepairTests sat unsharded for months and, once added, turned out to contain a test that could never have passed. To stop it recurring, ShardCoverageTests reflects over the assembly for every class carrying [Test]/[TestCase]/[TestCaseSource] and fails the build unless each is named in BOTH workflows. It needs no database. It identifies our workflows by content rather than filename, because eform-angular-frontend - the host app this plugin is copied into in dev mode - ships files with the same two names; it strips YAML comments first, so a commented-out shard entry cannot count as coverage; and when it cannot find or parse the workflows it fails rather than passes, because a guard that gives up silently is no guard. Expect red shards. These classes have not run in months, or ever. A failure here is information, not an obstacle - no assertion was weakened, no [Ignore] added, and no production code touched in this change. Co-Authored-By: Claude Opus 5 --- .github/workflows/dotnet-core-master.yml | 27 +- .github/workflows/dotnet-core-pr.yml | 27 +- .../ShardCoverageTests.cs | 232 ++++++++++++++++++ 3 files changed, 272 insertions(+), 14 deletions(-) create mode 100644 eFormAPI/Plugins/TimePlanning.Pn/TimePlanning.Pn.Test/ShardCoverageTests.cs diff --git a/.github/workflows/dotnet-core-master.yml b/.github/workflows/dotnet-core-master.yml index 82a18989..21a6d570 100644 --- a/.github/workflows/dotnet-core-master.yml +++ b/.github/workflows/dotnet-core-master.yml @@ -253,23 +253,36 @@ jobs: strategy: fail-fast: false matrix: + # Shard letters are arbitrary; the cost is not. Every fixture deriving from + # TestBaseSetup starts its own MariaDB Testcontainer, and those containers + # dominate a shard's wall-clock. Shard 'c' is already the DB-dense one (11 of + # its 12 classes derive TestBaseSetup), so put new DB-backed fixtures in a + # lighter shard instead. Per-shard durations swing a lot run to run (runner + # contention, image pulls), so check the last green master run before choosing + # rather than trusting a number written here. + # + # Every test class must appear in exactly one shard below AND in the matching + # filter in the other workflow (dotnet-core-pr.yml / dotnet-core-master.yml). + # A class named in no filter is silently never executed and CI still goes + # green — TimePlanning.Pn.Test.ShardCoverageTests fails the build when that + # happens, so do not delete it if it starts failing: add the class it names. shard: - name: a - filter: "FullyQualifiedName=TimePlanning.Pn.Test.AbsenceRequestServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.BreakPolicyControllerTests|FullyQualifiedName=TimePlanning.Pn.Test.BreakPolicyServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.CanaryInAColeMine|FullyQualifiedName=TimePlanning.Pn.Test.AbsenceRequestRemovedRowTests|FullyQualifiedName=TimePlanning.Pn.Test.PayrollExportRemovedPlanRegistrationTests|FullyQualifiedName=TimePlanning.Pn.Test.WorkingHoursExcelHolidayColumnTests" + filter: "FullyQualifiedName=TimePlanning.Pn.Test.AbsenceRequestServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.BreakPolicyControllerTests|FullyQualifiedName=TimePlanning.Pn.Test.BreakPolicyServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.CanaryInAColeMine|FullyQualifiedName=TimePlanning.Pn.Test.AbsenceRequestRemovedRowTests|FullyQualifiedName=TimePlanning.Pn.Test.PayrollExportRemovedPlanRegistrationTests|FullyQualifiedName=TimePlanning.Pn.Test.WorkingHoursExcelHolidayColumnTests|FullyQualifiedName=TimePlanning.Pn.Test.PlanTextHelperTests|FullyQualifiedName=TimePlanning.Pn.Test.PauseIdCorrectionTests" - name: b - filter: "FullyQualifiedName=TimePlanning.Pn.Test.PictureSnapshotServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.ContentHandoverServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.DanLonFileExporterTests|FullyQualifiedName=TimePlanning.Pn.Test.DataLonFileExporterTests|FullyQualifiedName=TimePlanning.Pn.Test.ContentHandoverRemovedRowTests|FullyQualifiedName=TimePlanning.Pn.Test.ConfigurationSeedDataTests|FullyQualifiedName=TimePlanning.Pn.Test.PlanTimerSheetColumnsTests" + filter: "FullyQualifiedName=TimePlanning.Pn.Test.PictureSnapshotServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.ContentHandoverServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.DanLonFileExporterTests|FullyQualifiedName=TimePlanning.Pn.Test.DataLonFileExporterTests|FullyQualifiedName=TimePlanning.Pn.Test.ContentHandoverRemovedRowTests|FullyQualifiedName=TimePlanning.Pn.Test.ConfigurationSeedDataTests|FullyQualifiedName=TimePlanning.Pn.Test.PlanTimerSheetColumnsTests|FullyQualifiedName=TimePlanning.Pn.Test.ComputeShiftPauseSecondsTests|FullyQualifiedName=TimePlanning.Pn.Test.PauseOverrideInferenceTests" - name: c filter: "FullyQualifiedName=TimePlanning.Pn.Test.PlanningServiceMultiShiftTests|FullyQualifiedName=TimePlanning.Pn.Test.ScheduleMessageReadTests|FullyQualifiedName=TimePlanning.Pn.Test.DeviceTokenServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.GpsCoordinateServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.PayDayTypeRuleServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.TimePlanningFlexServiceRemovedRowTests|FullyQualifiedName=TimePlanning.Pn.Test.PlanningUpdateByCurrentUserRemovedRowTests|FullyQualifiedName=TimePlanning.Pn.Test.DayLockHelperTests|FullyQualifiedName=TimePlanning.Pn.Test.DayLockInterceptorTests|FullyQualifiedName=TimePlanning.Pn.Test.DayLockWiringTests|FullyQualifiedName=TimePlanning.Pn.Test.ReconcileServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.CorruptedPauseIdRepairTests" - name: d - filter: "FullyQualifiedName=TimePlanning.Pn.Test.PlanRegistrationVersionHistoryTests|FullyQualifiedName=TimePlanning.Pn.Test.PayRuleSetControllerTests|FullyQualifiedName=TimePlanning.Pn.Test.PayRuleSetServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.PayTierRuleServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.PraktikantPayLineRoutingTests|FullyQualifiedName=TimePlanning.Pn.Test.MobileFlexRecomputeAndCascadeTests|FullyQualifiedName=TimePlanning.Pn.Test.SiteWorkerResolverTests" + filter: "FullyQualifiedName=TimePlanning.Pn.Test.PlanRegistrationVersionHistoryTests|FullyQualifiedName=TimePlanning.Pn.Test.PayRuleSetControllerTests|FullyQualifiedName=TimePlanning.Pn.Test.PayRuleSetServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.PayTierRuleServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.PraktikantPayLineRoutingTests|FullyQualifiedName=TimePlanning.Pn.Test.MobileFlexRecomputeAndCascadeTests|FullyQualifiedName=TimePlanning.Pn.Test.SiteWorkerResolverTests|FullyQualifiedName=TimePlanning.Pn.Test.PlanningServiceAdminEditNonRoundMinutesTests|FullyQualifiedName=TimePlanning.Pn.Test.CalculatePayLinesForDayTests" - name: e - filter: "FullyQualifiedName=TimePlanning.Pn.Test.PushNotificationIntegrationTests|FullyQualifiedName=TimePlanning.Pn.Test.WorkingHoursDisplayParityTests|FullyQualifiedName=TimePlanning.Pn.Test.PayTimeBandRuleServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.PlanRegistrationHelperComputationTests|FullyQualifiedName=TimePlanning.Pn.Test.PlanRegistrationHelperHolidayTests|FullyQualifiedName=TimePlanning.Pn.Test.WorkingHoursExcelExportTagsColumnTests" + filter: "FullyQualifiedName=TimePlanning.Pn.Test.PushNotificationIntegrationTests|FullyQualifiedName=TimePlanning.Pn.Test.WorkingHoursDisplayParityTests|FullyQualifiedName=TimePlanning.Pn.Test.PayTimeBandRuleServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.PlanRegistrationHelperComputationTests|FullyQualifiedName=TimePlanning.Pn.Test.PlanRegistrationHelperHolidayTests|FullyQualifiedName=TimePlanning.Pn.Test.WorkingHoursExcelExportTagsColumnTests|FullyQualifiedName=TimePlanning.Pn.Test.PauseIdSelfHealGuardTests|FullyQualifiedName=TimePlanning.Pn.Test.FirstUnlockedDateTests" - name: f - filter: "FullyQualifiedName=TimePlanning.Pn.Test.SettingsServiceExtendedTests|FullyQualifiedName=TimePlanning.Pn.Test.PlanRegistrationHelperDisplayParityTests|FullyQualifiedName=TimePlanning.Pn.Test.OneMinuteModeTimelineTests|FullyQualifiedName=TimePlanning.Pn.Test.OneMinuteIntervalsEffectiveDateTests|FullyQualifiedName=TimePlanning.Pn.Test.RunningFlexChainModeBoundaryTests|FullyQualifiedName=TimePlanning.Pn.Test.PlanRegistrationHelperReadBySiteAndDateTests|FullyQualifiedName=TimePlanning.Pn.Test.PlanRegistrationHelperTests|FullyQualifiedName=TimePlanning.Pn.Test.PushNotificationServiceTests" + filter: "FullyQualifiedName=TimePlanning.Pn.Test.SettingsServiceExtendedTests|FullyQualifiedName=TimePlanning.Pn.Test.PlanRegistrationHelperDisplayParityTests|FullyQualifiedName=TimePlanning.Pn.Test.OneMinuteModeTimelineTests|FullyQualifiedName=TimePlanning.Pn.Test.OneMinuteIntervalsEffectiveDateTests|FullyQualifiedName=TimePlanning.Pn.Test.RunningFlexChainModeBoundaryTests|FullyQualifiedName=TimePlanning.Pn.Test.PlanRegistrationHelperReadBySiteAndDateTests|FullyQualifiedName=TimePlanning.Pn.Test.PlanRegistrationHelperTests|FullyQualifiedName=TimePlanning.Pn.Test.PushNotificationServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.WorkingHoursGrpcKioskNonRoundMinutesTests|FullyQualifiedName=TimePlanning.Pn.Test.ResolveShiftSecondsTests|FullyQualifiedName=TimePlanning.Pn.Test.ShardCoverageTests" - name: g - filter: "FullyQualifiedName=TimePlanning.Pn.Test.SettingsServicePhoneNumberTests|FullyQualifiedName=TimePlanning.Pn.Test.TimePlanningWorkingHoursExportTests|FullyQualifiedName=TimePlanning.Pn.Test.GrpcServices.TimePlanningAbsenceRequestGrpcServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.GrpcServices.TimePlanningAuthGrpcServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.DagsoversigtWorksheetExportTests|FullyQualifiedName=TimePlanning.Pn.Test.WorkingHoursImportRemovedRowTests" + filter: "FullyQualifiedName=TimePlanning.Pn.Test.SettingsServicePhoneNumberTests|FullyQualifiedName=TimePlanning.Pn.Test.TimePlanningWorkingHoursExportTests|FullyQualifiedName=TimePlanning.Pn.Test.GrpcServices.TimePlanningAbsenceRequestGrpcServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.GrpcServices.TimePlanningAuthGrpcServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.DagsoversigtWorksheetExportTests|FullyQualifiedName=TimePlanning.Pn.Test.WorkingHoursImportRemovedRowTests|FullyQualifiedName=TimePlanning.Pn.Test.GrpcServices.TimePlanningPlanningsGrpcServiceMapTests|FullyQualifiedName=TimePlanning.Pn.Test.Helpers.PauseMinutesCalculatorTests" - name: h - filter: "FullyQualifiedName=TimePlanning.Pn.Test.SettingsServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.GrpcServices.TimePlanningContentHandoverGrpcServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.GrpcServices.TimePlanningPlanningsGrpcServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.GrpcServices.TimePlanningPlanningsGrpcServiceTimeZoneTests|FullyQualifiedName=TimePlanning.Pn.Test.GrpcServices.TimePlanningSettingsGrpcServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.GrpcServices.TimePlanningWorkingHoursGrpcServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.WorkingHoursExcelExportE2ETests|FullyQualifiedName=TimePlanning.Pn.Test.WorkingHoursExcelShiftColumnOrderTests|FullyQualifiedName=TimePlanning.Pn.Test.WorkingHoursMessagePersistenceTests" + filter: "FullyQualifiedName=TimePlanning.Pn.Test.SettingsServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.GrpcServices.TimePlanningContentHandoverGrpcServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.GrpcServices.TimePlanningPlanningsGrpcServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.GrpcServices.TimePlanningPlanningsGrpcServiceTimeZoneTests|FullyQualifiedName=TimePlanning.Pn.Test.GrpcServices.TimePlanningSettingsGrpcServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.GrpcServices.TimePlanningWorkingHoursGrpcServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.WorkingHoursExcelExportE2ETests|FullyQualifiedName=TimePlanning.Pn.Test.WorkingHoursExcelShiftColumnOrderTests|FullyQualifiedName=TimePlanning.Pn.Test.WorkingHoursMessagePersistenceTests|FullyQualifiedName=TimePlanning.Pn.Test.EnumerateShiftSegmentsTests|FullyQualifiedName=TimePlanning.Pn.Test.TimePlanningWorkingHoursServiceNullUserTests" steps: - uses: actions/checkout@v3 - name: Create docker network diff --git a/.github/workflows/dotnet-core-pr.yml b/.github/workflows/dotnet-core-pr.yml index 2c221dd9..b6d90686 100644 --- a/.github/workflows/dotnet-core-pr.yml +++ b/.github/workflows/dotnet-core-pr.yml @@ -242,23 +242,36 @@ jobs: strategy: fail-fast: false matrix: + # Shard letters are arbitrary; the cost is not. Every fixture deriving from + # TestBaseSetup starts its own MariaDB Testcontainer, and those containers + # dominate a shard's wall-clock. Shard 'c' is already the DB-dense one (11 of + # its 12 classes derive TestBaseSetup), so put new DB-backed fixtures in a + # lighter shard instead. Per-shard durations swing a lot run to run (runner + # contention, image pulls), so check the last green master run before choosing + # rather than trusting a number written here. + # + # Every test class must appear in exactly one shard below AND in the matching + # filter in the other workflow (dotnet-core-pr.yml / dotnet-core-master.yml). + # A class named in no filter is silently never executed and CI still goes + # green — TimePlanning.Pn.Test.ShardCoverageTests fails the build when that + # happens, so do not delete it if it starts failing: add the class it names. shard: - name: a - filter: "FullyQualifiedName=TimePlanning.Pn.Test.AbsenceRequestServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.BreakPolicyControllerTests|FullyQualifiedName=TimePlanning.Pn.Test.BreakPolicyServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.CanaryInAColeMine|FullyQualifiedName=TimePlanning.Pn.Test.AbsenceRequestRemovedRowTests|FullyQualifiedName=TimePlanning.Pn.Test.PayrollExportRemovedPlanRegistrationTests|FullyQualifiedName=TimePlanning.Pn.Test.WorkingHoursExcelHolidayColumnTests" + filter: "FullyQualifiedName=TimePlanning.Pn.Test.AbsenceRequestServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.BreakPolicyControllerTests|FullyQualifiedName=TimePlanning.Pn.Test.BreakPolicyServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.CanaryInAColeMine|FullyQualifiedName=TimePlanning.Pn.Test.AbsenceRequestRemovedRowTests|FullyQualifiedName=TimePlanning.Pn.Test.PayrollExportRemovedPlanRegistrationTests|FullyQualifiedName=TimePlanning.Pn.Test.WorkingHoursExcelHolidayColumnTests|FullyQualifiedName=TimePlanning.Pn.Test.PlanTextHelperTests|FullyQualifiedName=TimePlanning.Pn.Test.PauseIdCorrectionTests" - name: b - filter: "FullyQualifiedName=TimePlanning.Pn.Test.PictureSnapshotServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.ContentHandoverServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.DanLonFileExporterTests|FullyQualifiedName=TimePlanning.Pn.Test.DataLonFileExporterTests|FullyQualifiedName=TimePlanning.Pn.Test.ContentHandoverRemovedRowTests|FullyQualifiedName=TimePlanning.Pn.Test.ConfigurationSeedDataTests|FullyQualifiedName=TimePlanning.Pn.Test.PlanTimerSheetColumnsTests" + filter: "FullyQualifiedName=TimePlanning.Pn.Test.PictureSnapshotServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.ContentHandoverServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.DanLonFileExporterTests|FullyQualifiedName=TimePlanning.Pn.Test.DataLonFileExporterTests|FullyQualifiedName=TimePlanning.Pn.Test.ContentHandoverRemovedRowTests|FullyQualifiedName=TimePlanning.Pn.Test.ConfigurationSeedDataTests|FullyQualifiedName=TimePlanning.Pn.Test.PlanTimerSheetColumnsTests|FullyQualifiedName=TimePlanning.Pn.Test.ComputeShiftPauseSecondsTests|FullyQualifiedName=TimePlanning.Pn.Test.PauseOverrideInferenceTests" - name: c filter: "FullyQualifiedName=TimePlanning.Pn.Test.PlanningServiceMultiShiftTests|FullyQualifiedName=TimePlanning.Pn.Test.ScheduleMessageReadTests|FullyQualifiedName=TimePlanning.Pn.Test.DeviceTokenServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.GpsCoordinateServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.PayDayTypeRuleServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.TimePlanningFlexServiceRemovedRowTests|FullyQualifiedName=TimePlanning.Pn.Test.PlanningUpdateByCurrentUserRemovedRowTests|FullyQualifiedName=TimePlanning.Pn.Test.DayLockHelperTests|FullyQualifiedName=TimePlanning.Pn.Test.DayLockInterceptorTests|FullyQualifiedName=TimePlanning.Pn.Test.DayLockWiringTests|FullyQualifiedName=TimePlanning.Pn.Test.ReconcileServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.CorruptedPauseIdRepairTests" - name: d - filter: "FullyQualifiedName=TimePlanning.Pn.Test.PlanRegistrationVersionHistoryTests|FullyQualifiedName=TimePlanning.Pn.Test.PayRuleSetControllerTests|FullyQualifiedName=TimePlanning.Pn.Test.PayRuleSetServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.PayTierRuleServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.PraktikantPayLineRoutingTests|FullyQualifiedName=TimePlanning.Pn.Test.MobileFlexRecomputeAndCascadeTests|FullyQualifiedName=TimePlanning.Pn.Test.SiteWorkerResolverTests" + filter: "FullyQualifiedName=TimePlanning.Pn.Test.PlanRegistrationVersionHistoryTests|FullyQualifiedName=TimePlanning.Pn.Test.PayRuleSetControllerTests|FullyQualifiedName=TimePlanning.Pn.Test.PayRuleSetServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.PayTierRuleServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.PraktikantPayLineRoutingTests|FullyQualifiedName=TimePlanning.Pn.Test.MobileFlexRecomputeAndCascadeTests|FullyQualifiedName=TimePlanning.Pn.Test.SiteWorkerResolverTests|FullyQualifiedName=TimePlanning.Pn.Test.PlanningServiceAdminEditNonRoundMinutesTests|FullyQualifiedName=TimePlanning.Pn.Test.CalculatePayLinesForDayTests" - name: e - filter: "FullyQualifiedName=TimePlanning.Pn.Test.PushNotificationIntegrationTests|FullyQualifiedName=TimePlanning.Pn.Test.WorkingHoursDisplayParityTests|FullyQualifiedName=TimePlanning.Pn.Test.PayTimeBandRuleServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.PlanRegistrationHelperComputationTests|FullyQualifiedName=TimePlanning.Pn.Test.PlanRegistrationHelperHolidayTests|FullyQualifiedName=TimePlanning.Pn.Test.WorkingHoursExcelExportTagsColumnTests" + filter: "FullyQualifiedName=TimePlanning.Pn.Test.PushNotificationIntegrationTests|FullyQualifiedName=TimePlanning.Pn.Test.WorkingHoursDisplayParityTests|FullyQualifiedName=TimePlanning.Pn.Test.PayTimeBandRuleServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.PlanRegistrationHelperComputationTests|FullyQualifiedName=TimePlanning.Pn.Test.PlanRegistrationHelperHolidayTests|FullyQualifiedName=TimePlanning.Pn.Test.WorkingHoursExcelExportTagsColumnTests|FullyQualifiedName=TimePlanning.Pn.Test.PauseIdSelfHealGuardTests|FullyQualifiedName=TimePlanning.Pn.Test.FirstUnlockedDateTests" - name: f - filter: "FullyQualifiedName=TimePlanning.Pn.Test.SettingsServiceExtendedTests|FullyQualifiedName=TimePlanning.Pn.Test.PlanRegistrationHelperDisplayParityTests|FullyQualifiedName=TimePlanning.Pn.Test.OneMinuteModeTimelineTests|FullyQualifiedName=TimePlanning.Pn.Test.OneMinuteIntervalsEffectiveDateTests|FullyQualifiedName=TimePlanning.Pn.Test.RunningFlexChainModeBoundaryTests|FullyQualifiedName=TimePlanning.Pn.Test.PlanRegistrationHelperReadBySiteAndDateTests|FullyQualifiedName=TimePlanning.Pn.Test.PlanRegistrationHelperTests|FullyQualifiedName=TimePlanning.Pn.Test.PushNotificationServiceTests" + filter: "FullyQualifiedName=TimePlanning.Pn.Test.SettingsServiceExtendedTests|FullyQualifiedName=TimePlanning.Pn.Test.PlanRegistrationHelperDisplayParityTests|FullyQualifiedName=TimePlanning.Pn.Test.OneMinuteModeTimelineTests|FullyQualifiedName=TimePlanning.Pn.Test.OneMinuteIntervalsEffectiveDateTests|FullyQualifiedName=TimePlanning.Pn.Test.RunningFlexChainModeBoundaryTests|FullyQualifiedName=TimePlanning.Pn.Test.PlanRegistrationHelperReadBySiteAndDateTests|FullyQualifiedName=TimePlanning.Pn.Test.PlanRegistrationHelperTests|FullyQualifiedName=TimePlanning.Pn.Test.PushNotificationServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.WorkingHoursGrpcKioskNonRoundMinutesTests|FullyQualifiedName=TimePlanning.Pn.Test.ResolveShiftSecondsTests|FullyQualifiedName=TimePlanning.Pn.Test.ShardCoverageTests" - name: g - filter: "FullyQualifiedName=TimePlanning.Pn.Test.SettingsServicePhoneNumberTests|FullyQualifiedName=TimePlanning.Pn.Test.TimePlanningWorkingHoursExportTests|FullyQualifiedName=TimePlanning.Pn.Test.GrpcServices.TimePlanningAbsenceRequestGrpcServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.GrpcServices.TimePlanningAuthGrpcServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.DagsoversigtWorksheetExportTests|FullyQualifiedName=TimePlanning.Pn.Test.WorkingHoursImportRemovedRowTests" + filter: "FullyQualifiedName=TimePlanning.Pn.Test.SettingsServicePhoneNumberTests|FullyQualifiedName=TimePlanning.Pn.Test.TimePlanningWorkingHoursExportTests|FullyQualifiedName=TimePlanning.Pn.Test.GrpcServices.TimePlanningAbsenceRequestGrpcServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.GrpcServices.TimePlanningAuthGrpcServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.DagsoversigtWorksheetExportTests|FullyQualifiedName=TimePlanning.Pn.Test.WorkingHoursImportRemovedRowTests|FullyQualifiedName=TimePlanning.Pn.Test.GrpcServices.TimePlanningPlanningsGrpcServiceMapTests|FullyQualifiedName=TimePlanning.Pn.Test.Helpers.PauseMinutesCalculatorTests" - name: h - filter: "FullyQualifiedName=TimePlanning.Pn.Test.SettingsServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.GrpcServices.TimePlanningContentHandoverGrpcServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.GrpcServices.TimePlanningPlanningsGrpcServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.GrpcServices.TimePlanningPlanningsGrpcServiceTimeZoneTests|FullyQualifiedName=TimePlanning.Pn.Test.GrpcServices.TimePlanningSettingsGrpcServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.GrpcServices.TimePlanningWorkingHoursGrpcServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.WorkingHoursExcelExportE2ETests|FullyQualifiedName=TimePlanning.Pn.Test.WorkingHoursExcelShiftColumnOrderTests|FullyQualifiedName=TimePlanning.Pn.Test.WorkingHoursMessagePersistenceTests" + filter: "FullyQualifiedName=TimePlanning.Pn.Test.SettingsServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.GrpcServices.TimePlanningContentHandoverGrpcServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.GrpcServices.TimePlanningPlanningsGrpcServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.GrpcServices.TimePlanningPlanningsGrpcServiceTimeZoneTests|FullyQualifiedName=TimePlanning.Pn.Test.GrpcServices.TimePlanningSettingsGrpcServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.GrpcServices.TimePlanningWorkingHoursGrpcServiceTests|FullyQualifiedName=TimePlanning.Pn.Test.WorkingHoursExcelExportE2ETests|FullyQualifiedName=TimePlanning.Pn.Test.WorkingHoursExcelShiftColumnOrderTests|FullyQualifiedName=TimePlanning.Pn.Test.WorkingHoursMessagePersistenceTests|FullyQualifiedName=TimePlanning.Pn.Test.EnumerateShiftSegmentsTests|FullyQualifiedName=TimePlanning.Pn.Test.TimePlanningWorkingHoursServiceNullUserTests" steps: - uses: actions/checkout@v3 - name: Create docker network diff --git a/eFormAPI/Plugins/TimePlanning.Pn/TimePlanning.Pn.Test/ShardCoverageTests.cs b/eFormAPI/Plugins/TimePlanning.Pn/TimePlanning.Pn.Test/ShardCoverageTests.cs new file mode 100644 index 00000000..48736806 --- /dev/null +++ b/eFormAPI/Plugins/TimePlanning.Pn/TimePlanning.Pn.Test/ShardCoverageTests.cs @@ -0,0 +1,232 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Text; +using System.Text.RegularExpressions; +using NUnit.Framework; + +namespace TimePlanning.Pn.Test; + +/// +/// Meta-guard: every test class in this assembly must be named in the +/// FullyQualifiedName= shard filters of BOTH GitHub Actions workflows. +/// +/// WHY THIS EXISTS +/// --------------- +/// CI does not run `dotnet test` over the whole assembly. It fans out across a +/// matrix of shards, each one a `--filter` listing the classes it owns. A class +/// that is in no filter is never executed — vstest reports nothing, the shard is +/// green, the gate job is green, and the PR merges. The failure mode is total +/// silence, which is why it has repeatedly gone unnoticed for months at a time +/// ( sat unsharded long enough to +/// accumulate a test that could never have passed). +/// +/// This fixture turns that silence into a red build. It needs no database. +/// +[TestFixture] +public class ShardCoverageTests +{ + private static readonly string[] WorkflowFileNames = + [ + "dotnet-core-pr.yml", + "dotnet-core-master.yml" + ]; + + /// + /// The job that owns the shard matrix, and the path to the filter strings + /// inside it. Quoted in the failure message so the reader has an anchor into + /// a several-hundred-line YAML file instead of just a path to it. + /// + private const string ShardMatrixLocation = "job `test-dotnet`, `strategy.matrix.shard[].filter`"; + + /// + /// Identifies a workflow file as belonging to THIS repository. Filename alone + /// is not enough: the host application this plugin is copied into during dev + /// mode (eform-angular-frontend) ships workflows with these exact same two + /// names, and the walk-up below would otherwise stop there. + /// + private const string OurFilterMarker = "FullyQualifiedName=TimePlanning.Pn.Test."; + + /// + /// Matches one entry of a shard filter, e.g. + /// FullyQualifiedName=TimePlanning.Pn.Test.GrpcServices.FooTests. + /// + private static readonly Regex FilterEntry = new(@"FullyQualifiedName=([A-Za-z0-9_.+]+)"); + + [Test] + public void EveryTestClassIsAssignedToAShardInBothWorkflows() + { + var (workflowDir, shardedPerFile) = FindWorkflows(); + var discovered = DiscoverTestClasses(); + + // Non-vacuous self-check: this very fixture must come back out of + // discovery. Asserting merely that the list is non-empty would always + // hold — this class always qualifies — and would not exercise + // IsTestMethod, the part most likely to be broken by an NUnit upgrade or + // a careless edit. + Assert.That(discovered, Does.Contain(typeof(ShardCoverageTests).FullName), + "Discovery did not find ShardCoverageTests itself, so it is not reliably finding anything. " + + "Fix the reflection in this file — it is no longer protecting the suite."); + + var missing = discovered + .Select(fqn => ( + Fqn: fqn, + AbsentFrom: WorkflowFileNames.Where(f => !shardedPerFile[f].Contains(fqn)).ToArray())) + .Where(x => x.AbsentFrom.Length > 0) + .OrderBy(x => x.Fqn, StringComparer.Ordinal) + .ToList(); + + if (missing.Count == 0) + { + return; + } + + var message = new StringBuilder(); + message.AppendLine( + $"{missing.Count} test class(es) are missing from the CI shard filters " + + "and so do not run in at least one workflow:"); + message.AppendLine(); + foreach (var (fqn, absentFrom) in missing) + { + message.AppendLine($" {fqn}"); + message.AppendLine($" absent from: {string.Join(", ", absentFrom)}"); + } + + message.AppendLine(); + message.AppendLine("HOW TO FIX"); + message.AppendLine($" Add each class above to exactly one shard's `filter:` string ({ShardMatrixLocation})"); + message.AppendLine(" in BOTH files:"); + foreach (var name in WorkflowFileNames) + { + message.AppendLine($" {Path.Combine(workflowDir, name)}"); + } + + message.AppendLine(" The two files' shard filters must list the same classes — a class named in one"); + message.AppendLine(" and not the other runs on master but not on PRs (or the reverse), which is the"); + message.AppendLine(" same bug half-fixed. The files differ in other ways; only the filters must match."); + message.AppendLine(); + message.AppendLine(" Append to the chosen shard's filter, pipe-separated, e.g.:"); + message.AppendLine($" |FullyQualifiedName={missing[0].Fqn}"); + message.AppendLine(); + message.AppendLine(" Pick the shard BY COST, not alphabetically. Every fixture deriving from"); + message.AppendLine(" TestBaseSetup starts its own MariaDB Testcontainer, and those containers dominate"); + message.AppendLine(" a shard's wall-clock; a fixture with no database costs almost nothing and can go"); + message.AppendLine(" anywhere. Shard 'c' is already the DB-dense one (11 of its 12 classes derive"); + message.AppendLine(" TestBaseSetup), so keep new DB-backed fixtures out of it. Shard durations shift a"); + message.AppendLine(" lot run to run (runner contention, Testcontainers image pulls), so check the"); + message.AppendLine(" per-shard durations of the last green master run before choosing rather than"); + message.AppendLine(" trusting any figure written down here."); + message.AppendLine(); + message.AppendLine(" Do NOT silence this guard. If a class genuinely must not be sharded, it should"); + message.AppendLine(" not be carrying [Test] methods in this assembly."); + + throw new AssertionException(message.ToString()); + } + + /// + /// Every concrete type in this assembly that owns or inherits at least one + /// NUnit test method, named the way the vstest FullyQualifiedName= + /// filter matches on. + /// + private static List DiscoverTestClasses() + { + const BindingFlags MethodFlags = + BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.Static; + + return Assembly.GetExecutingAssembly() + .GetTypes() + .Where(t => t.IsClass) + // Excluded: abstract fixtures such as TestBaseSetup. NUnit never runs an + // abstract type as a fixture; any [Test] it declares executes under each + // concrete subclass's own name, and it is the subclass that needs sharding. + .Where(t => !t.IsAbstract) + // Excluded: open generic definitions. NUnit can only run those via + // [TestFixture(typeof(...))], which produces closed types, and there are + // none in this assembly today. If one ever appears this exclusion should be + // revisited rather than widened. + .Where(t => !t.IsGenericTypeDefinition) + .Where(t => t.GetMethods(MethodFlags).Any(IsTestMethod)) + .Select(t => t.FullName!) + .OrderBy(n => n, StringComparer.Ordinal) + .ToList(); + } + + private static bool IsTestMethod(MethodInfo method) => + method.GetCustomAttributes(inherit: true) + .Any(a => a is TestAttribute or TestCaseAttribute or TestCaseSourceAttribute); + + private static HashSet ParseShardedClasses(string workflowText) => + FilterEntry.Matches(workflowText) + .Select(m => m.Groups[1].Value) + .ToHashSet(StringComparer.Ordinal); + + /// + /// Reads a workflow with its YAML comment lines removed, so that a shard + /// entry someone commented out does not count as covered — that would be a + /// false-green, and it is the only way to fool this guard without editing the + /// C#. A class name can never contain '#', so dropping '#'-leading lines + /// cannot discard a live entry. + /// + private static string ReadUncommented(string path) => + string.Join('\n', File.ReadLines(path).Where(line => !line.TrimStart().StartsWith('#'))); + + /// + /// Walks up from the test binary's directory looking for THIS repository's + /// .github/workflows folder — the tests run out of + /// .../TimePlanning.Pn.Test/bin/<config>/<tfm>/, so the repo root is + /// several levels above and its depth differs between local and CI layouts. + /// + /// A directory qualifies only when both workflow files are present AND both + /// carry . Filenames alone do not identify us: + /// eform-angular-frontend — the host application this plugin is copied into + /// during Base/Full dev mode — has workflows with the same two names and no + /// shard filters at all. Matching on content walks past those and keeps + /// climbing, so a dev-mode run ends in the "not found" failure below, which + /// lists what it skipped, rather than misdiagnosing the host's workflows as + /// our own with a broken parser. + /// + private static (string Directory, Dictionary> ShardedPerFile) FindWorkflows() + { + var workflowDirsSeen = new List(); + + for (var dir = new DirectoryInfo(AppContext.BaseDirectory); dir != null; dir = dir.Parent) + { + var candidate = Path.Combine(dir.FullName, ".github", "workflows"); + if (!Directory.Exists(candidate)) + { + continue; + } + + workflowDirsSeen.Add(candidate); + + var paths = WorkflowFileNames.ToDictionary(f => f, f => Path.Combine(candidate, f)); + if (!paths.Values.All(File.Exists)) + { + continue; + } + + var texts = paths.ToDictionary(p => p.Key, p => ReadUncommented(p.Value)); + if (!texts.Values.All(t => t.Contains(OurFilterMarker, StringComparison.Ordinal))) + { + continue; + } + + return (candidate, texts.ToDictionary(t => t.Key, t => ParseShardedClasses(t.Value))); + } + + // Deliberately a failure, not a silent pass: a guard that gives up when it + // cannot find its input is indistinguishable from no guard at all, and would + // let exactly the bug it exists to catch back through. + throw new AssertionException( + "Could not locate this repository's .github/workflows directory — one holding " + + $"[{string.Join(", ", WorkflowFileNames)}] with shard filters naming '{OurFilterMarker}...'.\n" + + $" Started from: {AppContext.BaseDirectory}\n" + + " .github/workflows directories seen while walking up (none qualified): " + + (workflowDirsSeen.Count == 0 ? "(none)" : string.Join(", ", workflowDirsSeen)) + "\n" + + " If one of those belongs to eform-angular-frontend, this is a dev-mode copy of the plugin " + + "inside the host app and there is nothing here to check — run these tests from a full checkout " + + "of eform-angular-timeplanning-plugin. Otherwise fix the lookup; do not delete the guard."); + } +} From 6b68836210c327f451bf3593cdb381655359fb60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Schultz=20Madsen?= Date: Thu, 17 Sep 2026 16:55:10 +0200 Subject: [PATCH 2/2] test(ci): harden the shard guard against test shapes it would have missed Review follow-ups on ShardCoverageTests. None changes what it checks; all three close ways it could have stayed silent - the one failure mode a guard must not have. 1. Discovery matched three concrete attribute types, so a method carrying only [Theory], [Combinatorial], [Pairwise], [Sequential] or a custom builder attribute was invisible even though NUnit runs it. Match NUnit's ITestBuilder / ISimpleTestBuilder interfaces instead: strictly broader, covers every shape above, and needs no maintenance as NUnit adds more. None of these shapes exists in the assembly today - but the guard's whole purpose is the class that arrives tomorrow, and this was the shape it would have let through. 2. Open generic definitions were skipped with a comment asking a future reader to revisit. Nothing made them. `[TestFixture(typeof(int))] class FooTests` would have been run by NUnit, skipped here, and left unsharded - exactly the bug this guard prevents. The skip is now backed by an assertion that no generic type definition in the assembly carries a test method, so the exclusion is safe because it is enforced rather than remembered. 3. GetTypes() can throw ReflectionTypeLoadException on a partially loadable assembly, which would have replaced the guard's deliberately explicit diagnosis with an opaque loader stack trace. Caught, falling back to the types that did load. Verified by reflection in seven states: passing on a correct tree; a class dropped from one workflow only; a shard filter commented out; the dev-mode layout inside eform-angular-frontend; no repository above the binary; a [Theory]-only fixture with zero [Test] attributes (1); and a generic fixture (2). Co-Authored-By: Claude Opus 5 --- .../ShardCoverageTests.cs | 73 ++++++++++++++++--- 1 file changed, 61 insertions(+), 12 deletions(-) diff --git a/eFormAPI/Plugins/TimePlanning.Pn/TimePlanning.Pn.Test/ShardCoverageTests.cs b/eFormAPI/Plugins/TimePlanning.Pn/TimePlanning.Pn.Test/ShardCoverageTests.cs index 48736806..6ff3e0b7 100644 --- a/eFormAPI/Plugins/TimePlanning.Pn/TimePlanning.Pn.Test/ShardCoverageTests.cs +++ b/eFormAPI/Plugins/TimePlanning.Pn/TimePlanning.Pn.Test/ShardCoverageTests.cs @@ -6,6 +6,7 @@ using System.Text; using System.Text.RegularExpressions; using NUnit.Framework; +using NUnit.Framework.Interfaces; namespace TimePlanning.Pn.Test; @@ -132,30 +133,78 @@ public void EveryTestClassIsAssignedToAShardInBothWorkflows() /// private static List DiscoverTestClasses() { - const BindingFlags MethodFlags = - BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.Static; + var withTests = GetAssemblyTypes() + .Where(t => t.IsClass && HasTestMethod(t)) + .ToList(); + + // Open generic definitions are dropped below, because NUnit names the + // closed instantiations it builds from [TestFixture(typeof(...))] in a way + // this guard does not model. That exclusion is only safe while none + // exists, so it is asserted rather than left as a comment asking a future + // reader to remember — otherwise `[TestFixture(typeof(int))] class + // FooTests` would run in NUnit, be skipped here, and go unsharded: + // precisely the bug this fixture exists to prevent. + var genericFixtures = withTests + .Where(t => t.IsGenericTypeDefinition) + .Select(t => t.FullName!) + .OrderBy(n => n, StringComparer.Ordinal) + .ToList(); - return Assembly.GetExecutingAssembly() - .GetTypes() - .Where(t => t.IsClass) + Assert.That(genericFixtures, Is.Empty, + "Generic test fixture(s) carrying test methods were found:\n " + + string.Join("\n ", genericFixtures) + "\n" + + "NUnit runs these via [TestFixture(typeof(...))], but this guard cannot model how the " + + "closed instantiations are named, so it would leave them unsharded and silently unrun. " + + "Either make them non-generic fixtures, or teach this guard their naming AND add them to " + + "the shard filters by hand."); + + return withTests // Excluded: abstract fixtures such as TestBaseSetup. NUnit never runs an // abstract type as a fixture; any [Test] it declares executes under each // concrete subclass's own name, and it is the subclass that needs sharding. .Where(t => !t.IsAbstract) - // Excluded: open generic definitions. NUnit can only run those via - // [TestFixture(typeof(...))], which produces closed types, and there are - // none in this assembly today. If one ever appears this exclusion should be - // revisited rather than widened. .Where(t => !t.IsGenericTypeDefinition) - .Where(t => t.GetMethods(MethodFlags).Any(IsTestMethod)) .Select(t => t.FullName!) .OrderBy(n => n, StringComparer.Ordinal) .ToList(); } + /// + /// A partially loadable assembly still yields whatever types did load. Letting + /// escape would replace this + /// fixture's carefully worded diagnosis with an opaque loader stack trace. + /// + private static IEnumerable GetAssemblyTypes() + { + try + { + return Assembly.GetExecutingAssembly().GetTypes(); + } + catch (ReflectionTypeLoadException ex) + { + return ex.Types.Where(t => t is not null)!; + } + } + + private static bool HasTestMethod(Type type) + { + const BindingFlags MethodFlags = + BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.Static; + + return type.GetMethods(MethodFlags).Any(IsTestMethod); + } + + /// + /// Matched through NUnit's builder interfaces rather than a list of attribute + /// types. TestAttribute is an ; + /// TestCase, TestCaseSource, Theory, Combinatorial, + /// Pairwise and Sequential are all s — + /// as is any custom builder attribute. Naming three concrete types instead + /// would silently miss every other shape NUnit can run, which is exactly the + /// class of omission this fixture exists to catch. + /// private static bool IsTestMethod(MethodInfo method) => - method.GetCustomAttributes(inherit: true) - .Any(a => a is TestAttribute or TestCaseAttribute or TestCaseSourceAttribute); + method.GetCustomAttributes(inherit: true).Any(a => a is ITestBuilder or ISimpleTestBuilder); private static HashSet ParseShardedClasses(string workflowText) => FilterEntry.Matches(workflowText)