From 6fed9c67c92f85a121d37b296efb5d7c330100b7 Mon Sep 17 00:00:00 2001 From: Said Boudjelda Date: Sat, 17 Oct 2026 19:20:53 +0200 Subject: [PATCH] fix(engine): run retention keeps the run a baseline names, so regression checks keep working baseline_runs names a run by id, and neither retention query left that run out. Set a LOAD baseline on a run, and 90 days later the daily sweep, or an earlier kates test prune, deleted it like any other finished run. From then on GET /api/tests/{id}/report/regression answered 404 "Baseline run not found" for every LOAD run, so kates test baseline checks and the MCP assess_run regression part failed until someone set a new baseline by hand. Both queries behind RunRetention, the count and the ids, now leave out a run any baseline names, so DELETE /api/tests and the sweep keep it however old it is, and matched still equals what a call can delete: remaining reaches 0 and callers stop. DELETE /api/tests/{id} can still delete a baseline's run. TestResourcePruneTest covers both queries on H2 through the endpoint. The OpenAPI description, kates/docs/api-reference.md and the book's retention passages name the exception, and the anchored findWithResults caveat ref moves down by the six lines added above it. --- cli/cmd/mcp_caveats.go | 2 +- docs/book/02-architecture.md | 4 +-- docs/book/10-cli-reference.md | 4 +-- docs/book/11-api-reference.md | 4 +-- docs/book/12-deployment.md | 2 +- kates/docs/api-reference.md | 4 +-- .../com/bmscomp/kates/api/TestResource.java | 2 +- .../kates/service/TestRunRepository.java | 10 +++++-- .../kates/api/TestResourcePruneTest.java | 29 +++++++++++++++++++ 9 files changed, 48 insertions(+), 13 deletions(-) diff --git a/cli/cmd/mcp_caveats.go b/cli/cmd/mcp_caveats.go index 5d5ecebd..ea789711 100644 --- a/cli/cmd/mcp_caveats.go +++ b/cli/cmd/mcp_caveats.go @@ -168,7 +168,7 @@ var mcpCaveatsCore = []mcpCaveat{ Refs: []string{ mcpJava + "trend/TrendResource.java:31-60", mcpJava + "trend/TrendService.java:53-54,185-189", - mcpAnchoredRef(mcpJava+"service/TestRunRepository.java:354-374", + mcpAnchoredRef(mcpJava+"service/TestRunRepository.java:360-380", "List findWithResults(", "WHERE r.testType = :type AND r.status = :status", ".map(EntityMapper::toDomain)"), }, diff --git a/docs/book/02-architecture.md b/docs/book/02-architecture.md index 96084cd3..d5123ef6 100644 --- a/docs/book/02-architecture.md +++ b/docs/book/02-architecture.md @@ -379,9 +379,9 @@ The table lists what Kates keeps about a run, where each piece lives and how lon | What | Where it lives | How long it lasts | |:--|:--|:--| -| The run: its spec, status, per-task results, an INTEGRITY task's integrity result among them, and SLA | PostgreSQL: the `kates` chart's own, beside the Kates API, or an external database | 90 days; once a day the Kates API deletes finished runs older than that, and `kates test prune` deletes them sooner | +| The run: its spec, status, per-task results, an INTEGRITY task's integrity result among them, and SLA | PostgreSQL: the `kates` chart's own, beside the Kates API, or an external database | 90 days; once a day the Kates API deletes finished runs older than that, except a run a baseline names, and `kates test prune` deletes them sooner | | Disruption reports, schedules, webhooks and audit events | PostgreSQL | No automatic expiry | -| A baseline: the run each test type is compared with | PostgreSQL | The baseline never expires; the run it names is deleted at 90 days like any other | +| A baseline: the run each test type is compared with | PostgreSQL | The baseline never expires; retention and `kates test prune` keep the run it names. `kates test delete` can still delete it, and the regression check then answers `404` until a new baseline is set | | The report of a finished run, as `report show` prints it | Built from the stored run when first asked for, then kept in the Kates API's memory | The 200 most recently used, until the pod restarts; then built again | | A resilience run's report | The response to `kates resilience run` | Not stored; the test run inside it is kept like any other | | Heatmap rows, for native runs | The Kates API's memory | The 50 most recent runs, until the pod restarts | diff --git a/docs/book/10-cli-reference.md b/docs/book/10-cli-reference.md index 756d053b..b8bb0e21 100644 --- a/docs/book/10-cli-reference.md +++ b/docs/book/10-cli-reference.md @@ -711,13 +711,13 @@ kates test prune --older-than 60d --yes -o json | `--dry-run` | `false` | Count the runs and delete nothing | | `--yes`, `-y` | `false` | Delete without asking | -Delete the finished runs created more than `--older-than` ago, to keep the run history to a retention period of your own. Where `kates test cleanup` repairs, deleting runs still `RUNNING` long after their planned end, `prune` keeps a retention: it deletes only `DONE` and `FAILED` runs, judged by when they were created. A cancelled run is stored as `FAILED`, so it goes too. +Delete the finished runs created more than `--older-than` ago, to keep the run history to a retention period of your own. Where `kates test cleanup` repairs, deleting runs still `RUNNING` long after their planned end, `prune` keeps a retention: it deletes only `DONE` and `FAILED` runs, judged by when they were created. A cancelled run is stored as `FAILED`, so it goes too. The run a test type's baseline names is kept, however old it is. The command first counts the runs, deleting none, and prints how many it found, their statuses and the instant they were created before. It stops there when none match, or with `--dry-run`. Otherwise it asks before deleting them; without a terminal it refuses unless `--yes` is given, and it exits 1 when you decline. It then deletes the oldest first, up to 1,000 runs a call, until none are left, and prints the total. Each run goes as `kates test delete` deletes it, with its results, and leaves a row in the Kates API's audit log. The command stops after 100 calls, or after a call that deleted nothing. When runs still match then, it exits 1 and says how many are left; run it again to go on. With `-o json` it prints one answer for the whole command, in the fields of [DELETE /api/tests](11-api-reference.md#delete-apitests). `deleted` adds up every call, `matched` is the count the first delete found and `remaining` the one after the last; with `--dry-run`, or when none match, it prints the count. -Once a day the Kates API prunes by itself, the same way: it deletes the `DONE` and `FAILED` runs created more than 90 days ago (`kates.cleanup.retention-days`), each with an audit row. It leaves `PENDING`, `RUNNING` and `STOPPING` runs alone, however old they are. So `prune` is for keeping finished runs for less long. To prune on a schedule, enable the `kates` chart's cleanup CronJob, described in [Where Kates Stores Test Data](12-deployment.md#where-kates-stores-test-data). A Kates API too old to have `DELETE /api/tests` answers `405`, and the command fails, saying to upgrade it. +Once a day the Kates API prunes by itself, the same way: it deletes the `DONE` and `FAILED` runs created more than 90 days ago (`kates.cleanup.retention-days`), each with an audit row. It leaves `PENDING`, `RUNNING` and `STOPPING` runs alone, however old they are. It keeps the run a test type's baseline names too, however old it is. So `prune` is for keeping finished runs for less long. To prune on a schedule, enable the `kates` chart's cleanup CronJob, described in [Where Kates Stores Test Data](12-deployment.md#where-kates-stores-test-data). A Kates API too old to have `DELETE /api/tests` answers `405`, and the command fails, saying to upgrade it. #### test watch diff --git a/docs/book/11-api-reference.md b/docs/book/11-api-reference.md index 189ebd1c..b3e82927 100644 --- a/docs/book/11-api-reference.md +++ b/docs/book/11-api-reference.md @@ -373,11 +373,11 @@ Stop and delete a test run with its results. A run that is still `PENDING` or `R #### DELETE /api/tests -Delete the finished runs created before a cutoff, to keep the run history to a retention period of your own. Only `DONE` and `FAILED` runs match, and a cancelled run is stored as `FAILED`, so it matches too. A run still `PENDING`, `RUNNING` or `STOPPING` is never deleted here; [test cleanup](10-cli-reference.md#test-cleanup) deals with one left `RUNNING`. `kates test prune` and the `kates` chart's cleanup CronJob call this endpoint. +Delete the finished runs created before a cutoff, to keep the run history to a retention period of your own. Only `DONE` and `FAILED` runs match, and a cancelled run is stored as `FAILED`, so it matches too. The run a test type's baseline names never matches, however old it is. A run still `PENDING`, `RUNNING` or `STOPPING` is never deleted here; [test cleanup](10-cli-reference.md#test-cleanup) deals with one left `RUNNING`. `kates test prune` and the `kates` chart's cleanup CronJob call this endpoint. It needs a key with the `admin` scope, as the other deletes do. A call deletes the oldest matching runs first, by `createdAt`, and at most `limit` of them. Each goes as `DELETE /api/tests/{id}` deletes it, with its results, and leaves a row in the audit log naming who pruned it: action `DELETE`, type `test`, the run's ID, and the details `retention: created before `. A dry run leaves none, and a call that deletes nothing, or is refused, leaves one row with its status. `kates audit --type test` lists them. -Once a day the Kates API also prunes by itself, through the same delete: it deletes the `DONE` and `FAILED` runs created more than `kates.cleanup.retention-days` days ago, 90 by default, oldest first and 1,000 at a time. Each run it deletes leaves an audit row too, naming `system:scheduler`, with the details `retention sweep: created before (kates.cleanup.retention-days=)`. A run still `PENDING`, `RUNNING` or `STOPPING` is left alone, however old it is. This endpoint is for keeping finished runs for less long. +Once a day the Kates API also prunes by itself, through the same delete: it deletes the `DONE` and `FAILED` runs created more than `kates.cleanup.retention-days` days ago, 90 by default, oldest first and 1,000 at a time. Each run it deletes leaves an audit row too, naming `system:scheduler`, with the details `retention sweep: created before (kates.cleanup.retention-days=)`. A run still `PENDING`, `RUNNING` or `STOPPING` is left alone, however old it is, and so is the run a test type's baseline names. This endpoint is for keeping finished runs for less long. **Query Parameters:** diff --git a/docs/book/12-deployment.md b/docs/book/12-deployment.md index 74cb02e0..c9f81cae 100644 --- a/docs/book/12-deployment.md +++ b/docs/book/12-deployment.md @@ -79,7 +79,7 @@ The Kates API keeps every run in PostgreSQL. The `kates` chart runs its own Post Whichever database holds them, each replica of the Kates API sweeps out old runs when it starts and every 24 hours after. A sweep deletes the `DONE` and `FAILED` runs created more than `kates.cleanup.retention-days` days ago, 90 by default, so a lower setting deletes as soon as a replica starts with it. It deletes each run as `DELETE /api/tests/{id}` does, with its results, oldest first and 1,000 at a time. Each leaves a row in the audit log naming `system:scheduler`: action `DELETE`, type `test`, the run's ID, and the details `retention sweep: created before (kates.cleanup.retention-days=)`. When two replicas find the same run at once, it is deleted, and audited, once. An error ends that sweep with an `ERROR` line in the Kates API's log, and the next sweep goes on from there. -A sweep leaves a run still `PENDING`, `RUNNING` or `STOPPING` alone, however old it is. The Kates API fails a run still `PENDING` after the submission timeout, and fails a run still `RUNNING` or `STOPPING` after its run timeout, as [Test Management](11-api-reference.md#test-management) describes. A later sweep deletes it once it is older than the retention. +A sweep leaves a run still `PENDING`, `RUNNING` or `STOPPING` alone, however old it is. The Kates API fails a run still `PENDING` after the submission timeout, and fails a run still `RUNNING` or `STOPPING` after its run timeout, as [Test Management](11-api-reference.md#test-management) describes. A later sweep deletes it once it is older than the retention. A sweep also leaves the run a test type's baseline names alone, however old it is, so the regression check has it to compare with. To keep runs for less time, set `cleanup.enabled: true` in the `kates` chart's values. Its CronJob then deletes the finished runs older than `cleanup.retentionDays`, through [DELETE /api/tests](11-api-reference.md#delete-apitests), with the key from the Secret the `apiKey` values name. The table gives the values and their defaults: diff --git a/kates/docs/api-reference.md b/kates/docs/api-reference.md index 74800683..77a3141a 100644 --- a/kates/docs/api-reference.md +++ b/kates/docs/api-reference.md @@ -223,11 +223,11 @@ DELETE /api/tests?createdBefore=2026-09-07T00:00:00Z DELETE /api/tests?createdBefore=2026-09-07T00:00:00Z&status=FAILED&dryRun=true ``` -Deletes the finished runs created before a cutoff, to keep the run history to a retention period of your own. Only `DONE` and `FAILED` runs match; a cancelled run is stored as `FAILED`, so it matches too. A run still pending, running or stopping is never deleted here: `kates test cleanup` deals with one left running. `kates test prune` and the chart's cleanup CronJob (`cleanup.*` in `charts/kates/values.yaml`) call this endpoint. +Deletes the finished runs created before a cutoff, to keep the run history to a retention period of your own. Only `DONE` and `FAILED` runs match; a cancelled run is stored as `FAILED`, so it matches too. The run a test type's baseline names never matches, however old it is. A run still pending, running or stopping is never deleted here: `kates test cleanup` deals with one left running. `kates test prune` and the chart's cleanup CronJob (`cleanup.*` in `charts/kates/values.yaml`) call this endpoint. A call deletes the oldest matching runs first, by `createdAt`, and at most `limit` of them. Each goes through the same delete as `DELETE /api/tests/{id}`, and leaves an audit row: action `DELETE`, type `test`, the run's ID, and the details `retention: created before `. -Once a day the Kates API also prunes by itself, through the same delete: it deletes the `DONE` and `FAILED` runs created more than `kates.cleanup.retention-days` (90) days ago, oldest first and 1,000 at a time. Each leaves an audit row too, naming `system:scheduler`, with the details `retention sweep: created before (kates.cleanup.retention-days=)`. A run still pending, running or stopping is left alone, however old it is. This endpoint is for keeping finished runs for less long. +Once a day the Kates API also prunes by itself, through the same delete: it deletes the `DONE` and `FAILED` runs created more than `kates.cleanup.retention-days` (90) days ago, oldest first and 1,000 at a time. Each leaves an audit row too, naming `system:scheduler`, with the details `retention sweep: created before (kates.cleanup.retention-days=)`. A run still pending, running or stopping is left alone, however old it is, and so is the run a test type's baseline names. This endpoint is for keeping finished runs for less long. **Query Parameters:** diff --git a/kates/src/main/java/com/bmscomp/kates/api/TestResource.java b/kates/src/main/java/com/bmscomp/kates/api/TestResource.java index acf77094..ce960a34 100644 --- a/kates/src/main/java/com/bmscomp/kates/api/TestResource.java +++ b/kates/src/main/java/com/bmscomp/kates/api/TestResource.java @@ -282,7 +282,7 @@ public Response bulkDelete(com.bmscomp.kates.domain.BulkDeleteRequest request) { + " finished is never pruned. matched counts the runs that matched before the call, deleted" + " those the call deleted, and remaining those that still match: a caller calls again until" + " remaining is 0. A dry run counts and deletes nothing. The chart's cleanup CronJob and" - + " kates test prune call it.") + + " kates test prune call it. A run a test type's baseline names is never pruned.") @APIResponse( responseCode = "200", description = "What matched, what the call deleted, and what still matches", diff --git a/kates/src/main/java/com/bmscomp/kates/service/TestRunRepository.java b/kates/src/main/java/com/bmscomp/kates/service/TestRunRepository.java index a07c0e28..bc2e9834 100644 --- a/kates/src/main/java/com/bmscomp/kates/service/TestRunRepository.java +++ b/kates/src/main/java/com/bmscomp/kates/service/TestRunRepository.java @@ -286,11 +286,14 @@ public long countByStatus(com.bmscomp.kates.domain.TestResult.TaskStatus status) /** * How many runs in one of {@code statuses} were created before * {@code before}: what {@link RunRetention} would prune, and what it left. + * A run a test type's baseline names is left out, as it is from + * {@link #findIdsByStatusCreatedBefore}. */ public long countByStatusCreatedBefore( java.util.Set statuses, java.time.Instant before) { return em.createQuery( - "SELECT COUNT(r) FROM TestRunEntity r WHERE r.status IN :statuses AND r.createdAt < :before", + "SELECT COUNT(r) FROM TestRunEntity r WHERE r.status IN :statuses AND r.createdAt < :before" + + " AND r.id NOT IN (SELECT b.runId FROM BaselineEntity b)", Long.class) .setParameter("statuses", statuses) .setParameter("before", before) @@ -304,7 +307,9 @@ public long countByStatusCreatedBefore( *

Ids, not runs: {@link RunRetention} deletes each through the * orchestrator, which reads the run under its row lock itself. The id * orders runs created in the same instant, so a caller that prunes a - * limit's worth per call goes through them in one order. + * limit's worth per call goes through them in one order. A run a test + * type's baseline names is left out: the regression check compares runs + * against it, so retention keeps it however old it is. */ public List findIdsByStatusCreatedBefore( java.util.Set statuses, @@ -312,6 +317,7 @@ public List findIdsByStatusCreatedBefore( int limit) { return em.createQuery( "SELECT r.id FROM TestRunEntity r WHERE r.status IN :statuses AND r.createdAt < :before" + + " AND r.id NOT IN (SELECT b.runId FROM BaselineEntity b)" + " ORDER BY r.createdAt ASC, r.id ASC", String.class) .setParameter("statuses", statuses) diff --git a/kates/src/test/java/com/bmscomp/kates/api/TestResourcePruneTest.java b/kates/src/test/java/com/bmscomp/kates/api/TestResourcePruneTest.java index 136da264..c918d6c1 100644 --- a/kates/src/test/java/com/bmscomp/kates/api/TestResourcePruneTest.java +++ b/kates/src/test/java/com/bmscomp/kates/api/TestResourcePruneTest.java @@ -31,6 +31,7 @@ import com.bmscomp.kates.domain.TestRun; import com.bmscomp.kates.domain.TestType; import com.bmscomp.kates.service.AuditService; +import com.bmscomp.kates.service.BaselineService; import com.bmscomp.kates.service.TestRunRepository; /** @@ -59,6 +60,9 @@ class TestResourcePruneTest { @Inject AuditService auditService; + @Inject + BaselineService baselines; + @TestHTTPResource("/api/tests") URI tests; @@ -72,6 +76,7 @@ void start() { @AfterEach void deleteTheRuns() { + baselines.unset(TestType.LOAD); stored.forEach(repository::delete); stored.clear(); } @@ -214,6 +219,30 @@ void deletesTheOldestFinishedRunsFirstUpToTheLimit() { assertEquals(Map.of(oldest, row, cancelled, row, third, row), auditRows()); } + /** + * The run a baseline names stays, however old: the regression check + * compares each run of its type against it, and answered 404 for every + * one once retention had deleted it. matched leaves it out too, so + * remaining reaches 0 and a caller stops. + */ + @Test + void aBaselineRunIsNeverPruned() { + String baseline = store(TaskStatus.DONE, day(1)); + String other = store(TaskStatus.DONE, day(2)); + baselines.set(TestType.LOAD, baseline); + + assertEquals(1, prune(given().queryParam("dryRun", "true")).getLong("matched")); + + JsonPath answer = prune(given()); + + assertEquals(1, answer.getLong("matched")); + assertEquals(1, answer.getInt("deleted")); + assertEquals(0, answer.getLong("remaining")); + assertStored(baseline); + assertGone(other); + assertEquals(Map.of(other, "DELETE: retention: created before 2001-01-11T00:00:00Z"), auditRows()); + } + /** status is case-insensitive, and a run of a status not asked for stays. */ @Test void aStatusFilterPrunesOnlyThatStatus() {