From 3959f7b4c90c20fed7cfac5c09bf44035c65e83a Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 15 Nov 2026 23:54:52 +0000 Subject: [PATCH 1/3] fix(rollback): a batch rollback reverts newest first, and a second rollback is a no-op (E0-25) The manifest followed the order of File.ls/1 (no fixed order) and listed the outcomes oldest first, and a second rollback of the same batch wrote its false-positive rows again, because the rows it wrote carry the batch id. * The outcomes of a batch are read in the order of the log (file names sorted, one file for each month, then the lines of each file) and listed from the last to the first. They are not sorted by timestamp, repo or file: none is an order, and a tie would revert dependent changes the wrong way round. * A row that a rollback wrote (caused_by batch_rollback) is not an outcome of the batch. * A batch whose status is rolled_back returns {:ok, 0} and writes nothing. * Every write is read. The status is set only after all of them succeed; a failed write returns {:error, reason} and leaves the status, so a later call can finish. * A retry writes no second outcome row: for each key (recipe_id, repo, file), with N originals and M rollback rows, it writes N - M. A manifest is written once for a batch, so a retry after a failed status write adds no second list of instructions. Co-Authored-By: Claude --- lib/safety/batch_rollback.ex | 189 ++++++++++++++++++++------ test/safety_test.exs | 250 +++++++++++++++++++++++++++++++++++ 2 files changed, 401 insertions(+), 38 deletions(-) diff --git a/lib/safety/batch_rollback.ex b/lib/safety/batch_rollback.ex index dd8f730a..704cdae0 100644 --- a/lib/safety/batch_rollback.ex +++ b/lib/safety/batch_rollback.ex @@ -45,55 +45,162 @@ defmodule Hypatia.Safety.BatchRollback do update_batch_status(batch_id, "completed") end - @doc "Roll back an entire batch" + @doc """ + Roll back an entire batch. + + The manifest lists the revert instructions from the NEWEST outcome to the oldest, so + that a change that depends on an earlier one is reverted first. The order is the order + of the outcome log (files by name, one for each month, then the lines of each file), + from its last position to its first. It is not sorted by `timestamp`, `repo` or `file`: + none of them is an order, and a tie would revert dependent changes the wrong way round. + + A rollback is safe to repeat (E0-25): + + * a batch whose status is `rolled_back` returns `{:ok, 0}` and writes nothing; + * the status is set only after every write succeeded. A failed write returns + `{:error, reason}` and leaves the status as it was, so that a later call can + finish the rollback; + * a retry writes no second outcome row for an outcome that already has its rollback + row. For each key (`recipe_id`, `repo`, `file`), with N original outcomes and M + rollback rows already in the log, it writes N - M rows (none when M >= N). The + manifest still lists every original outcome, once. + """ def rollback(batch_id, reason) do - Logger.warning("ROLLBACK batch #{batch_id}: #{reason}") + if batch_status(batch_id) == "rolled_back" do + {:ok, 0} + else + do_rollback(batch_id, reason) + end + end - # 1. Find all outcomes for this batch - outcomes = find_batch_outcomes(batch_id) + defp do_rollback(batch_id, reason) do + Logger.warning("ROLLBACK batch #{batch_id}: #{reason}") - # 2. Mark all as false_positive - Enum.each(outcomes, fn outcome -> - recipe_id = Map.get(outcome, "recipe_id") + # 1. The rows of this batch, in the order of the log. A row that a rollback wrote + # is not an outcome of the batch: it is what a rollback did to one. + {originals, written} = + batch_id + |> find_batch_outcomes() + |> Enum.split_with(&(Map.get(&1, "caused_by") != "batch_rollback")) + + with :ok <- mark_false_positives(originals, written, batch_id), + :ok <- write_manifest(batch_id, reason, originals), + :ok <- set_status(batch_id, "rolled_back") do + Logger.warning( + "Rollback complete: #{length(originals)} outcomes reverted for batch #{batch_id}" + ) + + {:ok, length(originals)} + else + {:error, why} = error -> + Logger.error("Rollback of batch #{batch_id} did not complete: #{inspect(why)}") + error + end + end - if recipe_id do - Hypatia.OutcomeTracker.record_batch_rollback( - recipe_id, - Map.get(outcome, "repo", "unknown"), - Map.get(outcome, "file", ""), - batch_id - ) + # Write one rollback row for each original outcome that has none yet. Outcomes have + # no stable id, and 2 of them can share a batch, recipe, repo and file, so they are + # counted by key: the first M originals of a key are the ones that already have a row. + defp mark_false_positives(originals, written, batch_id) do + already = Enum.frequencies_by(written, &outcome_key/1) + + originals + |> Enum.reduce_while({:ok, already}, fn outcome, {:ok, skip} -> + key = outcome_key(outcome) + + cond do + is_nil(Map.get(outcome, "recipe_id")) -> + {:cont, {:ok, skip}} + + Map.get(skip, key, 0) > 0 -> + {:cont, {:ok, Map.update!(skip, key, &(&1 - 1))}} + + true -> + case Hypatia.OutcomeTracker.record_batch_rollback( + Map.get(outcome, "recipe_id"), + Map.get(outcome, "repo", "unknown"), + Map.get(outcome, "file", ""), + batch_id + ) do + {:ok, _record} -> {:cont, {:ok, skip}} + {:error, why} -> {:halt, {:error, {:outcome_write_failed, why}}} + other -> {:halt, {:error, {:outcome_write_failed, other}}} + end end end) + |> case do + {:ok, _skip} -> :ok + error -> error + end + end - # 3. Generate rollback manifest - rollback_manifest = %{ - "batch_id" => batch_id, - "reason" => reason, - "timestamp" => DateTime.utc_now() |> DateTime.to_iso8601(), - "affected_outcomes" => length(outcomes), - "action" => "revert", - "instructions" => - Enum.map(outcomes, fn o -> - %{ - "repo" => Map.get(o, "repo"), - "file" => Map.get(o, "file"), - "action" => "git revert" - } - end) - } + defp outcome_key(outcome) do + {Map.get(outcome, "recipe_id"), Map.get(outcome, "repo", "unknown"), + Map.get(outcome, "file", "")} + end + # The manifest, newest outcome first. One is written for a batch, ever: a retry after + # a failed status write must not add a second list of instructions. + defp write_manifest(batch_id, reason, originals) do rollback_path = Path.join([Path.expand(@verisimdb_data_path), "dispatch", "rollbacks.jsonl"]) - File.write(rollback_path, Jason.encode!(rollback_manifest) <> "\n", [:append, :utf8]) - # 4. Update batch status - update_batch_status(batch_id, "rolled_back") + if manifest_written?(rollback_path, batch_id) do + :ok + else + manifest = %{ + "batch_id" => batch_id, + "reason" => reason, + "timestamp" => DateTime.utc_now() |> DateTime.to_iso8601(), + "affected_outcomes" => length(originals), + "action" => "revert", + "instructions" => + originals + |> Enum.reverse() + |> Enum.map(fn o -> + %{ + "repo" => Map.get(o, "repo"), + "file" => Map.get(o, "file"), + "action" => "git revert" + } + end) + } + + case File.write(rollback_path, Jason.encode!(manifest) <> "\n", [:append, :utf8]) do + :ok -> :ok + {:error, why} -> {:error, {:manifest_write_failed, why}} + end + end + end - Logger.warning( - "Rollback complete: #{length(outcomes)} outcomes reverted for batch #{batch_id}" - ) + defp manifest_written?(path, batch_id) do + case File.read(path) do + {:ok, content} -> + content + |> String.split("\n", trim: true) + |> Enum.any?(fn line -> + match?({:ok, %{"batch_id" => ^batch_id}}, Jason.decode(line)) + end) - {:ok, length(outcomes)} + _ -> + false + end + end + + defp set_status(batch_id, status) do + case update_batch_status(batch_id, status) do + :ok -> :ok + {:error, why} -> {:error, {:status_write_failed, why}} + end + end + + # The latest status that the batch log gives for this batch, or nil. + defp batch_status(batch_id) do + list_batches_from_file(:infinity) + |> Enum.find(&(Map.get(&1, "batch_id") == batch_id and Map.has_key?(&1, "status"))) + |> case do + nil -> nil + entry -> Map.get(entry, "status") + end end @doc "List recent batches with their status" @@ -110,6 +217,9 @@ defmodule Hypatia.Safety.BatchRollback do {:ok, files} -> files |> Enum.filter(&String.ends_with?(&1, ".jsonl")) + # File.ls/1 has no fixed order. One file for each month, so a name sort is a + # time sort; the lines of a file keep their order. + |> Enum.sort() |> Enum.flat_map(fn f -> path = Path.join(outcomes_dir, f) @@ -147,6 +257,9 @@ defmodule Hypatia.Safety.BatchRollback do File.write(batch_path, update <> "\n", [:append, :utf8]) end + defp take_limit(batches, :infinity), do: batches + defp take_limit(batches, limit), do: Enum.take(batches, limit) + defp list_batches_from_file(limit) do batch_path = Path.join([Path.expand(@verisimdb_data_path), "dispatch", "batches.jsonl"]) @@ -158,7 +271,7 @@ defmodule Hypatia.Safety.BatchRollback do |> Enum.filter(&match?({:ok, _}, &1)) |> Enum.map(fn {:ok, b} -> b end) |> Enum.reverse() - |> Enum.take(limit) + |> take_limit(limit) _ -> [] diff --git a/test/safety_test.exs b/test/safety_test.exs index 34f54c49..c8dbf1f1 100644 --- a/test/safety_test.exs +++ b/test/safety_test.exs @@ -360,3 +360,253 @@ defmodule Hypatia.Safety.BatchRollbackTest do end end end + +# E0-25: the rollback manifest of a batch lists its revert instructions from the newest +# outcome to the oldest, and a second rollback of the same batch is a no-op. +defmodule Hypatia.Safety.BatchRollbackOrderTest do + use ExUnit.Case, async: false + + alias Hypatia.Safety.BatchRollback + + setup do + data = Application.fetch_env!(:hypatia, :verisimdb_data_path) |> Path.expand() + outcomes = Path.join(data, "outcomes") + dispatch = Path.join(data, "dispatch") + File.mkdir_p!(outcomes) + File.mkdir_p!(dispatch) + + uniq = System.unique_integer([:positive]) + {:ok, batch} = BatchRollback.create_batch(3, :auto_execute) + created = [Path.join(outcomes, "e0-25-#{uniq}-*.jsonl")] + + on_exit(fn -> + for pattern <- created, f <- Path.wildcard(pattern), do: File.rm(f) + File.rm_rf(Path.join(dispatch, "rollbacks.jsonl.moved-#{uniq}")) + end) + + %{ + batch: batch, + uniq: uniq, + outcomes: outcomes, + dispatch: dispatch, + manifests: Path.join(dispatch, "rollbacks.jsonl") + } + end + + defp row(batch, file, extra \\ %{}) do + Map.merge( + %{ + "recipe_id" => "recipe-e0-25-order", + "repo" => "owner/repo", + "file" => file, + "outcome" => "success", + "verification" => "verified", + "batch_id" => batch, + "timestamp" => "2026-11-15T10:00:00Z", + "bot" => "hypatia" + }, + extra + ) + end + + defp seed(ctx, name, rows) do + path = Path.join(ctx.outcomes, "e0-25-#{ctx.uniq}-#{name}.jsonl") + File.write!(path, Enum.map_join(rows, "", &(Jason.encode!(&1) <> "\n"))) + path + end + + defp lines(path) do + case File.read(path) do + {:ok, content} -> content |> String.split("\n", trim: true) |> Enum.map(&Jason.decode!/1) + _ -> [] + end + end + + defp batch_rows(ctx) do + ctx.outcomes + |> Path.join("*.jsonl") + |> Path.wildcard() + |> Enum.flat_map(&lines/1) + |> Enum.filter(&(&1["batch_id"] == ctx.batch)) + end + + defp rollback_rows(ctx), + do: Enum.filter(batch_rows(ctx), &(&1["caused_by"] == "batch_rollback")) + + defp manifests_of(ctx), do: Enum.filter(lines(ctx.manifests), &(&1["batch_id"] == ctx.batch)) + defp instruction_files(manifest), do: Enum.map(manifest["instructions"], & &1["file"]) + + defp status(ctx) do + BatchRollback.list_batches(10_000) + |> Enum.find(&(&1["batch_id"] == ctx.batch and Map.has_key?(&1, "status"))) + |> Map.fetch!("status") + end + + # Make rollbacks.jsonl unwritable (a directory cannot be appended to), and put it + # back afterwards. + defp break_manifest_log(ctx) do + moved = ctx.manifests <> ".moved-#{ctx.uniq}" + if File.exists?(ctx.manifests), do: File.rename!(ctx.manifests, moved) + File.mkdir_p!(ctx.manifests) + + fn -> + File.rm_rf!(ctx.manifests) + if File.exists?(moved), do: File.rename!(moved, ctx.manifests) + end + end + + test "3 outcomes written oldest first give a manifest that lists them newest first", ctx do + seed(ctx, "a", [row(ctx.batch, "f1.txt"), row(ctx.batch, "f2.txt"), row(ctx.batch, "f3.txt")]) + + assert {:ok, 3} = BatchRollback.rollback(ctx.batch, "t") + assert [manifest] = manifests_of(ctx) + assert instruction_files(manifest) == ["f3.txt", "f2.txt", "f1.txt"] + assert manifest["affected_outcomes"] == 3 + end + + test "2 outcomes with the same timestamp are listed in the reverse of the log, not by repo and file", + ctx do + # Written b, then a: against the order of file names. Same timestamp. + seed(ctx, "a", [row(ctx.batch, "b.txt"), row(ctx.batch, "a.txt")]) + + assert {:ok, 2} = BatchRollback.rollback(ctx.batch, "t") + assert [manifest] = manifests_of(ctx) + assert instruction_files(manifest) == ["a.txt", "b.txt"] + end + + test "outcomes in 2 monthly files: the later month is listed first, whatever File.ls gives", + ctx do + # Month names whose File.ls order is not their time order, if this file system + # has one. If it lists them in time order the case below cannot tell the + # difference, and it says so. + months = for m <- 1..12, do: "2099-" <> String.pad_leading("#{m}", 2, "0") + for m <- months, do: seed(ctx, m, []) + + listed = + File.ls!(ctx.outcomes) + |> Enum.filter(&String.starts_with?(&1, "e0-25-#{ctx.uniq}-")) + |> Enum.map( + &(&1 + |> String.replace_prefix("e0-25-#{ctx.uniq}-", "") + |> String.trim_trailing(".jsonl")) + ) + + pair = + for {a, i} <- Enum.with_index(listed), + {b, j} <- Enum.with_index(listed), + i < j, + a > b, + do: {b, a} + + {early, late} = List.first(pair) || {"2099-01", "2099-02"} + + if pair == [], + do: + IO.puts( + "note: File.ls lists the month files in time order here; the order is not exercised" + ) + + seed(ctx, early, [row(ctx.batch, "early.txt")]) + seed(ctx, late, [row(ctx.batch, "late.txt")]) + + assert {:ok, 2} = BatchRollback.rollback(ctx.batch, "t") + assert [manifest] = manifests_of(ctx) + assert instruction_files(manifest) == ["late.txt", "early.txt"] + end + + test "a failed write of rollbacks.jsonl returns an error and does not set rolled_back", ctx do + seed(ctx, "a", [row(ctx.batch, "f1.txt")]) + restore = break_manifest_log(ctx) + + try do + assert {:error, _reason} = BatchRollback.rollback(ctx.batch, "t") + refute status(ctx) == "rolled_back" + after + restore.() + end + end + + test "a retry after a failed manifest write completes the rollback with no second outcome row", + ctx do + seed(ctx, "a", [row(ctx.batch, "f1.txt"), row(ctx.batch, "f2.txt")]) + restore = break_manifest_log(ctx) + + try do + assert {:error, _} = BatchRollback.rollback(ctx.batch, "t") + after + restore.() + end + + written_by_the_first_try = length(rollback_rows(ctx)) + assert written_by_the_first_try == 2 + + assert {:ok, 2} = BatchRollback.rollback(ctx.batch, "t") + + assert length(rollback_rows(ctx)) == 2, "the retry wrote a second outcome row" + assert [manifest] = manifests_of(ctx) + assert instruction_files(manifest) == ["f2.txt", "f1.txt"] + assert status(ctx) == "rolled_back" + end + + test "2 originals with the same key, 1 rollback row already written: the retry writes exactly 1 more", + ctx do + seed(ctx, "a", [row(ctx.batch, "same.txt"), row(ctx.batch, "same.txt")]) + + # The first rollback row was written, and the second write failed. + assert {:ok, _} = + Hypatia.OutcomeTracker.record_batch_rollback( + "recipe-e0-25-order", + "owner/repo", + "same.txt", + ctx.batch + ) + + assert length(rollback_rows(ctx)) == 1 + + assert {:ok, 2} = BatchRollback.rollback(ctx.batch, "t") + assert length(rollback_rows(ctx)) == 2 + assert [manifest] = manifests_of(ctx) + assert instruction_files(manifest) == ["same.txt", "same.txt"] + end + + test "a failed status write after the manifest was written: the retry adds no second manifest", + ctx do + seed(ctx, "a", [row(ctx.batch, "f1.txt"), row(ctx.batch, "f2.txt")]) + + # Make batches.jsonl unwritable for the status update only. + batches = Path.join(ctx.dispatch, "batches.jsonl") + moved = batches <> ".moved-#{ctx.uniq}" + File.rename!(batches, moved) + File.mkdir_p!(batches) + + try do + assert {:error, {:status_write_failed, _}} = BatchRollback.rollback(ctx.batch, "t") + after + File.rm_rf!(batches) + File.rename!(moved, batches) + end + + assert length(manifests_of(ctx)) == 1 + assert length(rollback_rows(ctx)) == 2 + refute status(ctx) == "rolled_back" + + assert {:ok, 2} = BatchRollback.rollback(ctx.batch, "t") + assert length(manifests_of(ctx)) == 1, "the retry wrote a second manifest" + assert length(rollback_rows(ctx)) == 2 + assert status(ctx) == "rolled_back" + end + + test "a second rollback of the same batch adds no outcome row and no manifest line", ctx do + seed(ctx, "a", [row(ctx.batch, "f1.txt"), row(ctx.batch, "f2.txt")]) + + assert {:ok, 2} = BatchRollback.rollback(ctx.batch, "t") + assert status(ctx) == "rolled_back" + rows_after_first = length(batch_rows(ctx)) + manifests_after_first = length(manifests_of(ctx)) + assert manifests_after_first == 1 + + assert {:ok, 0} = BatchRollback.rollback(ctx.batch, "again") + assert length(batch_rows(ctx)) == rows_after_first + assert length(manifests_of(ctx)) == manifests_after_first + end +end From 8882cd033efaf4056568a82b5bda7ca748e7c112 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 02:28:10 +0000 Subject: [PATCH 2/3] fix(rollback): read failures stop a rollback, and one rollback of a batch runs at a time Review findings on E0-25: - an unreadable outcome file or directory is an error, not an empty list: a partial snapshot gave an incomplete manifest and then a final rolled_back status - only :enoent means "no manifest" / "no batch status"; any other read failure of rollbacks.jsonl or batches.jsonl is returned as an error before anything is appended - the status check, snapshot, writes and status write run under a per-batch :global lock (this VM), with the status read again inside it Tests: 4 new (unreadable outcome file, manifest log, batch log; 8 overlapping rollbacks). The status-write-failure test now seeds the state a failed status write leaves, because that fault can no longer be injected with an unreadable batch log. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01J4QvDj7XpKyxE4WbsgTsTh --- lib/safety/batch_rollback.ex | 178 ++++++++++++++++++++++------------- test/safety_test.exs | 106 ++++++++++++++++++--- 2 files changed, 206 insertions(+), 78 deletions(-) diff --git a/lib/safety/batch_rollback.ex b/lib/safety/batch_rollback.ex index 704cdae0..597333a8 100644 --- a/lib/safety/batch_rollback.ex +++ b/lib/safety/batch_rollback.ex @@ -66,24 +66,36 @@ defmodule Hypatia.Safety.BatchRollback do manifest still lists every original outcome, once. """ def rollback(batch_id, reason) do - if batch_status(batch_id) == "rolled_back" do - {:ok, 0} - else - do_rollback(batch_id, reason) + # One rollback of a batch at a time, in this VM: the status check, the outcome + # snapshot, the writes and the status write are one critical section, so that two + # callers cannot both read `pending` and both append rows and a manifest. The + # status is read again inside the lock. (A lock between VMs or OS processes is not + # taken: the module has one writer node.) + :global.trans({{__MODULE__, batch_id}, self()}, fn -> locked_rollback(batch_id, reason) end) + end + + defp locked_rollback(batch_id, reason) do + case batch_status(batch_id) do + {:ok, "rolled_back"} -> {:ok, 0} + {:ok, _status} -> do_rollback(batch_id, reason) + {:error, _} = error -> fail(batch_id, error) end end + defp fail(batch_id, {:error, why} = error) do + Logger.error("Rollback of batch #{batch_id} did not complete: #{inspect(why)}") + error + end + defp do_rollback(batch_id, reason) do Logger.warning("ROLLBACK batch #{batch_id}: #{reason}") # 1. The rows of this batch, in the order of the log. A row that a rollback wrote # is not an outcome of the batch: it is what a rollback did to one. - {originals, written} = - batch_id - |> find_batch_outcomes() - |> Enum.split_with(&(Map.get(&1, "caused_by") != "batch_rollback")) - - with :ok <- mark_false_positives(originals, written, batch_id), + with {:ok, outcomes} <- find_batch_outcomes(batch_id), + {originals, written} = + Enum.split_with(outcomes, &(Map.get(&1, "caused_by") != "batch_rollback")), + :ok <- mark_false_positives(originals, written, batch_id), :ok <- write_manifest(batch_id, reason, originals), :ok <- set_status(batch_id, "rolled_back") do Logger.warning( @@ -92,9 +104,7 @@ defmodule Hypatia.Safety.BatchRollback do {:ok, length(originals)} else - {:error, why} = error -> - Logger.error("Rollback of batch #{batch_id} did not complete: #{inspect(why)}") - error + {:error, _} = error -> fail(batch_id, error) end end @@ -144,45 +154,55 @@ defmodule Hypatia.Safety.BatchRollback do defp write_manifest(batch_id, reason, originals) do rollback_path = Path.join([Path.expand(@verisimdb_data_path), "dispatch", "rollbacks.jsonl"]) - if manifest_written?(rollback_path, batch_id) do - :ok - else - manifest = %{ - "batch_id" => batch_id, - "reason" => reason, - "timestamp" => DateTime.utc_now() |> DateTime.to_iso8601(), - "affected_outcomes" => length(originals), - "action" => "revert", - "instructions" => - originals - |> Enum.reverse() - |> Enum.map(fn o -> - %{ - "repo" => Map.get(o, "repo"), - "file" => Map.get(o, "file"), - "action" => "git revert" - } - end) - } + case manifest_written?(rollback_path, batch_id) do + {:ok, true} -> :ok + {:ok, false} -> append_manifest(rollback_path, batch_id, reason, originals) + {:error, _} = error -> error + end + end - case File.write(rollback_path, Jason.encode!(manifest) <> "\n", [:append, :utf8]) do - :ok -> :ok - {:error, why} -> {:error, {:manifest_write_failed, why}} - end + defp append_manifest(rollback_path, batch_id, reason, originals) do + manifest = %{ + "batch_id" => batch_id, + "reason" => reason, + "timestamp" => DateTime.utc_now() |> DateTime.to_iso8601(), + "affected_outcomes" => length(originals), + "action" => "revert", + "instructions" => + originals + |> Enum.reverse() + |> Enum.map(fn o -> + %{ + "repo" => Map.get(o, "repo"), + "file" => Map.get(o, "file"), + "action" => "git revert" + } + end) + } + + case File.write(rollback_path, Jason.encode!(manifest) <> "\n", [:append, :utf8]) do + :ok -> :ok + {:error, why} -> {:error, {:manifest_write_failed, why}} end end defp manifest_written?(path, batch_id) do case File.read(path) do {:ok, content} -> - content - |> String.split("\n", trim: true) - |> Enum.any?(fn line -> - match?({:ok, %{"batch_id" => ^batch_id}}, Jason.decode(line)) - end) - - _ -> - false + {:ok, + content + |> String.split("\n", trim: true) + |> Enum.any?(fn line -> + match?({:ok, %{"batch_id" => ^batch_id}}, Jason.decode(line)) + end)} + + # Only a missing log means that no manifest was written. Any other failure may hide + # one, and appending a second would break the idempotency of a retry. + {:error, :enoent} -> + {:ok, false} + + {:error, why} -> + {:error, {:manifest_read_failed, why}} end end @@ -194,12 +214,30 @@ defmodule Hypatia.Safety.BatchRollback do end # The latest status that the batch log gives for this batch, or nil. + # {:error, _} when the log cannot be read: that is not "no status", and a rolled_back + # row may be in it. A missing log is no status. defp batch_status(batch_id) do - list_batches_from_file(:infinity) - |> Enum.find(&(Map.get(&1, "batch_id") == batch_id and Map.has_key?(&1, "status"))) - |> case do - nil -> nil - entry -> Map.get(entry, "status") + batch_path = Path.join([Path.expand(@verisimdb_data_path), "dispatch", "batches.jsonl"]) + + case File.read(batch_path) do + {:ok, content} -> + status = + content + |> String.split("\n", trim: true) + |> Enum.map(&Jason.decode/1) + |> Enum.flat_map(fn + {:ok, %{"batch_id" => ^batch_id, "status" => status}} -> [status] + _ -> [] + end) + |> List.last() + + {:ok, status} + + {:error, :enoent} -> + {:ok, nil} + + {:error, why} -> + {:error, {:batch_log_read_failed, why}} end end @@ -210,6 +248,9 @@ defmodule Hypatia.Safety.BatchRollback do # --- Internal --- + # {:ok, rows} in log order, or {:error, _} if the log directory or any monthly file + # cannot be read: a partial list would give an incomplete manifest, and the status + # that follows would make that final. A missing directory is an empty log. defp find_batch_outcomes(batch_id) do outcomes_dir = Path.join(Path.expand(@verisimdb_data_path), "outcomes") @@ -220,27 +261,36 @@ defmodule Hypatia.Safety.BatchRollback do # File.ls/1 has no fixed order. One file for each month, so a name sort is a # time sort; the lines of a file keep their order. |> Enum.sort() - |> Enum.flat_map(fn f -> + |> Enum.reduce_while({:ok, []}, fn f, {:ok, acc} -> path = Path.join(outcomes_dir, f) case File.read(path) do {:ok, content} -> - content - |> String.split("\n", trim: true) - |> Enum.map(&Jason.decode/1) - |> Enum.filter(fn - {:ok, o} -> Map.get(o, "batch_id") == batch_id - _ -> false - end) - |> Enum.map(fn {:ok, o} -> o end) - - _ -> - [] + rows = + content + |> String.split("\n", trim: true) + |> Enum.map(&Jason.decode/1) + |> Enum.flat_map(fn + {:ok, %{"batch_id" => ^batch_id} = o} -> [o] + _ -> [] + end) + + {:cont, {:ok, [rows | acc]}} + + {:error, why} -> + {:halt, {:error, {:outcome_read_failed, f, why}}} end end) + |> case do + {:ok, chunks} -> {:ok, chunks |> Enum.reverse() |> Enum.concat()} + error -> error + end - _ -> - [] + {:error, :enoent} -> + {:ok, []} + + {:error, why} -> + {:error, {:outcome_read_failed, outcomes_dir, why}} end end diff --git a/test/safety_test.exs b/test/safety_test.exs index c8dbf1f1..c6313aa9 100644 --- a/test/safety_test.exs +++ b/test/safety_test.exs @@ -569,25 +569,30 @@ defmodule Hypatia.Safety.BatchRollbackOrderTest do assert instruction_files(manifest) == ["same.txt", "same.txt"] end - test "a failed status write after the manifest was written: the retry adds no second manifest", + test "a rollback that left its rows and manifest but no status: the retry adds no second manifest", ctx do seed(ctx, "a", [row(ctx.batch, "f1.txt"), row(ctx.batch, "f2.txt")]) - # Make batches.jsonl unwritable for the status update only. - batches = Path.join(ctx.dispatch, "batches.jsonl") - moved = batches <> ".moved-#{ctx.uniq}" - File.rename!(batches, moved) - File.mkdir_p!(batches) - - try do - assert {:error, {:status_write_failed, _}} = BatchRollback.rollback(ctx.batch, "t") - after - File.rm_rf!(batches) - File.rename!(moved, batches) + # The state that a rollback leaves when its status write fails (a status write cannot + # be made to fail here once an unreadable batch log is an error, and the tests may + # run as root): both rollback rows and the manifest are in the logs, the status is + # still `pending`. + for f <- ["f1.txt", "f2.txt"] do + {:ok, _} = + Hypatia.OutcomeTracker.record_batch_rollback( + "recipe-e0-25-order", + "owner/repo", + f, + ctx.batch + ) end - assert length(manifests_of(ctx)) == 1 - assert length(rollback_rows(ctx)) == 2 + File.write!( + ctx.manifests, + Jason.encode!(%{"batch_id" => ctx.batch, "instructions" => []}) <> "\n", + [:append, :utf8] + ) + refute status(ctx) == "rolled_back" assert {:ok, 2} = BatchRollback.rollback(ctx.batch, "t") @@ -609,4 +614,77 @@ defmodule Hypatia.Safety.BatchRollbackOrderTest do assert length(batch_rows(ctx)) == rows_after_first assert length(manifests_of(ctx)) == manifests_after_first end + + test "an unreadable monthly outcome file stops the rollback; the retry reverts every outcome", + ctx do + seed(ctx, "a", [row(ctx.batch, "f1.txt")]) + # A directory with a .jsonl name cannot be read as a file (EISDIR); it sorts last. + bad = Path.join(ctx.outcomes, "e0-25-#{ctx.uniq}-b.jsonl") + File.mkdir_p!(bad) + + try do + assert {:error, {:outcome_read_failed, _, _}} = BatchRollback.rollback(ctx.batch, "t") + assert manifests_of(ctx) == [] + assert rollback_rows(ctx) == [] + refute status(ctx) == "rolled_back" + after + File.rm_rf!(bad) + end + + seed(ctx, "b", [row(ctx.batch, "f2.txt")]) + assert {:ok, 2} = BatchRollback.rollback(ctx.batch, "t") + assert [manifest] = manifests_of(ctx) + assert instruction_files(manifest) == ["f2.txt", "f1.txt"] + assert status(ctx) == "rolled_back" + end + + test "an unreadable manifest log is an error, not proof that no manifest exists", ctx do + seed(ctx, "a", [row(ctx.batch, "f1.txt")]) + restore = break_manifest_log(ctx) + + try do + assert {:error, {:manifest_read_failed, _}} = BatchRollback.rollback(ctx.batch, "t") + refute status(ctx) == "rolled_back" + after + restore.() + end + + assert {:ok, 1} = BatchRollback.rollback(ctx.batch, "t") + assert length(manifests_of(ctx)) == 1 + end + + test "an unreadable batch log is an error, not proof that the batch is not rolled back", + ctx do + seed(ctx, "a", [row(ctx.batch, "f1.txt")]) + assert {:ok, 1} = BatchRollback.rollback(ctx.batch, "t") + rows = length(batch_rows(ctx)) + + batches = Path.join(ctx.dispatch, "batches.jsonl") + moved = batches <> ".moved-#{ctx.uniq}" + File.rename!(batches, moved) + File.mkdir_p!(batches) + + try do + assert {:error, {:batch_log_read_failed, _}} = BatchRollback.rollback(ctx.batch, "again") + after + File.rm_rf!(batches) + File.rename!(moved, batches) + end + + assert length(batch_rows(ctx)) == rows + assert {:ok, 0} = BatchRollback.rollback(ctx.batch, "again") + end + + test "overlapping rollbacks of one batch write one set of rows and one manifest", ctx do + seed(ctx, "a", [row(ctx.batch, "f1.txt"), row(ctx.batch, "f2.txt")]) + + results = + 1..8 + |> Enum.map(fn _ -> Task.async(fn -> BatchRollback.rollback(ctx.batch, "t") end) end) + |> Task.await_many(30_000) + + assert Enum.sort(results) == List.duplicate({:ok, 0}, 7) ++ [{:ok, 2}] + assert length(rollback_rows(ctx)) == 2 + assert length(manifests_of(ctx)) == 1 + end end From 6fef121f909758878172f621eb30cf9c15818ec4 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 03:06:42 +0000 Subject: [PATCH 3/3] fix(safety): batch rollback takes an exclusive lock file across VMs and processes (E0-25) Owner ruling 18 Nov on #1174: the :global.trans lock holds only within one Erlang VM. rollback/2 now also takes /dispatch/locks/.lock by exclusive create, released in an after (also on raise) only if the token is still ours. A held lock returns {:error, {:locked, holder}} and writes nothing; a stale lock (older than 15 min) is reported, never stolen. A lock that cannot be taken, or a batch id that is not a plain name, fails closed. :global.trans stays as the outer lock so same-VM callers queue instead of failing. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01J4QvDj7XpKyxE4WbsgTsTh --- .gitignore | 3 + lib/safety/batch_rollback.ex | 150 ++++++++++++++++++++++++- test/batch_rollback_lock_test.exs | 177 ++++++++++++++++++++++++++++++ 3 files changed, 324 insertions(+), 6 deletions(-) create mode 100644 test/batch_rollback_lock_test.exs diff --git a/.gitignore b/.gitignore index 27f49774..f2324c96 100644 --- a/.gitignore +++ b/.gitignore @@ -163,3 +163,6 @@ dist/ /.yts # Local checkout of the governance-pinned baseline implementation. /.standards/ + +# BatchRollback cross-process lock files (transient) +data/verisim/dispatch/locks/ diff --git a/lib/safety/batch_rollback.ex b/lib/safety/batch_rollback.ex index 597333a8..ade757ed 100644 --- a/lib/safety/batch_rollback.ex +++ b/lib/safety/batch_rollback.ex @@ -54,6 +54,13 @@ defmodule Hypatia.Safety.BatchRollback do from its last position to its first. It is not sorted by `timestamp`, `repo` or `file`: none of them is an order, and a tie would revert dependent changes the wrong way round. + Only one rollback of a batch runs at a time, across VMs and OS processes on the same + store: it takes the exclusive lock file `/dispatch/locks/.lock`. When + another holder has it, the call returns `{:error, {:locked, holder}}` and writes + nothing. `holder` has the holder's `os_pid`, `node`, `acquired_at`, `age_s`, `lock_path` + and `stale` (older than 15 minutes). A stale lock is reported, never taken over. + A batch id that is not a plain file name returns `{:error, {:invalid_batch_id, id}}`. + A rollback is safe to repeat (E0-25): * a batch whose status is `rolled_back` returns `{:ok, 0}` and writes nothing; @@ -66,12 +73,143 @@ defmodule Hypatia.Safety.BatchRollback do manifest still lists every original outcome, once. """ def rollback(batch_id, reason) do - # One rollback of a batch at a time, in this VM: the status check, the outcome - # snapshot, the writes and the status write are one critical section, so that two - # callers cannot both read `pending` and both append rows and a manifest. The - # status is read again inside the lock. (A lock between VMs or OS processes is not - # taken: the module has one writer node.) - :global.trans({{__MODULE__, batch_id}, self()}, fn -> locked_rollback(batch_id, reason) end) + # Two locks, for two scopes. The outer `:global.trans` queues the callers in this VM + # (and in connected nodes), so that overlapping callers wait their turn and each sees + # the status the one before it wrote. The inner lock file is the one that holds between + # separate VMs and OS processes on the same store: an exclusive create (O_EXCL) that + # the filesystem makes atomic. Without the outer lock, same-VM callers would fail + # with `{:error, {:locked, _}}` instead of waiting. The status is read again inside both. + # The lock is not held across hosts: a store that two hosts write through separate + # checkouts is out of scope (git is what merges those). + with :ok <- check_batch_id(batch_id) do + :global.trans({{__MODULE__, batch_id}, self()}, fn -> + with_lock_file(batch_id, fn -> locked_rollback(batch_id, reason) end) + end) + end + end + + # The batch id is a file name here, so it must be a plain one. + defp check_batch_id(batch_id) do + if is_binary(batch_id) and batch_id =~ ~r/\A[A-Za-z0-9][A-Za-z0-9_.-]{0,199}\z/ and + not String.contains?(batch_id, "..") do + :ok + else + {:error, {:invalid_batch_id, batch_id}} + end + end + + # A lock older than this is reported as `"stale" => true`. It is still never taken over: + # a rollback can be slow, and the pid of a holder on another VM cannot be checked, so the + # age is evidence for the operator, not a licence to steal. To clear a stale lock, read + # the holder in the error, confirm that the process is gone, and remove the file at + # `"lock_path"`. + @stale_after_s 15 * 60 + + defp lock_path(batch_id), + do: Path.join([Path.expand(@verisimdb_data_path), "dispatch", "locks", "#{batch_id}.lock"]) + + # Run `fun` while this process holds the lock file of the batch. Fails closed: when the + # lock cannot be taken, `fun` does not run and nothing is written. The lock is removed + # in `after` (also when `fun` raises), and only if it still carries our token. + defp with_lock_file(batch_id, fun) do + path = lock_path(batch_id) + + case acquire_lock(path) do + {:ok, token} -> + try do + fun.() + after + release_lock(path, token) + end + + {:error, _} = error -> + fail(batch_id, error) + end + end + + defp acquire_lock(path) do + token = Base.encode16(:crypto.strong_rand_bytes(8), case: :lower) + + holder = %{ + "token" => token, + "os_pid" => System.pid(), + "node" => Atom.to_string(node()), + "acquired_unix" => System.os_time(:second), + "acquired_at" => DateTime.utc_now() |> DateTime.to_iso8601() + } + + with :ok <- mkdir_for_lock(path) do + case File.open(path, [:write, :exclusive]) do + {:ok, io} -> write_holder(io, path, holder, token) + {:error, :eexist} -> {:error, {:locked, read_holder(path)}} + {:error, why} -> {:error, {:lock_failed, why}} + end + end + end + + defp mkdir_for_lock(path) do + case File.mkdir_p(Path.dirname(path)) do + :ok -> :ok + {:error, why} -> {:error, {:lock_failed, why}} + end + end + + # A lock that was created but could not be written is removed again: an empty lock + # would block every later rollback of the batch. + defp write_holder(io, path, holder, token) do + result = + try do + IO.binwrite(io, Jason.encode!(holder)) + rescue + e -> {:error, e} + end + + File.close(io) + + case result do + :ok -> + {:ok, token} + + other -> + File.rm(path) + {:error, {:lock_failed, other}} + end + end + + defp read_holder(path) do + info = + case File.read(path) do + {:ok, content} -> + case Jason.decode(content) do + {:ok, %{} = holder} -> holder + _ -> %{"unreadable" => true} + end + + {:error, why} -> + %{"unreadable" => true, "read_error" => inspect(why)} + end + + age = System.os_time(:second) - (info["acquired_unix"] || 0) + + info + |> Map.put("lock_path", path) + |> Map.put("age_s", age) + |> Map.put("stale", age > @stale_after_s) + end + + defp release_lock(path, token) do + case File.read(path) do + {:ok, content} -> + case Jason.decode(content) do + {:ok, %{"token" => ^token}} -> File.rm(path) + _ -> Logger.error("Lock #{path} is no longer ours; left in place") + end + + _ -> + Logger.error("Lock #{path} was gone at release") + end + + :ok end defp locked_rollback(batch_id, reason) do diff --git a/test/batch_rollback_lock_test.exs b/test/batch_rollback_lock_test.exs new file mode 100644 index 00000000..3151572b --- /dev/null +++ b/test/batch_rollback_lock_test.exs @@ -0,0 +1,177 @@ +# SPDX-License-Identifier: MPL-2.0 +# Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# E0-25 (owner ruling 18 Nov): BatchRollback.rollback/2 takes an exclusive lock file +# per batch, so that two VMs or two OS processes on one store cannot both run the +# rollback of a batch. The lock lives at /dispatch/locks/.lock. +defmodule Hypatia.Safety.BatchRollbackLockTest do + use ExUnit.Case, async: false + + alias Hypatia.Safety.BatchRollback + + setup do + data = Application.fetch_env!(:hypatia, :verisimdb_data_path) |> Path.expand() + outcomes = Path.join(data, "outcomes") + dispatch = Path.join(data, "dispatch") + File.mkdir_p!(outcomes) + File.mkdir_p!(dispatch) + + uniq = System.unique_integer([:positive]) + {:ok, batch} = BatchRollback.create_batch(1, :auto_execute) + log = Path.join(outcomes, "e0-25-lock-#{uniq}.jsonl") + lock = Path.join([dispatch, "locks", "#{batch}.lock"]) + + seed = %{ + "recipe_id" => "recipe-e0-25-lock", + "repo" => "owner/repo", + "file" => "a.txt", + "outcome" => "success", + "verification" => "verified", + "batch_id" => batch, + "timestamp" => DateTime.utc_now() |> DateTime.to_iso8601(), + "bot" => "hypatia" + } + + File.write!(log, Jason.encode!(seed) <> "\n") + + on_exit(fn -> + File.rm(log) + File.rm(lock) + end) + + manifests = fn -> + case File.read(Path.join(dispatch, "rollbacks.jsonl")) do + {:ok, c} -> + c + |> String.split("\n", trim: true) + |> Enum.map(&Jason.decode!/1) + |> Enum.filter(&(&1["batch_id"] == batch)) + + _ -> + [] + end + end + + rollback_rows = fn -> + outcomes + |> Path.join("*.jsonl") + |> Path.wildcard() + |> Enum.flat_map(&(File.read!(&1) |> String.split("\n", trim: true))) + |> Enum.map(&Jason.decode!/1) + |> Enum.filter(&(&1["batch_id"] == batch and &1["caused_by"] == "batch_rollback")) + end + + %{ + batch: batch, + lock: lock, + dispatch: dispatch, + uniq: uniq, + manifests: manifests, + rollback_rows: rollback_rows + } + end + + # An independent OS process (its own VM, not a Task) that takes the lock file the same + # way (exclusive create), holds it until `release` exists, then removes it. + defp start_foreign_holder(lock, release, now_offset_s \\ 0) do + File.mkdir_p!(Path.dirname(lock)) + + script = """ + [lock, release, off] = System.argv() + {:ok, f} = File.open(lock, [:write, :exclusive]) + at = System.os_time(:second) - String.to_integer(off) + IO.binwrite(f, ~s({"os_pid":"foreign","node":"foreign","token":"foreign-token","acquired_unix":\#{at}})) + File.close(f) + wait = fn wait -> if File.exists?(release), do: :ok, else: (Process.sleep(50); wait.(wait)) end + wait.(wait) + File.rm(lock) + """ + + exe = System.find_executable("elixir") + + port = + Port.open({:spawn_executable, exe}, [ + :binary, + :exit_status, + args: ["-e", script, lock, release, Integer.to_string(now_offset_s)] + ]) + + wait_until(fn -> File.exists?(lock) and File.read!(lock) != "" end, 60_000) + port + end + + defp wait_until(fun, budget) do + cond do + fun.() -> :ok + budget <= 0 -> flunk("condition not met in time") + true -> Process.sleep(50) && wait_until(fun, budget - 50) + end + end + + test "a lock held by another OS process stops the rollback; nothing is written, then it proceeds", + ctx do + release = Path.join(ctx.dispatch, "release-#{ctx.uniq}") + on_exit(fn -> File.write(release, "") end) + port = start_foreign_holder(ctx.lock, release) + + assert {:error, {:locked, holder}} = BatchRollback.rollback(ctx.batch, "t") + assert holder["os_pid"] == "foreign" + assert ctx.rollback_rows.() == [] + assert ctx.manifests.() == [] + assert {:ok, [%{"status" => status} | _]} = {:ok, BatchRollback.list_batches()} + refute status == "rolled_back" + + File.write!(release, "") + assert_receive {^port, {:exit_status, 0}}, 30_000 + refute File.exists?(ctx.lock) + + assert {:ok, 1} = BatchRollback.rollback(ctx.batch, "t") + assert length(ctx.rollback_rows.()) == 1 + assert length(ctx.manifests.()) == 1 + refute File.exists?(ctx.lock) + end + + test "a stale lock (old, holder gone) is refused and reported, never stolen", ctx do + release = Path.join(ctx.dispatch, "release-#{ctx.uniq}") + on_exit(fn -> File.write(release, "") end) + port = start_foreign_holder(ctx.lock, release, 7 * 24 * 3600) + before = File.read!(ctx.lock) + + assert {:error, {:locked, holder}} = BatchRollback.rollback(ctx.batch, "t") + assert holder["stale"] == true + assert holder["age_s"] >= 7 * 24 * 3600 + assert holder["lock_path"] == ctx.lock + assert File.read!(ctx.lock) == before + assert ctx.rollback_rows.() == [] + assert ctx.manifests.() == [] + + File.write!(release, "") + assert_receive {^port, {:exit_status, 0}}, 30_000 + end + + test "a holder that raises releases the lock", ctx do + # An invalid UTF-8 reason makes the manifest encode raise inside the critical section. + assert_raise Jason.EncodeError, fn -> BatchRollback.rollback(ctx.batch, <<255, 254>>) end + refute File.exists?(ctx.lock) + assert {:ok, 1} = BatchRollback.rollback(ctx.batch, "after the crash") + refute File.exists?(ctx.lock) + end + + test "a lock that cannot be taken fails closed: error, no writes", ctx do + # A regular file where the lock directory should be. + locks = Path.dirname(ctx.lock) + File.rm_rf!(locks) + File.write!(locks, "not a directory") + on_exit(fn -> File.rm(locks) end) + + assert {:error, {:lock_failed, _}} = BatchRollback.rollback(ctx.batch, "t") + assert ctx.rollback_rows.() == [] + assert ctx.manifests.() == [] + end + + test "a batch id that is not a plain name is refused", ctx do + assert {:error, {:invalid_batch_id, _}} = BatchRollback.rollback("../escape", "t") + assert {:error, {:invalid_batch_id, _}} = BatchRollback.rollback("", "t") + assert ctx.rollback_rows.() == [] + end +end