diff --git a/.gitignore b/.gitignore index 27f49774..f2324c96 100644 --- a/.gitignore +++ b/.gitignore @@ -163,3 +163,6 @@ dist/ /.yts # Local checkout of the governance-pinned baseline implementation. /.standards/ + +# BatchRollback cross-process lock files (transient) +data/verisim/dispatch/locks/ diff --git a/lib/safety/batch_rollback.ex b/lib/safety/batch_rollback.ex index dd8f730a..ade757ed 100644 --- a/lib/safety/batch_rollback.ex +++ b/lib/safety/batch_rollback.ex @@ -45,36 +45,271 @@ defmodule Hypatia.Safety.BatchRollback do update_batch_status(batch_id, "completed") end - @doc "Roll back an entire batch" + @doc """ + Roll back an entire batch. + + The manifest lists the revert instructions from the NEWEST outcome to the oldest, so + that a change that depends on an earlier one is reverted first. The order is the order + of the outcome log (files by name, one for each month, then the lines of each file), + from its last position to its first. It is not sorted by `timestamp`, `repo` or `file`: + none of them is an order, and a tie would revert dependent changes the wrong way round. + + Only one rollback of a batch runs at a time, across VMs and OS processes on the same + store: it takes the exclusive lock file `/dispatch/locks/.lock`. When + another holder has it, the call returns `{:error, {:locked, holder}}` and writes + nothing. `holder` has the holder's `os_pid`, `node`, `acquired_at`, `age_s`, `lock_path` + and `stale` (older than 15 minutes). A stale lock is reported, never taken over. + A batch id that is not a plain file name returns `{:error, {:invalid_batch_id, id}}`. + + A rollback is safe to repeat (E0-25): + + * a batch whose status is `rolled_back` returns `{:ok, 0}` and writes nothing; + * the status is set only after every write succeeded. A failed write returns + `{:error, reason}` and leaves the status as it was, so that a later call can + finish the rollback; + * a retry writes no second outcome row for an outcome that already has its rollback + row. For each key (`recipe_id`, `repo`, `file`), with N original outcomes and M + rollback rows already in the log, it writes N - M rows (none when M >= N). The + manifest still lists every original outcome, once. + """ def rollback(batch_id, reason) do - Logger.warning("ROLLBACK batch #{batch_id}: #{reason}") + # Two locks, for two scopes. The outer `:global.trans` queues the callers in this VM + # (and in connected nodes), so that overlapping callers wait their turn and each sees + # the status the one before it wrote. The inner lock file is the one that holds between + # separate VMs and OS processes on the same store: an exclusive create (O_EXCL) that + # the filesystem makes atomic. Without the outer lock, same-VM callers would fail + # with `{:error, {:locked, _}}` instead of waiting. The status is read again inside both. + # The lock is not held across hosts: a store that two hosts write through separate + # checkouts is out of scope (git is what merges those). + with :ok <- check_batch_id(batch_id) do + :global.trans({{__MODULE__, batch_id}, self()}, fn -> + with_lock_file(batch_id, fn -> locked_rollback(batch_id, reason) end) + end) + end + end + + # The batch id is a file name here, so it must be a plain one. + defp check_batch_id(batch_id) do + if is_binary(batch_id) and batch_id =~ ~r/\A[A-Za-z0-9][A-Za-z0-9_.-]{0,199}\z/ and + not String.contains?(batch_id, "..") do + :ok + else + {:error, {:invalid_batch_id, batch_id}} + end + end + + # A lock older than this is reported as `"stale" => true`. It is still never taken over: + # a rollback can be slow, and the pid of a holder on another VM cannot be checked, so the + # age is evidence for the operator, not a licence to steal. To clear a stale lock, read + # the holder in the error, confirm that the process is gone, and remove the file at + # `"lock_path"`. + @stale_after_s 15 * 60 + + defp lock_path(batch_id), + do: Path.join([Path.expand(@verisimdb_data_path), "dispatch", "locks", "#{batch_id}.lock"]) + + # Run `fun` while this process holds the lock file of the batch. Fails closed: when the + # lock cannot be taken, `fun` does not run and nothing is written. The lock is removed + # in `after` (also when `fun` raises), and only if it still carries our token. + defp with_lock_file(batch_id, fun) do + path = lock_path(batch_id) + + case acquire_lock(path) do + {:ok, token} -> + try do + fun.() + after + release_lock(path, token) + end + + {:error, _} = error -> + fail(batch_id, error) + end + end - # 1. Find all outcomes for this batch - outcomes = find_batch_outcomes(batch_id) + defp acquire_lock(path) do + token = Base.encode16(:crypto.strong_rand_bytes(8), case: :lower) - # 2. Mark all as false_positive - Enum.each(outcomes, fn outcome -> - recipe_id = Map.get(outcome, "recipe_id") + holder = %{ + "token" => token, + "os_pid" => System.pid(), + "node" => Atom.to_string(node()), + "acquired_unix" => System.os_time(:second), + "acquired_at" => DateTime.utc_now() |> DateTime.to_iso8601() + } + + with :ok <- mkdir_for_lock(path) do + case File.open(path, [:write, :exclusive]) do + {:ok, io} -> write_holder(io, path, holder, token) + {:error, :eexist} -> {:error, {:locked, read_holder(path)}} + {:error, why} -> {:error, {:lock_failed, why}} + end + end + end - if recipe_id do - Hypatia.OutcomeTracker.record_batch_rollback( - recipe_id, - Map.get(outcome, "repo", "unknown"), - Map.get(outcome, "file", ""), - batch_id - ) + defp mkdir_for_lock(path) do + case File.mkdir_p(Path.dirname(path)) do + :ok -> :ok + {:error, why} -> {:error, {:lock_failed, why}} + end + end + + # A lock that was created but could not be written is removed again: an empty lock + # would block every later rollback of the batch. + defp write_holder(io, path, holder, token) do + result = + try do + IO.binwrite(io, Jason.encode!(holder)) + rescue + e -> {:error, e} + end + + File.close(io) + + case result do + :ok -> + {:ok, token} + + other -> + File.rm(path) + {:error, {:lock_failed, other}} + end + end + + defp read_holder(path) do + info = + case File.read(path) do + {:ok, content} -> + case Jason.decode(content) do + {:ok, %{} = holder} -> holder + _ -> %{"unreadable" => true} + end + + {:error, why} -> + %{"unreadable" => true, "read_error" => inspect(why)} + end + + age = System.os_time(:second) - (info["acquired_unix"] || 0) + + info + |> Map.put("lock_path", path) + |> Map.put("age_s", age) + |> Map.put("stale", age > @stale_after_s) + end + + defp release_lock(path, token) do + case File.read(path) do + {:ok, content} -> + case Jason.decode(content) do + {:ok, %{"token" => ^token}} -> File.rm(path) + _ -> Logger.error("Lock #{path} is no longer ours; left in place") + end + + _ -> + Logger.error("Lock #{path} was gone at release") + end + + :ok + end + + defp locked_rollback(batch_id, reason) do + case batch_status(batch_id) do + {:ok, "rolled_back"} -> {:ok, 0} + {:ok, _status} -> do_rollback(batch_id, reason) + {:error, _} = error -> fail(batch_id, error) + end + end + + defp fail(batch_id, {:error, why} = error) do + Logger.error("Rollback of batch #{batch_id} did not complete: #{inspect(why)}") + error + end + + defp do_rollback(batch_id, reason) do + Logger.warning("ROLLBACK batch #{batch_id}: #{reason}") + + # 1. The rows of this batch, in the order of the log. A row that a rollback wrote + # is not an outcome of the batch: it is what a rollback did to one. + with {:ok, outcomes} <- find_batch_outcomes(batch_id), + {originals, written} = + Enum.split_with(outcomes, &(Map.get(&1, "caused_by") != "batch_rollback")), + :ok <- mark_false_positives(originals, written, batch_id), + :ok <- write_manifest(batch_id, reason, originals), + :ok <- set_status(batch_id, "rolled_back") do + Logger.warning( + "Rollback complete: #{length(originals)} outcomes reverted for batch #{batch_id}" + ) + + {:ok, length(originals)} + else + {:error, _} = error -> fail(batch_id, error) + end + end + + # Write one rollback row for each original outcome that has none yet. Outcomes have + # no stable id, and 2 of them can share a batch, recipe, repo and file, so they are + # counted by key: the first M originals of a key are the ones that already have a row. + defp mark_false_positives(originals, written, batch_id) do + already = Enum.frequencies_by(written, &outcome_key/1) + + originals + |> Enum.reduce_while({:ok, already}, fn outcome, {:ok, skip} -> + key = outcome_key(outcome) + + cond do + is_nil(Map.get(outcome, "recipe_id")) -> + {:cont, {:ok, skip}} + + Map.get(skip, key, 0) > 0 -> + {:cont, {:ok, Map.update!(skip, key, &(&1 - 1))}} + + true -> + case Hypatia.OutcomeTracker.record_batch_rollback( + Map.get(outcome, "recipe_id"), + Map.get(outcome, "repo", "unknown"), + Map.get(outcome, "file", ""), + batch_id + ) do + {:ok, _record} -> {:cont, {:ok, skip}} + {:error, why} -> {:halt, {:error, {:outcome_write_failed, why}}} + other -> {:halt, {:error, {:outcome_write_failed, other}}} + end end end) + |> case do + {:ok, _skip} -> :ok + error -> error + end + end + + defp outcome_key(outcome) do + {Map.get(outcome, "recipe_id"), Map.get(outcome, "repo", "unknown"), + Map.get(outcome, "file", "")} + end + + # The manifest, newest outcome first. One is written for a batch, ever: a retry after + # a failed status write must not add a second list of instructions. + defp write_manifest(batch_id, reason, originals) do + rollback_path = Path.join([Path.expand(@verisimdb_data_path), "dispatch", "rollbacks.jsonl"]) + + case manifest_written?(rollback_path, batch_id) do + {:ok, true} -> :ok + {:ok, false} -> append_manifest(rollback_path, batch_id, reason, originals) + {:error, _} = error -> error + end + end - # 3. Generate rollback manifest - rollback_manifest = %{ + defp append_manifest(rollback_path, batch_id, reason, originals) do + manifest = %{ "batch_id" => batch_id, "reason" => reason, "timestamp" => DateTime.utc_now() |> DateTime.to_iso8601(), - "affected_outcomes" => length(outcomes), + "affected_outcomes" => length(originals), "action" => "revert", "instructions" => - Enum.map(outcomes, fn o -> + originals + |> Enum.reverse() + |> Enum.map(fn o -> %{ "repo" => Map.get(o, "repo"), "file" => Map.get(o, "file"), @@ -83,17 +318,65 @@ defmodule Hypatia.Safety.BatchRollback do end) } - rollback_path = Path.join([Path.expand(@verisimdb_data_path), "dispatch", "rollbacks.jsonl"]) - File.write(rollback_path, Jason.encode!(rollback_manifest) <> "\n", [:append, :utf8]) + case File.write(rollback_path, Jason.encode!(manifest) <> "\n", [:append, :utf8]) do + :ok -> :ok + {:error, why} -> {:error, {:manifest_write_failed, why}} + end + end - # 4. Update batch status - update_batch_status(batch_id, "rolled_back") + defp manifest_written?(path, batch_id) do + case File.read(path) do + {:ok, content} -> + {:ok, + content + |> String.split("\n", trim: true) + |> Enum.any?(fn line -> + match?({:ok, %{"batch_id" => ^batch_id}}, Jason.decode(line)) + end)} + + # Only a missing log means that no manifest was written. Any other failure may hide + # one, and appending a second would break the idempotency of a retry. + {:error, :enoent} -> + {:ok, false} + + {:error, why} -> + {:error, {:manifest_read_failed, why}} + end + end - Logger.warning( - "Rollback complete: #{length(outcomes)} outcomes reverted for batch #{batch_id}" - ) + defp set_status(batch_id, status) do + case update_batch_status(batch_id, status) do + :ok -> :ok + {:error, why} -> {:error, {:status_write_failed, why}} + end + end - {:ok, length(outcomes)} + # The latest status that the batch log gives for this batch, or nil. + # {:error, _} when the log cannot be read: that is not "no status", and a rolled_back + # row may be in it. A missing log is no status. + defp batch_status(batch_id) do + batch_path = Path.join([Path.expand(@verisimdb_data_path), "dispatch", "batches.jsonl"]) + + case File.read(batch_path) do + {:ok, content} -> + status = + content + |> String.split("\n", trim: true) + |> Enum.map(&Jason.decode/1) + |> Enum.flat_map(fn + {:ok, %{"batch_id" => ^batch_id, "status" => status}} -> [status] + _ -> [] + end) + |> List.last() + + {:ok, status} + + {:error, :enoent} -> + {:ok, nil} + + {:error, why} -> + {:error, {:batch_log_read_failed, why}} + end end @doc "List recent batches with their status" @@ -103,6 +386,9 @@ defmodule Hypatia.Safety.BatchRollback do # --- Internal --- + # {:ok, rows} in log order, or {:error, _} if the log directory or any monthly file + # cannot be read: a partial list would give an incomplete manifest, and the status + # that follows would make that final. A missing directory is an empty log. defp find_batch_outcomes(batch_id) do outcomes_dir = Path.join(Path.expand(@verisimdb_data_path), "outcomes") @@ -110,27 +396,39 @@ defmodule Hypatia.Safety.BatchRollback do {:ok, files} -> files |> Enum.filter(&String.ends_with?(&1, ".jsonl")) - |> Enum.flat_map(fn f -> + # File.ls/1 has no fixed order. One file for each month, so a name sort is a + # time sort; the lines of a file keep their order. + |> Enum.sort() + |> Enum.reduce_while({:ok, []}, fn f, {:ok, acc} -> path = Path.join(outcomes_dir, f) case File.read(path) do {:ok, content} -> - content - |> String.split("\n", trim: true) - |> Enum.map(&Jason.decode/1) - |> Enum.filter(fn - {:ok, o} -> Map.get(o, "batch_id") == batch_id - _ -> false - end) - |> Enum.map(fn {:ok, o} -> o end) - - _ -> - [] + rows = + content + |> String.split("\n", trim: true) + |> Enum.map(&Jason.decode/1) + |> Enum.flat_map(fn + {:ok, %{"batch_id" => ^batch_id} = o} -> [o] + _ -> [] + end) + + {:cont, {:ok, [rows | acc]}} + + {:error, why} -> + {:halt, {:error, {:outcome_read_failed, f, why}}} end end) + |> case do + {:ok, chunks} -> {:ok, chunks |> Enum.reverse() |> Enum.concat()} + error -> error + end - _ -> - [] + {:error, :enoent} -> + {:ok, []} + + {:error, why} -> + {:error, {:outcome_read_failed, outcomes_dir, why}} end end @@ -147,6 +445,9 @@ defmodule Hypatia.Safety.BatchRollback do File.write(batch_path, update <> "\n", [:append, :utf8]) end + defp take_limit(batches, :infinity), do: batches + defp take_limit(batches, limit), do: Enum.take(batches, limit) + defp list_batches_from_file(limit) do batch_path = Path.join([Path.expand(@verisimdb_data_path), "dispatch", "batches.jsonl"]) @@ -158,7 +459,7 @@ defmodule Hypatia.Safety.BatchRollback do |> Enum.filter(&match?({:ok, _}, &1)) |> Enum.map(fn {:ok, b} -> b end) |> Enum.reverse() - |> Enum.take(limit) + |> take_limit(limit) _ -> [] diff --git a/test/batch_rollback_lock_test.exs b/test/batch_rollback_lock_test.exs new file mode 100644 index 00000000..3151572b --- /dev/null +++ b/test/batch_rollback_lock_test.exs @@ -0,0 +1,177 @@ +# SPDX-License-Identifier: MPL-2.0 +# Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# E0-25 (owner ruling 18 Nov): BatchRollback.rollback/2 takes an exclusive lock file +# per batch, so that two VMs or two OS processes on one store cannot both run the +# rollback of a batch. The lock lives at /dispatch/locks/.lock. +defmodule Hypatia.Safety.BatchRollbackLockTest do + use ExUnit.Case, async: false + + alias Hypatia.Safety.BatchRollback + + setup do + data = Application.fetch_env!(:hypatia, :verisimdb_data_path) |> Path.expand() + outcomes = Path.join(data, "outcomes") + dispatch = Path.join(data, "dispatch") + File.mkdir_p!(outcomes) + File.mkdir_p!(dispatch) + + uniq = System.unique_integer([:positive]) + {:ok, batch} = BatchRollback.create_batch(1, :auto_execute) + log = Path.join(outcomes, "e0-25-lock-#{uniq}.jsonl") + lock = Path.join([dispatch, "locks", "#{batch}.lock"]) + + seed = %{ + "recipe_id" => "recipe-e0-25-lock", + "repo" => "owner/repo", + "file" => "a.txt", + "outcome" => "success", + "verification" => "verified", + "batch_id" => batch, + "timestamp" => DateTime.utc_now() |> DateTime.to_iso8601(), + "bot" => "hypatia" + } + + File.write!(log, Jason.encode!(seed) <> "\n") + + on_exit(fn -> + File.rm(log) + File.rm(lock) + end) + + manifests = fn -> + case File.read(Path.join(dispatch, "rollbacks.jsonl")) do + {:ok, c} -> + c + |> String.split("\n", trim: true) + |> Enum.map(&Jason.decode!/1) + |> Enum.filter(&(&1["batch_id"] == batch)) + + _ -> + [] + end + end + + rollback_rows = fn -> + outcomes + |> Path.join("*.jsonl") + |> Path.wildcard() + |> Enum.flat_map(&(File.read!(&1) |> String.split("\n", trim: true))) + |> Enum.map(&Jason.decode!/1) + |> Enum.filter(&(&1["batch_id"] == batch and &1["caused_by"] == "batch_rollback")) + end + + %{ + batch: batch, + lock: lock, + dispatch: dispatch, + uniq: uniq, + manifests: manifests, + rollback_rows: rollback_rows + } + end + + # An independent OS process (its own VM, not a Task) that takes the lock file the same + # way (exclusive create), holds it until `release` exists, then removes it. + defp start_foreign_holder(lock, release, now_offset_s \\ 0) do + File.mkdir_p!(Path.dirname(lock)) + + script = """ + [lock, release, off] = System.argv() + {:ok, f} = File.open(lock, [:write, :exclusive]) + at = System.os_time(:second) - String.to_integer(off) + IO.binwrite(f, ~s({"os_pid":"foreign","node":"foreign","token":"foreign-token","acquired_unix":\#{at}})) + File.close(f) + wait = fn wait -> if File.exists?(release), do: :ok, else: (Process.sleep(50); wait.(wait)) end + wait.(wait) + File.rm(lock) + """ + + exe = System.find_executable("elixir") + + port = + Port.open({:spawn_executable, exe}, [ + :binary, + :exit_status, + args: ["-e", script, lock, release, Integer.to_string(now_offset_s)] + ]) + + wait_until(fn -> File.exists?(lock) and File.read!(lock) != "" end, 60_000) + port + end + + defp wait_until(fun, budget) do + cond do + fun.() -> :ok + budget <= 0 -> flunk("condition not met in time") + true -> Process.sleep(50) && wait_until(fun, budget - 50) + end + end + + test "a lock held by another OS process stops the rollback; nothing is written, then it proceeds", + ctx do + release = Path.join(ctx.dispatch, "release-#{ctx.uniq}") + on_exit(fn -> File.write(release, "") end) + port = start_foreign_holder(ctx.lock, release) + + assert {:error, {:locked, holder}} = BatchRollback.rollback(ctx.batch, "t") + assert holder["os_pid"] == "foreign" + assert ctx.rollback_rows.() == [] + assert ctx.manifests.() == [] + assert {:ok, [%{"status" => status} | _]} = {:ok, BatchRollback.list_batches()} + refute status == "rolled_back" + + File.write!(release, "") + assert_receive {^port, {:exit_status, 0}}, 30_000 + refute File.exists?(ctx.lock) + + assert {:ok, 1} = BatchRollback.rollback(ctx.batch, "t") + assert length(ctx.rollback_rows.()) == 1 + assert length(ctx.manifests.()) == 1 + refute File.exists?(ctx.lock) + end + + test "a stale lock (old, holder gone) is refused and reported, never stolen", ctx do + release = Path.join(ctx.dispatch, "release-#{ctx.uniq}") + on_exit(fn -> File.write(release, "") end) + port = start_foreign_holder(ctx.lock, release, 7 * 24 * 3600) + before = File.read!(ctx.lock) + + assert {:error, {:locked, holder}} = BatchRollback.rollback(ctx.batch, "t") + assert holder["stale"] == true + assert holder["age_s"] >= 7 * 24 * 3600 + assert holder["lock_path"] == ctx.lock + assert File.read!(ctx.lock) == before + assert ctx.rollback_rows.() == [] + assert ctx.manifests.() == [] + + File.write!(release, "") + assert_receive {^port, {:exit_status, 0}}, 30_000 + end + + test "a holder that raises releases the lock", ctx do + # An invalid UTF-8 reason makes the manifest encode raise inside the critical section. + assert_raise Jason.EncodeError, fn -> BatchRollback.rollback(ctx.batch, <<255, 254>>) end + refute File.exists?(ctx.lock) + assert {:ok, 1} = BatchRollback.rollback(ctx.batch, "after the crash") + refute File.exists?(ctx.lock) + end + + test "a lock that cannot be taken fails closed: error, no writes", ctx do + # A regular file where the lock directory should be. + locks = Path.dirname(ctx.lock) + File.rm_rf!(locks) + File.write!(locks, "not a directory") + on_exit(fn -> File.rm(locks) end) + + assert {:error, {:lock_failed, _}} = BatchRollback.rollback(ctx.batch, "t") + assert ctx.rollback_rows.() == [] + assert ctx.manifests.() == [] + end + + test "a batch id that is not a plain name is refused", ctx do + assert {:error, {:invalid_batch_id, _}} = BatchRollback.rollback("../escape", "t") + assert {:error, {:invalid_batch_id, _}} = BatchRollback.rollback("", "t") + assert ctx.rollback_rows.() == [] + end +end diff --git a/test/safety_test.exs b/test/safety_test.exs index 34f54c49..c6313aa9 100644 --- a/test/safety_test.exs +++ b/test/safety_test.exs @@ -360,3 +360,331 @@ defmodule Hypatia.Safety.BatchRollbackTest do end end end + +# E0-25: the rollback manifest of a batch lists its revert instructions from the newest +# outcome to the oldest, and a second rollback of the same batch is a no-op. +defmodule Hypatia.Safety.BatchRollbackOrderTest do + use ExUnit.Case, async: false + + alias Hypatia.Safety.BatchRollback + + setup do + data = Application.fetch_env!(:hypatia, :verisimdb_data_path) |> Path.expand() + outcomes = Path.join(data, "outcomes") + dispatch = Path.join(data, "dispatch") + File.mkdir_p!(outcomes) + File.mkdir_p!(dispatch) + + uniq = System.unique_integer([:positive]) + {:ok, batch} = BatchRollback.create_batch(3, :auto_execute) + created = [Path.join(outcomes, "e0-25-#{uniq}-*.jsonl")] + + on_exit(fn -> + for pattern <- created, f <- Path.wildcard(pattern), do: File.rm(f) + File.rm_rf(Path.join(dispatch, "rollbacks.jsonl.moved-#{uniq}")) + end) + + %{ + batch: batch, + uniq: uniq, + outcomes: outcomes, + dispatch: dispatch, + manifests: Path.join(dispatch, "rollbacks.jsonl") + } + end + + defp row(batch, file, extra \\ %{}) do + Map.merge( + %{ + "recipe_id" => "recipe-e0-25-order", + "repo" => "owner/repo", + "file" => file, + "outcome" => "success", + "verification" => "verified", + "batch_id" => batch, + "timestamp" => "2026-11-15T10:00:00Z", + "bot" => "hypatia" + }, + extra + ) + end + + defp seed(ctx, name, rows) do + path = Path.join(ctx.outcomes, "e0-25-#{ctx.uniq}-#{name}.jsonl") + File.write!(path, Enum.map_join(rows, "", &(Jason.encode!(&1) <> "\n"))) + path + end + + defp lines(path) do + case File.read(path) do + {:ok, content} -> content |> String.split("\n", trim: true) |> Enum.map(&Jason.decode!/1) + _ -> [] + end + end + + defp batch_rows(ctx) do + ctx.outcomes + |> Path.join("*.jsonl") + |> Path.wildcard() + |> Enum.flat_map(&lines/1) + |> Enum.filter(&(&1["batch_id"] == ctx.batch)) + end + + defp rollback_rows(ctx), + do: Enum.filter(batch_rows(ctx), &(&1["caused_by"] == "batch_rollback")) + + defp manifests_of(ctx), do: Enum.filter(lines(ctx.manifests), &(&1["batch_id"] == ctx.batch)) + defp instruction_files(manifest), do: Enum.map(manifest["instructions"], & &1["file"]) + + defp status(ctx) do + BatchRollback.list_batches(10_000) + |> Enum.find(&(&1["batch_id"] == ctx.batch and Map.has_key?(&1, "status"))) + |> Map.fetch!("status") + end + + # Make rollbacks.jsonl unwritable (a directory cannot be appended to), and put it + # back afterwards. + defp break_manifest_log(ctx) do + moved = ctx.manifests <> ".moved-#{ctx.uniq}" + if File.exists?(ctx.manifests), do: File.rename!(ctx.manifests, moved) + File.mkdir_p!(ctx.manifests) + + fn -> + File.rm_rf!(ctx.manifests) + if File.exists?(moved), do: File.rename!(moved, ctx.manifests) + end + end + + test "3 outcomes written oldest first give a manifest that lists them newest first", ctx do + seed(ctx, "a", [row(ctx.batch, "f1.txt"), row(ctx.batch, "f2.txt"), row(ctx.batch, "f3.txt")]) + + assert {:ok, 3} = BatchRollback.rollback(ctx.batch, "t") + assert [manifest] = manifests_of(ctx) + assert instruction_files(manifest) == ["f3.txt", "f2.txt", "f1.txt"] + assert manifest["affected_outcomes"] == 3 + end + + test "2 outcomes with the same timestamp are listed in the reverse of the log, not by repo and file", + ctx do + # Written b, then a: against the order of file names. Same timestamp. + seed(ctx, "a", [row(ctx.batch, "b.txt"), row(ctx.batch, "a.txt")]) + + assert {:ok, 2} = BatchRollback.rollback(ctx.batch, "t") + assert [manifest] = manifests_of(ctx) + assert instruction_files(manifest) == ["a.txt", "b.txt"] + end + + test "outcomes in 2 monthly files: the later month is listed first, whatever File.ls gives", + ctx do + # Month names whose File.ls order is not their time order, if this file system + # has one. If it lists them in time order the case below cannot tell the + # difference, and it says so. + months = for m <- 1..12, do: "2099-" <> String.pad_leading("#{m}", 2, "0") + for m <- months, do: seed(ctx, m, []) + + listed = + File.ls!(ctx.outcomes) + |> Enum.filter(&String.starts_with?(&1, "e0-25-#{ctx.uniq}-")) + |> Enum.map( + &(&1 + |> String.replace_prefix("e0-25-#{ctx.uniq}-", "") + |> String.trim_trailing(".jsonl")) + ) + + pair = + for {a, i} <- Enum.with_index(listed), + {b, j} <- Enum.with_index(listed), + i < j, + a > b, + do: {b, a} + + {early, late} = List.first(pair) || {"2099-01", "2099-02"} + + if pair == [], + do: + IO.puts( + "note: File.ls lists the month files in time order here; the order is not exercised" + ) + + seed(ctx, early, [row(ctx.batch, "early.txt")]) + seed(ctx, late, [row(ctx.batch, "late.txt")]) + + assert {:ok, 2} = BatchRollback.rollback(ctx.batch, "t") + assert [manifest] = manifests_of(ctx) + assert instruction_files(manifest) == ["late.txt", "early.txt"] + end + + test "a failed write of rollbacks.jsonl returns an error and does not set rolled_back", ctx do + seed(ctx, "a", [row(ctx.batch, "f1.txt")]) + restore = break_manifest_log(ctx) + + try do + assert {:error, _reason} = BatchRollback.rollback(ctx.batch, "t") + refute status(ctx) == "rolled_back" + after + restore.() + end + end + + test "a retry after a failed manifest write completes the rollback with no second outcome row", + ctx do + seed(ctx, "a", [row(ctx.batch, "f1.txt"), row(ctx.batch, "f2.txt")]) + restore = break_manifest_log(ctx) + + try do + assert {:error, _} = BatchRollback.rollback(ctx.batch, "t") + after + restore.() + end + + written_by_the_first_try = length(rollback_rows(ctx)) + assert written_by_the_first_try == 2 + + assert {:ok, 2} = BatchRollback.rollback(ctx.batch, "t") + + assert length(rollback_rows(ctx)) == 2, "the retry wrote a second outcome row" + assert [manifest] = manifests_of(ctx) + assert instruction_files(manifest) == ["f2.txt", "f1.txt"] + assert status(ctx) == "rolled_back" + end + + test "2 originals with the same key, 1 rollback row already written: the retry writes exactly 1 more", + ctx do + seed(ctx, "a", [row(ctx.batch, "same.txt"), row(ctx.batch, "same.txt")]) + + # The first rollback row was written, and the second write failed. + assert {:ok, _} = + Hypatia.OutcomeTracker.record_batch_rollback( + "recipe-e0-25-order", + "owner/repo", + "same.txt", + ctx.batch + ) + + assert length(rollback_rows(ctx)) == 1 + + assert {:ok, 2} = BatchRollback.rollback(ctx.batch, "t") + assert length(rollback_rows(ctx)) == 2 + assert [manifest] = manifests_of(ctx) + assert instruction_files(manifest) == ["same.txt", "same.txt"] + end + + test "a rollback that left its rows and manifest but no status: the retry adds no second manifest", + ctx do + seed(ctx, "a", [row(ctx.batch, "f1.txt"), row(ctx.batch, "f2.txt")]) + + # The state that a rollback leaves when its status write fails (a status write cannot + # be made to fail here once an unreadable batch log is an error, and the tests may + # run as root): both rollback rows and the manifest are in the logs, the status is + # still `pending`. + for f <- ["f1.txt", "f2.txt"] do + {:ok, _} = + Hypatia.OutcomeTracker.record_batch_rollback( + "recipe-e0-25-order", + "owner/repo", + f, + ctx.batch + ) + end + + File.write!( + ctx.manifests, + Jason.encode!(%{"batch_id" => ctx.batch, "instructions" => []}) <> "\n", + [:append, :utf8] + ) + + refute status(ctx) == "rolled_back" + + assert {:ok, 2} = BatchRollback.rollback(ctx.batch, "t") + assert length(manifests_of(ctx)) == 1, "the retry wrote a second manifest" + assert length(rollback_rows(ctx)) == 2 + assert status(ctx) == "rolled_back" + end + + test "a second rollback of the same batch adds no outcome row and no manifest line", ctx do + seed(ctx, "a", [row(ctx.batch, "f1.txt"), row(ctx.batch, "f2.txt")]) + + assert {:ok, 2} = BatchRollback.rollback(ctx.batch, "t") + assert status(ctx) == "rolled_back" + rows_after_first = length(batch_rows(ctx)) + manifests_after_first = length(manifests_of(ctx)) + assert manifests_after_first == 1 + + assert {:ok, 0} = BatchRollback.rollback(ctx.batch, "again") + assert length(batch_rows(ctx)) == rows_after_first + assert length(manifests_of(ctx)) == manifests_after_first + end + + test "an unreadable monthly outcome file stops the rollback; the retry reverts every outcome", + ctx do + seed(ctx, "a", [row(ctx.batch, "f1.txt")]) + # A directory with a .jsonl name cannot be read as a file (EISDIR); it sorts last. + bad = Path.join(ctx.outcomes, "e0-25-#{ctx.uniq}-b.jsonl") + File.mkdir_p!(bad) + + try do + assert {:error, {:outcome_read_failed, _, _}} = BatchRollback.rollback(ctx.batch, "t") + assert manifests_of(ctx) == [] + assert rollback_rows(ctx) == [] + refute status(ctx) == "rolled_back" + after + File.rm_rf!(bad) + end + + seed(ctx, "b", [row(ctx.batch, "f2.txt")]) + assert {:ok, 2} = BatchRollback.rollback(ctx.batch, "t") + assert [manifest] = manifests_of(ctx) + assert instruction_files(manifest) == ["f2.txt", "f1.txt"] + assert status(ctx) == "rolled_back" + end + + test "an unreadable manifest log is an error, not proof that no manifest exists", ctx do + seed(ctx, "a", [row(ctx.batch, "f1.txt")]) + restore = break_manifest_log(ctx) + + try do + assert {:error, {:manifest_read_failed, _}} = BatchRollback.rollback(ctx.batch, "t") + refute status(ctx) == "rolled_back" + after + restore.() + end + + assert {:ok, 1} = BatchRollback.rollback(ctx.batch, "t") + assert length(manifests_of(ctx)) == 1 + end + + test "an unreadable batch log is an error, not proof that the batch is not rolled back", + ctx do + seed(ctx, "a", [row(ctx.batch, "f1.txt")]) + assert {:ok, 1} = BatchRollback.rollback(ctx.batch, "t") + rows = length(batch_rows(ctx)) + + batches = Path.join(ctx.dispatch, "batches.jsonl") + moved = batches <> ".moved-#{ctx.uniq}" + File.rename!(batches, moved) + File.mkdir_p!(batches) + + try do + assert {:error, {:batch_log_read_failed, _}} = BatchRollback.rollback(ctx.batch, "again") + after + File.rm_rf!(batches) + File.rename!(moved, batches) + end + + assert length(batch_rows(ctx)) == rows + assert {:ok, 0} = BatchRollback.rollback(ctx.batch, "again") + end + + test "overlapping rollbacks of one batch write one set of rows and one manifest", ctx do + seed(ctx, "a", [row(ctx.batch, "f1.txt"), row(ctx.batch, "f2.txt")]) + + results = + 1..8 + |> Enum.map(fn _ -> Task.async(fn -> BatchRollback.rollback(ctx.batch, "t") end) end) + |> Task.await_many(30_000) + + assert Enum.sort(results) == List.duplicate({:ok, 0}, 7) ++ [{:ok, 2}] + assert length(rollback_rows(ctx)) == 2 + assert length(manifests_of(ctx)) == 1 + end +end