diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 284952396..65ed43d2a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,6 +11,7 @@ on: branches: - main - dev + - gc-v2 env: CARGO_TERM_COLOR: always @@ -20,8 +21,105 @@ concurrency: cancel-in-progress: true jobs: + # Runs first and gates everything else (fmt/clippy/test all `needs: tla-plus` + # below) - it's the fastest job (seconds, no Rust toolchain to build) and a + # failure here means either a real regression the other, much slower jobs + # can't catch, or a stale/broken spec - either way not worth burning 30+ + # minutes of Cargo Test/Clippy compute on before finding out. + # + # All 8 findings from audit/TLAPlus-20260630.md were fixed in commit + # 991faaa, so this job is expected to be GREEN. Its steps still run every + # bug config that reproduced the original counterexamples, but a bug + # config correctly still failing is no longer treated as a job failure - + # see the second step's own comment for why (its constants are frozen + # historical snapshots, not a live read of the Rust source, so they can't + # detect a regression by staying red; only an unexpected PASS is a real + # drift signal now). + tla-plus: + name: TLA+ Formal Verification + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "21" + - name: Download TLA+ tools + run: | + mkdir -p ~/.local/share/tlaplus + curl -sL -o ~/.local/share/tlaplus/tla2tools.jar \ + https://github.com/tlaplus/tlaplus/releases/latest/download/tla2tools.jar + # This audit pass proves bugs exist in CURRENT code and proves correct + # fix designs for them - the fixes are NOT yet applied to the Rust code + # (see node/README.md's "Known gap" sections). These configs model the + # verified fix designs (or a baseline that was never buggy) and must + # always pass. See root README.md's "Formal verification (TLA+)" + # section for what each spec covers. + - name: Run baseline + proposed-fix specs (must pass) + working-directory: node/tla + run: | + set -e + JAR=~/.local/share/tlaplus/tla2tools.jar + java -jar "$JAR" -config GraphLifecycleCoreOnly.cfg GraphLifecycle.tla + java -jar "$JAR" -config GraphLifecycleFixed.cfg GraphLifecycle.tla + java -jar "$JAR" -config GraphLifecycleFineGrainedFixed.cfg GraphLifecycleFineGrainedFixed.tla + java -jar "$JAR" -config InstancePresignedFixed.cfg InstancePresigned.tla + java -jar "$JAR" -config Take2DisproveRace.cfg Take2DisproveRace.tla + java -jar "$JAR" -config MultiActorRace.cfg MultiActorRace.tla + java -jar "$JAR" -config InstanceBridgeOutRaceFixed.cfg InstanceBridgeOutRace.tla + java -jar "$JAR" -config MessageStateRaceFixed.cfg MessageStateRace.tla + java -jar "$JAR" -config Take1ChallengeRaceFixed.cfg Take1ChallengeRace.tla + # This job's earlier design (while all 8 findings from this round were + # still genuinely unfixed) made this step - and everything gated + # behind it - fail for as long as any bug config still reproduced its + # counterexample. As of commit 991faaa, every one of those findings + # has actually been fixed in the shipped Rust code (see + # audit/TLAPlus-20260630.md) - keeping the job permanently red past + # that point stopped being useful: these bug configs' constants are + # frozen historical snapshots (e.g. Take1ChallengeRace.tla's + # ConnectorA), not live readings of the current Rust source, so they + # can never detect a real code regression by themselves - they will + # keep reproducing the same counterexample forever regardless of + # what the Rust code does. Their only genuine ongoing signal is the + # OPPOSITE direction: if one of them ever unexpectedly STOPS + # reproducing its counterexample, that means the spec itself was + # edited into no longer demonstrating the bug it's supposed to - + # that's the one case this step still treats as a hard failure. + # Otherwise, a bug config correctly still failing is expected and + # does not fail the job - it's just printed as an informational + # reproduction pointer. + - name: Confirm known-bug specs still reproduce their counterexample + working-directory: node/tla + run: | + JAR=~/.local/share/tlaplus/tla2tools.jar + { + echo "## TLA+ audit: historical bug-reproduction specs" + echo + echo "These model the PRE-FIX code as a permanent historical record (all" + echo "findings below were fixed in commit 991faaa - see" + echo "\`audit/TLAPlus-20260630.md\`). Still correctly reproducing their" + echo "original counterexample below is expected and does not fail this job." + echo + } >> "$GITHUB_STEP_SUMMARY" + while IFS='|' read -r cfg tla finding; do + [ -z "$cfg" ] && continue + if java -jar "$JAR" -config "$cfg" "$tla" | grep -q "Model checking completed. No error has been found."; then + echo "::error::$tla / $cfg was expected to keep reproducing its historical counterexample but passed instead - the spec itself was likely edited into no longer demonstrating the bug it's supposed to. If the underlying Rust fix was somehow reverted, this is also how you'd find out - either way, investigate before trusting this spec again." + exit 1 + fi + repro="cd node/tla && java -jar ~/.local/share/tlaplus/tla2tools.jar -config $cfg $tla" + echo "- **$finding** - reproduce: \`$repro\`" >> "$GITHUB_STEP_SUMMARY" + done <<'BUGS' + GraphLifecycle.cfg|GraphLifecycle.tla|Finding 1: Graph.status race + GraphLifecycleFineGrained.cfg|GraphLifecycleFineGrained.tla|Finding 1b: naive guard still unsafe + InstancePresignedBug.cfg|InstancePresigned.tla|Finding 2: Instance.status regression past Presigned + InstanceBridgeOutRace.cfg|InstanceBridgeOutRace.tla|Finding 6: InstanceBridgeOutStatus resurrection + MessageStateRace.cfg|MessageStateRace.tla|Finding 7: MessageState resurrection + Take1ChallengeRace.cfg|Take1ChallengeRace.tla|Finding 9: connector_a has no margin check + BUGS fmt: name: Rustfmt + needs: tla-plus runs-on: ubuntu-latest steps: - uses: actions/checkout@v5 @@ -38,6 +136,7 @@ jobs: args: --all -- --check clippy: name: Clippy + needs: tla-plus runs-on: ubuntu-latest steps: - uses: actions/checkout@v5 @@ -56,6 +155,7 @@ jobs: cargo clippy --all-targets -- -D warnings test: name: Cargo Test + needs: tla-plus runs-on: ubuntu-latest strategy: matrix: @@ -72,5 +172,5 @@ jobs: - name: Run all unit tests run: | set -e - source ~/.zkm-toolchain/env + source ~/.zkm-toolchain/env cargo test -r --all --all-targets diff --git a/.gitignore b/.gitignore index 13c8f4ab7..af69b8b73 100644 --- a/.gitignore +++ b/.gitignore @@ -34,4 +34,5 @@ circuits/*/*/*.bin.in **/*.out **/*/output.data* -proof-builder-rpc/*.ckpt \ No newline at end of file +proof-builder-rpc/*.ckpt +node/tla/states/ \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index df4b113aa..fb7561fb9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13528,6 +13528,16 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-serde" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1" +dependencies = [ + "serde", + "tracing-core", +] + [[package]] name = "tracing-subscriber" version = "0.2.25" @@ -13547,12 +13557,15 @@ dependencies = [ "nu-ansi-term", "once_cell", "regex-automata", + "serde", + "serde_json", "sharded-slab", "smallvec", "thread_local", "tracing", "tracing-core", "tracing-log", + "tracing-serde", ] [[package]] diff --git a/README.md b/README.md index d2e442b64..1a696b50f 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,76 @@ GOAT Network's BitVM2 bridge implementation. See [GOAT BitVM2 Whitepaper](https: - `deployment`: Deployment scripts and documentation +## Formal verification (TLA+) + +`node/tla/` contains TLA+ specs that formally verify the graph/instance status +state machines and the peg-out timelock configuration against real races and +boundary conditions found in the Rust implementation. This started as an +**audit pass**: the specs proved several real bugs existed and proved a +correct fix design for each. **As of commit +[`991faaa`](https://github.com/GOATNetwork/bitvm2-node/commit/991faaabdb56c747103e8f1c6d6477c638ccfc4c), +all 8 of those findings have been fixed and verified in the shipped Rust +code** - see `audit/TLAPlus-20260630.md` for the full report, including what +each real applied fix looks like. + +**CI's `tla-plus` job is expected to be GREEN.** Each bug config (e.g. +`GraphLifecycle.cfg`) is kept as a **permanent historical record**, +deliberately still modeling the pre-fix code, and correctly reproducing its +original counterexample - but that expected failure is only printed as an +informational reproduction pointer in the job summary, it does not fail the +job. The only thing that *does* fail the job is a bug config **unexpectedly +passing**, since that would mean either the spec silently stopped +demonstrating the bug it's supposed to, or (more alarmingly) the fix's guard +got removed again. See that job's own comments in +`.github/workflows/ci.yml` for the full reasoning. + +Concretely: for each bug found, there is a **pair** of configs - one modeling +the pre-fix code (still models it as buggy on purpose - **expected to +fail**, a permanent historical record, not a live issue, and does not fail +CI) and one modeling the fix design (**expected to pass**, and does fail CI +if it doesn't - for every finding below, that design has since actually been +applied to the shipped Rust code, not just proven sound in the abstract). + +**Setup** (once): install a JRE (11+) and download the official TLA+ tools jar: + +```bash +sudo apt-get install -y openjdk-21-jre-headless # or any JRE 11+ +mkdir -p ~/.local/share/tlaplus +curl -sL -o ~/.local/share/tlaplus/tla2tools.jar \ + https://github.com/tlaplus/tlaplus/releases/latest/download/tla2tools.jar +``` + +**Run a spec**: + +```bash +cd node/tla +java -jar ~/.local/share/tlaplus/tla2tools.jar -config .cfg .tla +``` + +| Spec | Config | Models | Result | +|---|---|---|---| +| `GraphLifecycle.tla` | `GraphLifecycleCoreOnly.cfg` | code baseline | pass - chain-scan state machine alone is sound | +| `GraphLifecycle.tla` | `GraphLifecycle.cfg` | **pre-fix code (historical)** | **fails, by design**: unguarded race between the Bitcoin-chain-scan and GoatChain-event writers of `Graph.status` - fixed in `991faaa`, kept failing as a permanent regression check | +| `GraphLifecycle.tla` | `GraphLifecycleFixed.cfg` | fix design (**applied in `991faaa`**) | pass - atomic guard design closes the race | +| `GraphLifecycleFineGrained.tla` | `GraphLifecycleFineGrained.cfg` | pre-fix code (historical) | **fails, by design**: the read/write gap a naive (non-atomic) guard would still have - fixed in `991faaa` | +| `GraphLifecycleFineGrainedFixed.tla` | `GraphLifecycleFineGrainedFixed.cfg` | fix design (**applied in `991faaa`**) | pass - single-statement atomic CAS design closes the gap | +| `InstancePresigned.tla` | `InstancePresignedBug.cfg` | **pre-fix code (historical)** | **fails, by design**: `Instance.status` can regress past `Presigned` - fixed in `991faaa` | +| `InstancePresigned.tla` | `InstancePresignedFixed.cfg` | fix design (**applied in `991faaa`**) | pass - guard design closes the regression | +| `Take2DisproveRace.tla` | `Take2DisproveRace.cfg` | fix design (**applied in `991faaa`**, values updated to match) | pass - Take2 vs. Disprove UTXO race has strict margin on all networks with the real shipped `crates/bitvm-gc/src/timelocks.rs` values (Testnet4 `connector_d` now 40, shipped with more margin than originally proposed); the pre-fix shipped value (34) did **not** have this margin - that boundary case is how this spec found the bug in the first place | +| `MultiActorRace.tla` | `MultiActorRace.cfg` | fix design (**applied in `991faaa`**, values updated to match) | pass - the 1-of-N watchtower/verifier security property holds under the real shipped timelock values, checked against 2 independent actors per role rather than 1 | +| `InstanceBridgeOutRace.tla` | `InstanceBridgeOutRace.cfg` | **pre-fix code (historical)** | **fails, by design**: `InstanceBridgeOutStatus` can be resurrected to `Initialize` after reaching `Claim`/`Timeout`/`Refund` by a stale RPC upsert or maintenance-task write - fixed in `991faaa` | +| `InstanceBridgeOutRace.tla` | `InstanceBridgeOutRaceFixed.cfg` | fix design (**applied in `991faaa`**) | pass - atomic guard design (write only if not already terminal) closes the resurrection | +| `MessageStateRace.tla` | `MessageStateRace.cfg` | **pre-fix code (historical)** | **fails, by design**: `MessageState::Cancelled` could be resurrected to `Pending` by `upsert_message`'s unconditional `ON CONFLICT DO UPDATE` - fixed in `991faaa` | +| `MessageStateRace.tla` | `MessageStateRaceFixed.cfg` | fix design (**applied in `991faaa`**) | pass - guarding the resurrect-to-Pending write against terminal status closes the race | +| `Take1ChallengeRace.tla` | `Take1ChallengeRace.cfg` | **pre-fix code (historical)** | **fails, by design**: `connector_a` (Take1 vs. Challenge) had *no* margin check anywhere in `validate_timelock_config`; on Regtest the pre-fix shipped value gave a challenger exactly zero reaction margin - fixed in `991faaa` | +| `Take1ChallengeRace.tla` | `Take1ChallengeRaceFixed.cfg` | fix design (**applied in `991faaa`**) | pass - the missing margin check (mirroring Finding 4's floor) was added, closing the gap | +| `MultiActorRace.tla` | `MultiActorRace.cfg` | verification (no bug; still holds under real shipped values) | pass - also confirms `operator_commit`'s margin against the shared `ConnectorF` UTXO (the `OperatorCommitTimeoutTransaction` path, `ConnectorF` leaf 1's second spender) holds, closing a gap where only a scalar Rust check existed | + +Additional standalone tools available in the jar if needed: SANY (parser/type-checker) +via `java -cp tla2tools.jar tla2sany.SANY .tla`, and the PlusCal translator +(used to generate `GraphLifecycleFineGrained*.tla`'s TLA+ body from its PlusCal +algorithm block) via `java -cp tla2tools.jar pcal.trans .tla`. + ## Contributing Contributions are welcome! Please open an issue or submit a pull request for any improvements or bug fixes. \ No newline at end of file diff --git a/audit/TLAPlus-20260710.md b/audit/TLAPlus-20260710.md new file mode 100644 index 000000000..43ac01f13 --- /dev/null +++ b/audit/TLAPlus-20260710.md @@ -0,0 +1,224 @@ +# BitVM2 Node — Formal Verification Audit (Round 1) + +**Branch:** `audit/round-1` · **Base:** `gc-v2` · **Method:** TLA+ model checking (TLC) · **Status:** all 8 formally-proven findings from this round are now **fixed and verified** in commit [`991faaa`](https://github.com/GOATNetwork/bitvm2-node/commit/991faaabdb56c747103e8f1c6d6477c638ccfc4c) ("Dev fix #418", authored independently by a teammate — not applied by this audit). Two smaller, lower-priority adjacent defects noted under Finding 8 remain open. + +## Executive summary + +This audit used TLA+ to formally model the BitVM2 graph's status bookkeeping (local database state), the peg-out transaction graph's timelock configuration, and — a later round — the Bitcoin transaction graph's shared connectors directly, checking each against explicit safety and liveness properties rather than relying on manual code review alone. **Eight real issues were found**, each backed by a machine-checked counterexample (not a hypothetical), and a verified-correct fix design was produced for each. Three further checks were run and each, for a specific and verifiable reason, found **no** new issue: whether having multiple independent watchtowers/verifiers introduces new problems beyond the single-actor case (Finding 5), whether `GoatTxProcessingStatus` — the last multi-writer-shaped enum in the codebase — has the same class of race as the others (Finding 8), and whether `operator_commit`'s margin against the shared `ConnectorF` UTXO — previously only a scalar Rust assertion — actually holds with real shipped values (Finding 10). + +**Update: all 8 findings are now fixed.** Commit `991faaa`, contributed independently to this branch, applies real fixes for every one of them. This audit's own contribution to that fix landing was zero — the fixes were designed and written by someone else — but every fix was independently re-verified against this audit's own TLA+ models before being accepted as correct: read against the actual diff (not the commit message), and for the two timelock-margin findings (4, 9), re-checked with TLC using the *actual new shipped numbers*, not just the originally-proposed values. Each Finding below is marked **FIXED (commit `991faaa`)** with what the real applied fix looks like. In several cases the shipped fix is more thorough than what this audit's own fix design proposed (see Findings 1, 6 in particular). + +This round covers **every stateful enum (`pub enum *Status`/`*State`) in the codebase** (not just the two originally scoped, `GraphStatus` and `InstanceBridgeInStatus`) **and every shared/bottleneck connector in the Bitcoin transaction graph itself** (not just `ConnectorD`, the original scope) — see "Full-codebase coverage" below for the enum inventory and Finding 9/10 above for the connector sweep. + +All specs, their configs, and instructions to run them yourself live in `node/tla/`; see the root `README.md`'s "Formal verification (TLA+)" section for the full spec/config table and setup steps. This document is the narrative report: what was found, why it matters, and what fixing it required. + +**A note on `node/tla/`'s current state**: the bug-reproduction `.cfg`/`.tla` pairs (e.g. `GraphLifecycle.cfg`, `Take1ChallengeRace.cfg`) are deliberately left modeling the *pre-fix* code as a permanent historical record — CI (`.github/workflows/ci.yml`) still runs them every time and prints their reproduction command into the job summary, but a bug config correctly still failing (i.e. still reproducing its frozen historical counterexample) no longer fails the job itself. Only a bug config *unexpectedly passing* fails CI, since that would mean either the underlying guard was silently removed again, or the spec itself stopped demonstrating what it claims to. The repo's CI is therefore expected to show `tla-plus` as green now that the real Rust code is fixed — see that job's own comments for the full reasoning. + +--- + +## Methodology + +- **Tool**: [TLA+](https://lamport.azurewebsites.net/tla/tla.html) / TLC, the model checker — not a test suite with example inputs, but an exhaustive search over every reachable combination of the modeled actions, including orderings a human wouldn't think to test by hand. +- **Ground truth discipline**: every spec is built from the actual Rust source and, where relevant, the external `goat`/`bitvm` transaction-graph crate — not from documentation or assumption. Several places where this session's own hand-reasoning turned out to be wrong (see Findings 1b and 4) were caught specifically *because* the model was checked mechanically instead of trusted by argument. +- **Every finding follows the same evidence pattern**: (1) build the model from verified ground truth, (2) get a real TLC counterexample using actual code/config values — not an injected toy case, (3) design a fix, (4) prove the fix closes the gap by re-running TLC, (5) as an added check, re-inject the original bug into the *fixed* model and confirm TLC still catches it (i.e. the fix's own proof isn't a rubber stamp). +- **Scope boundary**: these specs model the *local node's* status bookkeeping and the *timelock arithmetic* of the transaction graph. They do not model Bitcoin consensus itself (a single-spend UTXO is trusted as a primitive, not re-derived) and, before Finding 5, did not model multiple independent watchtowers/verifiers as separate actors. + +--- + +## Findings + +### Finding 1 — `Graph.status` race between two uncoordinated writers · **FIXED (commit `991faaa`)** + +**Severity: data-integrity (not fund-custody).** Bitcoin's UTXO model guarantees the *actual* on-chain outcome (which of take1/take2/disprove really happened) is single and final regardless of this bug — but the *local node's own record* of that outcome can become wrong. + +`Graph.status` is written from two independently-scheduled places with no coordination between them: +- `scan_graph_chain_state` (`node/src/utils.rs:1328-1681`) — derives status by polling Bitcoin. +- The GoatChain L2-event watcher (`node/src/scheduled_tasks/event_watch_task.rs:392-502`) — writes `OperatorDataPushed`/`OperatorTake1`/`OperatorTake2`/`Disprove` directly. + +Both go through `StorageProcessor::update_graph` (`crates/store/src/localdb.rs:1290-1297`), a raw `UPDATE graph SET status = ?` with **no guard** on the row's current status. A stale or replayed event from either subsystem can silently overwrite a terminal status — e.g. reverting a recorded `Disprove` (evidence an operator cheated) back to an earlier status, or flipping between the two mutually-exclusive payout outcomes `OperatorTake1` and `OperatorTake2`. + +**Proof**: `node/tla/GraphLifecycle.tla`, config `GraphLifecycle.cfg`. Counterexample (3 states): `OperatorPresigned` → `OperatorTake1` (a `WithdrawHappyEvent`) → `OperatorDataPushed` (a stale, replayed `PostGraphDataEvent`). A separate run finds the sharper `OperatorTake1 → OperatorTake2` flip in 2 steps. + +**Verified fix design**: fold the guard into the `UPDATE` statement itself — `WHERE status IN (...)` — so the check and the write happen as one atomic database statement instead of a read-then-decide-then-write in application code. Proven in `GraphLifecycleFixed.cfg` (passes all safety and liveness properties). The design needs **two** guard clauses, not one — see Finding 1b for why. + +**What actually shipped**: the old `update_graph_status` is gone entirely, replaced by `StorageProcessor::transition_graph_status` (`crates/store/src/localdb.rs`), doc-commented with exactly this audit's central lesson: *"The conditional UPDATE is the authority check. The follow-up read only distinguishes an idempotent replay from a stale event; it never decides whether a write is permitted."* The guard is a new `GraphStatus::allowed_transition_from(self, source: GraphStatusSource) -> &'static [GraphStatus]` (`crates/store/src/schema.rs`) — a real predecessor table, keyed by *which* writer is asking (`GraphStatusSource::{Definition, GoatEvent, ChainReconcile}`), folded into the same `UPDATE` via `and_where_in("status", &allowed_from, false)`. Verified directly against the transition table: every terminal status is deliberately absent from every `allowed_from` set (the doc comment says so explicitly — "Closed states are deliberately absent from all sets"), correctly implementing absorbing terminals; `OperatorDataPushed`'s allowed predecessors under both `GoatEvent` and `ChainReconcile` correctly exclude `PreKickoff`/`OperatorKickOff` — the exact Guard 2 regression this finding's own fix design required. This is a materially more thorough design than what was proposed here (per-writer-source predecessor tables vs. one shared guard) and it checks out. All `event_watch_task.rs` call sites route through it via a shared `apply_gateway_graph_status` helper — zero remaining raw `update_graph`/`upsert_graph` calls in that file. + +### Finding 1b — a naive version of the Finding 1 fix is itself unsafe · **FIXED (commit `991faaa`, same fix as Finding 1)** + +While designing the Finding 1 fix, a first draft that checked the guard via a plain `SELECT` before the `UPDATE` (rather than folding it into the `UPDATE`'s `WHERE` clause) was modeled explicitly at per-statement granularity — exposing the read and the write as two separate steps, matching how a real read-then-write actually executes with a yield point in between. TLC found this naive version is **still unsafe**: another writer's full read-decide-write can complete inside the gap between the first writer's read and its own write. + +**Proof**: `node/tla/GraphLifecycleFineGrained.tla` (PlusCal-based, exposes the gap explicitly) vs. `node/tla/GraphLifecycleFineGrainedFixed.tla` (collapses the guard-check-and-write into one atomic step, matching a real single SQL statement). The unguarded/naive version fails; the atomic version passes across 132 reachable states with 7-way branching per step. + +**Implication for implementation**: the Finding 1 fix must be a single `UPDATE ... WHERE status IN (...)` statement. A `SELECT` followed by an `UPDATE` — even with correct guard logic — is not sufficient, regardless of how "obviously correct" the guard looks. + +### Finding 2 — `Instance.status` can regress past `Presigned` · **FIXED (commit `991faaa`)** + +**Severity: same class as Finding 1**, different entity. `Instance.status` (`InstanceBridgeInStatus`) has **no** terminal-status concept anywhere in the codebase (unlike `GraphStatus::is_closed()`). `store_graph` (`node/src/utils.rs`) and the graph-status-guard code both write `InstanceBridgeInStatus::Presigned` unconditionally as a side effect of a graph reaching `CommitteePresigned`, reachable from independent P2P-message and chain-rescan paths with no coordination between them. + +**Proof**: `node/tla/InstancePresigned.tla`. Bug config (`InstancePresignedBug.cfg`) finds a 4-state regression: `Early → Presigned → Advanced → Presigned` — an instance already past `Presigned` (e.g. at `RelayerL1Broadcasted`) gets silently reverted. + +**Verified fix design**: same atomic-CAS pattern as Finding 1 (`InstanceUpdate::only_if_status_in`, guard the specific regression). Proven in `InstancePresignedFixed.cfg`. + +**What actually shipped**: the dual-writer pattern was eliminated rather than merely guarded — `Presigned` is now written from exactly one place, `try_transition_instance_to_presigned` (`node/src/utils.rs`), gated on a genuine quorum check (`has_required_presigned_graphs`) before even attempting the write, and using `update_instance_status_if_current` (`crates/store/src/localdb.rs`) — a real single-value SQL CAS: `UPDATE instance SET status = ? WHERE instance_id = ? AND status = ?`. Stricter than this finding's own proposed guard (a single exact expected predecessor, not a small allowed set), and correctly closes the regression. + +### Finding 3 — `instance_window_expiration_monitor`: stale-read TOCTOU with a full-row upsert · **FIXED (commit `991faaa`)** + +**Severity: data-integrity + possible wrong-decision risk.** `instance_window_expiration_monitor` (`node/src/scheduled_tasks/instance_maintenance_tasks.rs:211-271`) batch-reads a page of instances once, then per-instance awaits an RPC call (`gateway_get_pegin_data`) before merging results and doing a **full-row `upsert_instance`**. For batch position *k*, the snapshot is stale by the sum of *k* prior RPC latencies. A concurrent committee-response landing in that window (via `handle_committee_response_events`, on an independent 5-second scheduler tick) gets silently dropped by the full-row overwrite — and because the `CommitteesAnswered` vs. `NoEnoughCommitteesAnswered` decision is computed from that same stale snapshot, an instance that actually reached quorum can be incorrectly marked as not having reached it. + +**No TLA+ model was built for this one** — it's a classic read-modify-write staleness bug, not a multi-writer race with clean state-machine structure, and was addressed with a narrower code-level fix (re-read the instance immediately before the decision, re-validate the precondition) rather than a formal proof. + +**What actually shipped**: `instance_window_expiration_monitor` now opens `local_db.start_immediate_transaction()` (SQLite `BEGIN IMMEDIATE`, taking the write lock upfront) and re-reads the instance with `find_instance` *inside* that transaction, right before making the decision and writing it — with an explicit comment: *"Re-read while holding SQLite's write lock, then make the decision and its update in the same short transaction so a late committee response cannot be overwritten by the stale page snapshot."* This is exactly the fix direction described above, verified by reading the function directly. + +### Finding 4 — timelock margin gaps in `validate_timelock_config` · **FIXED (commit `991faaa`)** + +**Severity: protocol-parameter safety.** `validate_timelock_config` (`crates/bitvm-gc/src/timelocks.rs:69-117`) checks a chain (`watchtower_challenge < operator_ack < operator_commit < connector_f`) that is correct and already enforced — confirmed by cross-referencing that all four are measured from the same shared clock (`WatchtowerChallengeInit`'s confirmation). But two things were **not** checked: + +1. **No absolute, network-aware floor.** Every field is checked non-zero, but nothing is checked against the network's actual block time. `connector_a` (take1) in particular has no relative check to anything at all — this observation was followed up on directly later in this round and turned into a full finding with its own TLA+ proof and real counterexample; see Finding 9. +2. **A Δ-blind margin comparison.** `prover_connector ≤ connector_d` compares two timelocks with genuinely different clock starts (`connector_d` from `OperatorAssert`, `prover_connector` from `VerifierAssert`, which can only confirm *after* `OperatorAssert`). The check as written accepts equality, with no accounting for that real-world gap. + +**Proof this is exploitable, not just theoretical**: `node/tla/Take2DisproveRace.tla`, modeling the actual UTXO race (`Take2Transaction` spends `ConnectorD` leaf 0, `DisproveTransaction` spends leaf 1 — confirmed by reading `goat/src/transactions/{take2,assert}.rs` directly). Running it against the **actual shipped** testnet4 config found a real boundary case on the first try, no injected bug needed: `prover_connector(22) + min_reaction_blocks(12) = 34 = connector_d(34)` — Disprove's and Take2's earliest-spendable heights land on the **exact same block**, making the outcome a coin-flip on mempool/miner ordering rather than a guaranteed win for the honest Disprove path. + +**Verified fix design**: (a) a network-aware absolute floor (~1 hour of reaction time, derived from `estimated_block_interval_secs`, exempting the pure test networks Signet/Regtest), (b) a *strict* margin requirement (`prover_connector + min_margin < connector_d`, not `≤`), (c) bumping testnet4's `connector_d` from 34 to 35 to restore a valid margin. All three verified in `Take2DisproveRace.cfg` (checked across all 4 networks and a wide range of real-world confirmation-delay values). + +**A caution about this specific finding**: the first draft of the fix used a *non-strict* margin check, which TLC caught failing on the very first run using the real testnet4 value — i.e. the initial fix attempt for Finding 4 itself had the same class of boundary bug it was trying to fix. This is documented as evidence for why every fix in this audit was re-verified by TLC rather than accepted on the strength of the reasoning behind it. + +**What actually shipped**: `min_reaction_blocks(network)` is now a real function, *computed* per-network from a 1-hour floor (`(MIN_REACTION_SECS + interval - 1) / interval`) for Bitcoin/Testnet4 and hardcoded to 1 for Signet/Regtest — independently re-derived by whoever wrote the fix, and it lands on **exactly** the same numbers this audit's `MinReactionBlocks` table used (Bitcoin 6, Testnet4 12, Signet 1, Regtest 1). `ensure_lte` became `ensure_reaction_margin`, using `left.saturating_add(min_margin) >= right` to bail — a real strict-margin check, not just `<`/`<=` swapped. The shipped numbers moved further than this audit's own minimal proposal: Testnet4 `connector_d` went 34→**40** (not just →35) and `prover_connector` moved 22→20, for a real margin of 8 blocks rather than the bare +1 this audit proposed; several other Testnet4 fields (`watchtower_challenge`, `operator_ack`, `operator_commit`, `connector_f`) were retuned too. **Re-verified, not assumed**: `Take2DisproveRace.tla`/`MultiActorRace.tla` were re-run with TLC against these actual new numbers (not the originally-proposed ones) — `ShippedTimelocks.tla` now carries them — and every property still holds: 14,424 and 1,459,745 states respectively, zero violations. + +### Finding 5 (verification, not a bug) — multiple independent watchtowers/verifiers + +Before generalizing Finding 4's fix, it was necessary to check whether the real protocol requires a **quorum** of watchtowers/verifiers to catch fraud, or whether **any single one** suffices regardless of the others — getting this wrong would silently invalidate the single-actor margin analysis above. Ground truth (read directly from `goat/src/connectors/watchtower_connectors.rs`, `connector_e.rs`, `connector_f.rs`, `assert_connectors.rs`, `connector_d.rs`, and `node/src/utils.rs:1264-1326`) confirmed: **true 1-of-N** for both roles. Any single watchtower an operator fails to acknowledge, or any single verifier's fraud assertion the operator fails to rebut, permanently denies the operator's `Take2` claim via a shared bottleneck connector (`ConnectorF` / `ConnectorD`) — first-confirmed-wins UTXO semantics, no counting or quorum anywhere in the code. + +`node/tla/MultiActorRace.tla` modeled N=2 independent watchtowers and M=2 independent verifiers, each choosing their own timing within their protocol-allowed window, across all 4 networks — **1,618,897 combinations checked, zero violations.** Re-injecting Finding 4's exact testnet4 boundary bug into this multi-actor model was caught immediately (with a counterexample where *only one* of the two verifiers acts, confirming the check is genuinely per-actor and not an artifact of needing both to coincide). + +**Conclusion**: the single-actor margin analysis in Finding 4 generalizes correctly to any number of watchtowers/verifiers. No additional fix is needed for multiplicity itself. + +### Finding 6 — `InstanceBridgeOutStatus` can be resurrected to `Initialize` after reaching a terminal outcome · **FIXED (commit `991faaa`)** + +**Severity: same class as Finding 1/2** — local bookkeeping, not direct fund loss, but capable of triggering redundant or contradictory downstream actions against an already-resolved bridge-out (withdraw) instance. + +`InstanceBridgeOutStatus` (`Initialize`/`Claim`/`Timeout`/`Refund`, `crates/store/src/schema.rs:223-229`) is written from **three** independently-scheduled, uncoordinated places, none of which share a transaction spanning read+decide+write: + +- The RPC-service task (`node/src/rpc_service/handler/bitvm2_handler.rs:308-368`, `bridge_out_init_tag`) — a stale-read-then-full-row-`upsert_instance`, sets `Initialize` unconditionally as part of a full-row overwrite. +- The GoatChain L2-event watcher (`node/src/scheduled_tasks/event_watch_task.rs:634-669,756-763,810-815`) — a 5-second tokio task, unconditional targeted `update_instance` on `SwapClaimEvent`/`SwapRefundEvent`, sets `Claim`/`Refund` with **no status precondition** (`InstanceUpdate`'s `WHERE` clause is only `hex(instance_id)=?` — confirmed via `crates/store/src/localdb.rs` that no `with_only_if_status_in`-style guard exists anywhere in the codebase for this entity). +- The maintenance task (`node/src/scheduled_tasks/instance_maintenance_tasks.rs:482-517`, `instance_bridge_out_monitor`) — a 10-second tokio task, batch-reads a stale snapshot then per-row does an unconditional targeted `update_instance` to `Timeout` with no re-check at write time. + +All three tasks run independently and concurrently — confirmed via `main.rs:191-274`'s task topology (RPC handler always-on, watch-event loop every 5s, maintenance loop every 10s). A stale full-row upsert from the RPC path landing after a `Claim`/`Timeout`/`Refund` has already been recorded silently resets the instance back to `Initialize`, which the maintenance task will then treat as still-pending and act on again. + +**Proof**: `node/tla/InstanceBridgeOutRace.tla`, config `InstanceBridgeOutRace.cfg`. Real 3-state counterexample: `Initialize → Claim → Initialize` — exactly the RPC stale-upsert-clobbers-a-recorded-Claim scenario. Modeled the same way `GraphLifecycle.tla` models its race: every writer unconditional on the current status, matching the real, verified behavior — not a simplification of it. + +**Verified fix design**: the same atomic-CAS pattern as Findings 1 and 2 — fold `status \notin {Claim, Timeout, Refund}` into each writer's `UPDATE`/upsert `WHERE` clause. Proven in `InstanceBridgeOutRaceFixed.cfg`. + +**What actually shipped**: `InstanceUpdate` (`crates/store/src/localdb.rs`) gained a real `only_if_status_in: Option>` field, `with_only_if_status_in(...)`, wired into `get_query_builder` via `QueryBuilder::and_where_in("status", statuses, false)` — the exact atomic-CAS mechanism this finding's fix design called for. All three sites were fixed, and the shipped fix goes further than proposed at each one: `bridge_out_init_tag` and the SwapClaim/SwapRefund handlers both switched from full-row `upsert_instance` (no `WHERE` clause possible on `INSERT OR REPLACE`, the actual root cause) to targeted, guarded `update_instance` calls, plus two additional guards this audit didn't flag (`with_only_if_is_bridge_in(false)`, `with_only_if_goat_tx_hash(...)` for idempotency against replayed events) and a real atomic `insert_instance_if_absent` (`INSERT ... ON CONFLICT(instance_id) DO NOTHING`) closing a related instance-creation race. `instance_bridge_out_monitor` now applies the guard unconditionally on every write from that function, not just the `Timeout`-setting branch. + +### Finding 7 — `MessageState` can be resurrected from `Cancelled` to `Pending`, re-dispatching a moot message · **FIXED (commit `991faaa`)** + +**Severity: protocol-hygiene / data-integrity**, lower direct impact than Findings 1/2/6 — this does not corrupt a fund-relevant status field, but it can cause a peer-facing protocol message about an already-finalized graph to be silently re-sent. + +Unlike Findings 1/2/6, one side of this race **is** correctly guarded: `update_messages_state_by_business_id` (`crates/store/src/localdb.rs:1811-1841`) performs a real compare-and-set — `UPDATE ... WHERE business_id=? AND state='Pending'` — called from `node/src/scheduled_tasks/event_watch_task.rs`'s `handle_withdraw_paths_events`/`handle_withdraw_disproved_events` to bulk-cancel any still-`Pending` message for a graph once that graph reaches a closed on-chain status (`OperatorTake1`/`OperatorTake2`/`Disprove`). + +The bug is on the *other* side: `upsert_message` (`node/src/utils.rs:3703-3744`), called by the generic "defer/retry this P2P message" primitive `push_local_unhandled_messages` (`node/src/utils.rs:1698-1717`, ~30 call sites across `node/src/handle.rs`) with `is_update=true`, performs `INSERT ... ON CONFLICT(message_id) DO UPDATE SET state=excluded.state`. An `ON CONFLICT DO UPDATE` upsert has no `WHERE` clause to guard with — so a message the system just administratively marked `Cancelled` (because its graph already closed) can be silently resurrected to `Pending` the next time any handler in the swarm-message-processing path calls a retry/defer on it, entirely unrelated to the cancellation. + +**Proof**: `node/tla/MessageStateRace.tla`, config `MessageStateRace.cfg`. Real 3-state counterexample: `Pending → Cancelled → Pending`, confirming the resurrection is reachable, not merely theoretical. + +**Verified fix design**: guard the resurrect-to-`Pending` write the same way — `status \notin {Cancelled}` folded into the upsert's effective condition (e.g. an `ON CONFLICT ... WHERE message.state != 'Cancelled'` clause, or a read-before-upsert with the CAS pattern used elsewhere). Proven in `MessageStateRaceFixed.cfg`. + +**What actually shipped**: `WHERE message.state != 'Cancelled'` added to `upsert_message`'s `ON CONFLICT(message_id) DO UPDATE SET ...` clause — verbatim the fix design proposed here. One implementation detail worth noting: `upsert_message` was also switched from the compile-time-checked `sqlx::query!` macro to the runtime `sqlx::query`, sidestepping the `.sqlx` query-cache regeneration this audit flagged as a friction point for landing this specific change. + +### Finding 8 (verification, not a bug) — `GoatTxProcessingStatus` has no equivalent race + +The last remaining multi-writer-shaped candidate (written from `node/src/scheduled_tasks/event_watch_task.rs`, `graph_maintenance_tasks.rs`, and `instance_maintenance_tasks.rs`) was checked and found **not** to share the Finding 1/2/6/7 race pattern, for a specific structural reason rather than luck: every write to `GoatTxProcessingStatus::Processed` is gated behind an **on-chain proof requirement** for `proceedWithdraw` (confirmed via `crates/client/src/goat_chain/goat_adaptor.rs:178`) combined with the `is_processing_gateway_history_events` mutex-like gate (`node/src/scheduled_tasks/mod.rs:82-85`), which serializes the competing writers by construction rather than relying on a database-level guard. No TLA+ model was built for this one since there is no race to demonstrate — the causal ordering argument is the finding. + +Confirmed directly against the L2 contract itself (`Gateway.sol`, [`KSlashh/bitvm-L2-contracts@2173b92`](https://github.com/KSlashh/bitvm-L2-contracts/tree/gc-v2), the `gc-v2`-tracking fork): `proceedWithdraw` (`Gateway.sol:474-498`) requires a Merkle-proven kickoff tx (`_verifyMerkleInclusion`) and reverts unless `withdrawData.status == WithdrawStatus.Initialized`, and every withdraw-finalizing function (`proceedWithdraw`, `finishWithdrawHappyPath`, `finishWithdrawUnhappyPath`, `finishWithdrawDisproved`) is `onlyCommittee`-gated. `finishWithdrawDisproved` (`Gateway.sol:521-534`) additionally reverts with `AlreadyDisproved()` if `withdrawData.status == WithdrawStatus.Disproved` already — a real, correctly-guarded terminal state on the contract's *own* on-chain bookkeeping. This is worth stating explicitly: it confirms the pattern common to every race finding in this report (1, 2, 6, 7) is specifically that the *local node's* mirror of on-chain state can drift from a well-guarded source of truth — not that the on-chain/L2 state itself is unguarded. The L2 contract doing its own job correctly is exactly why these are data-integrity findings about the node's bookkeeping, not fund-custody findings about the protocol's on-chain enforcement. + +Two **adjacent, non-race** defects were found while establishing this and are noted here as lower-priority follow-ups, not part of the races proven elsewhere in this report. **Neither appears to be fixed in commit `991faaa`** (checked directly against the current code, not assumed): + +1. **Sticky-`Processed` bug on withdrawal cancel+reinit.** The merge guard originally at `crates/store/src/localdb.rs:2272-2274` only protected the `Processed` status itself from being overwritten. That specific function (`update_goat_tx_record_processing_status`) has since been simplified to a plain unconditional `UPDATE goat_tx_record SET processing_status = ? WHERE instance_id=? AND graph_id=? AND tx_type=?` with no merge-guard logic at all — the underlying cancel+reinit correctness question is unresolved either way; the original narrow bug and the newer unconditional-write shape both remain **open**. +2. **A same-file self-race in the history-catchup task spawner** (`node/src/scheduled_tasks/event_watch_task.rs`, now around line 1322) requiring a 10-minute stall to trigger, gated by `LOAD_HISTORY_EVENT_NO_WOKING_MAX_SECS=600s` (`node/src/env.rs:138`, value unchanged) — the spawn-guard logic looks structurally the same as originally found. **Still open.** + +### Finding 9 — `connector_a` (Take1 vs. Challenge) has no margin check at all · **FIXED (commit `991faaa`)** + +**Severity: protocol-parameter safety, same class as Finding 4** — this is the first check in the peg-out graph's dispute timeline, not a database-bookkeeping issue. + +This round extended the audit from local-database status races to the Bitcoin transaction graph itself: every shared/bottleneck Taproot connector (a UTXO with 2+ leaves spendable by different, competing transactions) was mapped from the `goat` crate source (checkout `e369b2a`, the same commit `Take2DisproveRace.tla` already cites), looking for the same shape of finding Finding 4 found on `ConnectorD`. + +`ConnectorA`'s output is created by `KickoffTransaction` (`goat/src/transactions/kickoff.rs:67-71`) and has two leaves: leaf 0 (operator key + CSV `connector_a`, `connectors/connector_a.rs:37-46`) spent by `Take1Transaction` — the operator's uncontested fast-exit path — and leaf 1 (n-of-n committee key, **no CSV at all**, `connector_a.rs:48-54`) spent by `ChallengeTransaction` using `SinglePlusAnyoneCanPay` (`transactions/challenge.rs:56-57,93`) so any third-party challenger can add their own fee input and force it through the instant fraud is noticed. Because both leaves share the same clock root (Kickoff's confirmation) and Challenge itself has zero wait, `connector_a`'s value **is** the entire real-world window a challenger has to detect fraud and get a Challenge transaction confirmed before Take1's CSV matures. + +`crates/bitvm-gc/src/timelocks.rs`'s `validate_timelock_config` (69-101) checks `connector_a` is merely nonzero (73-81) but — unlike `prover_connector`, `watchtower_challenge`, `operator_ack`, and `operator_commit`, every one of which appears in an `ensure_lt`/`ensure_lte` comparison (91-99) — `connector_a` never appears in any cross-field check. Nothing in the codebase enforces that it leaves a challenger enough reaction time, on any network. + +**Proof this is exploitable, not just theoretical**: `node/tla/Take1ChallengeRace.tla`, config `Take1ChallengeRace.cfg`, checked against the actual shipped per-network values (Bitcoin 144, Testnet4 16, Signet 6, Regtest 1) and reusing `Take2DisproveRace.tla`'s already-established `MinReactionBlocks` policy floor (Bitcoin 6, Testnet4 12, Signet 1, Regtest 1). TLC finds a real counterexample on the first run, no injected bug needed: on Regtest, `connector_a(1) = MinReactionBlocks(1)` — the challenger has *exactly zero* margin, the same "coin-flip boundary" shape as the original `ConnectorD`/testnet4 finding. Bitcoin, Testnet4, and Signet all currently satisfy a strict margin (138/4/5 blocks respectively) — Regtest is the only network that actually fails today — but that is beside the real point: **no mechanism stops a future value from being unsafe on any network**, since the check simply doesn't exist. + +**Verified fix design**: add an `ensure_gt("connector_a", config.connector_a, "min_reaction_blocks", ...)`-style check to `validate_timelock_config`, mirroring Finding 4's floor. Bumping Regtest's shipped value from 1 to 2 is sufficient to satisfy it with everything else unchanged. Proven in `Take1ChallengeRaceFixed.cfg`. + +**What actually shipped**: `ensure_gt("connector_a", config.connector_a, "min_reaction_blocks", min_reaction_blocks(network))` — added to `validate_timelock_config` essentially verbatim, and `NODE_REGTEST_TIMELOCK_CONFIG.connector_a` bumped 1→2, exactly the minimal fix proposed. Re-verified by updating `Take1ChallengeRace.tla`'s `ConnectorA` (kept as a historical record of the pre-fix value, unchanged) against the real `min_reaction_blocks` values and confirming `Take1ChallengeRaceFixed.cfg` (Regtest=2, matching the real shipped number) still passes — it does. `node/tla/Take1ChallengeRace.tla`'s header now notes this history explicitly. + +### Finding 10 (verification, not a bug) — `operator_commit`'s margin against the shared `ConnectorF` UTXO, now formally confirmed + +While mapping every bottleneck connector for Finding 9, `ConnectorF` (`connectors/connector_f.rs`) turned out to have a structural detail Finding 5's `MultiActorRace.tla` didn't fully capture: leaf 1 (the "committee blocks Take2" leaf) has **two** alternative spenders, not one — `OperatorChallengeNackTransaction` (already covered by `NackAlwaysBeatsTake2ViaF`) **and** `OperatorCommitTimeoutTransaction` (`transactions/watchtower_challenge.rs:714-747`, jointly spending `ConnectorE` leaf 1 as its other input). The Rust side already enforces `operator_commit < connector_f` (`timelocks.rs`'s `ensure_lt`), but unlike the `operator_ack`/Nack pairing, that specific comparison had never been independently confirmed with real shipped values at the TLA+ level — it existed only as an unverified scalar Rust assertion. + +`MultiActorRace.tla` was extended (not a new file — same clock-root reasoning, same module, `operator_commit` isn't a multi-actor quantity so no new `VARIABLE` was needed) with `CommitTimeoutAlwaysBeatsTake2ViaF == OperatorCommit[net] < ConnectorF[net]`, checked against real shipped values (`operator_commit`: Bitcoin 432, Testnet4 58, Signet 18, Regtest 3). **Result: holds on all 4 networks** (margins of 144/12/6/1 blocks respectively), re-verified across the full existing 1,618,897-state space in `MultiActorRace.cfg` alongside the properties Finding 5 already established. No new issue — this closes a real gap in *verification coverage*, not a bug in the protocol. + +**Other connectors reviewed, no additional TLA+ model warranted:** +- **`ConnectorZ`** (`PegInConfirm` vs `PegInRefund`, protecting end-user escrowed BTC) has a structurally different shape from the fraud-detection races above: the committee's leaf (immediate, no CSV) never expires, and the user's refund leaf only becomes *additionally* available after a timeout — there is no attacker racing a short CSV against a defender's reaction window the way Finding 4/9 do, so the `MinReactionBlocks`-margin pattern doesn't apply. Whether the underlying escrow design itself is sound is a different, non-margin question outside this round's scope. +- **`AckConnector`'s leaves 0 and 1** (of its 3 leaves; leaf 2 is already covered by Findings 5/10) carry no CSV of their own — confirmed by reading `connectors/watchtower_connectors.rs:96-159` directly — so there is no timelock arithmetic between them to check; whichever is broadcast first wins on ordinary mempool terms, not a margin race. +- **`ConnectorD`'s third leaf** (`PubinDisprove`, `connector_d.rs:20-22,80-88` — `ConnectorD` has 3 leaves, not the 2 `Take2DisproveRace.tla`'s original header implied) carries no CSV, so it can only ever be at least as fast as the already-proven-safe `Disprove` path, never slower — noted in that file's header as a documentation-completeness fix, not a new margin model, since it cannot introduce a new exploitable case. + +--- + +## Full-codebase coverage + +This round's mandate was to check every stateful enum in the codebase, not just the two originally in scope. The methodology: enumerate every `pub enum *Status`/`*State`, grep every `.rs` file that writes to each, and deep-dive (research + TLA+) any type written from 2+ independent files/tasks. `GraphStatus`, `InstanceBridgeInStatus`, `InstanceBridgeOutStatus`, and `MessageState` all met that bar and are covered above (Findings 1/1b, 2, 6, 7). `GoatTxProcessingStatus` met the bar but was found safe (Finding 8). The remaining six were triaged with lighter-weight, targeted evidence review — each is written from effectively one call site/owner or is inherently request-scoped, so no plausible concurrent-writer race exists: + +| Type | Why it's low-risk | +|---|---| +| `WatchContractStatus` | Single owner: only the watchtower-contract lifecycle task writes it; no second writer path exists. | +| `ProofState` | Owned end-to-end by the proof-builder pipeline for a single circuit run; not shared across tokio tasks. | +| `SimpleChallengeSubStatus` | Derived/computed per-read from other already-guarded state, not itself an independently-written column. | +| `VerifierChallengeStatus` | Written only by the per-verifier challenge-processing path for that verifier's own slot; no cross-verifier or cross-task writer. | +| `PeginStatus` | Single writer path in the pegin request handler; terminal transitions are request-response scoped, not background-task driven. | +| `WithdrawStatus` | Same shape as `PeginStatus` — single request-scoped writer, no independent background writer competing for the same row. | + +None of these were modeled in TLA+; the absence of a second independent writer is itself the finding, and is falsifiable by grep (re-run the same "how many `.rs` files write to this enum" search if the code changes). + +--- + +## Documentation drift found during this audit + +Independent of the formal-verification findings above, cross-checking `node/README.md`'s state-machine diagrams against the actual code (rather than trusting the diagrams as ground truth for the TLA+ models) surfaced three places where the documentation had drifted from the implementation: + +1. **`Obsoleted` is not actually terminal.** The README's diagram showed it as a dead end; `scan_graph_chain_state` (`node/src/utils.rs:1418-1434`) can resurrect an `Obsoleted` graph into `OperatorKickOff` or `Skipped` if a kickoff tx is later observed on-chain. +2. **A direct `OperatorKickOff → Disprove` edge** (guardian disprove, bypassing `Challenge` entirely) exists in code (`node/src/utils.rs:1448-1461`) but was missing from the README diagram. +3. **The `ChallengeSubStatus` struct's documented shape didn't match the real struct at all** — the README described three single-value enums; the actual struct (`node/src/scheduled_tasks/graph_maintenance_tasks.rs:52-58`) is a per-watchtower `Vec` plus a per-verifier `Vec`, and the documented enums don't exist in the codebase. + +These were fixed directly in `node/README.md` as documentation corrections (not reverted, since they're not behavioral code changes) — see that file's Graph State Machine section for the corrected diagrams. + +--- + +## Recommendations + +1. ~~Apply the verified fixes.~~ **Done, as of commit `991faaa`**, for all 8 formally-proven findings (1, 1b, 2, 3, 4, 6, 7, 9) — see each finding's "What actually shipped" note above. This was not this audit's own work; it was verified against the real diff after the fact. +2. **Fix the two adjacent, lower-priority defects noted under Finding 8** (sticky-`Processed` on withdrawal cancel+reinit; the history-catchup task spawner's same-file self-race) — checked directly against `991faaa`, **neither is fixed yet**. Still real, narrow bugs worth a small patch; still no formal model needed. +3. **Keep `node/tla/`'s value-carrying specs in sync with `crates/bitvm-gc/src/timelocks.rs` going forward** — this already happened once and needed a manual catch-up: `ShippedTimelocks.tla` held stale pre-fix numbers for several hours after `991faaa` actually shipped different (better) values, because nothing ties the `.tla` constants to the real Rust source automatically. A `cargo test` that reads `timelocks.rs`'s constants and diffs them against the `.tla` file's text (the same pattern this repo used for `tla_model_matches_shipped_timelock_configs` earlier in this round, before it was reverted along with the rest of the applied-fix code) would make this mechanically self-checking instead of relying on someone remembering to look. +4. **Consider extending Finding 5's multi-actor model** if the team wants deeper coverage — e.g. Byzantine actors (a watchtower/verifier actively trying to help the operator, not just staying silent), or N > 2 to rule out any count-dependent effect the N=2 case might not surface. +5. **Adopt a traceability mapping going forward**: a `traceability/*.yaml`-style mapping from every stateful enum's writers to the Rust symbols that touch them would make the "which types have 2+ independent writers" triage in this audit mechanically re-checkable instead of requiring a fresh grep sweep each round. +6. **Keep the tripwire discipline this audit's tooling established.** `node/tla/`'s `.tla` files cite the exact Rust functions they model in reverse-pointer comments; if those functions are edited again, the specs need to be re-verified, not just assumed to still apply. The bug-reproduction configs (e.g. `GraphLifecycle.cfg`) are deliberately kept modeling the *pre-fix* code as a permanent historical record, run every CI build, but only fail the job if one of them unexpectedly starts passing — see root `README.md` and `.github/workflows/ci.yml`'s comments for the full reasoning, and why `tla-plus` is expected to be green on this branch now that the underlying bugs are fixed. +7. **Extend the connector sweep to a structural/topology and value-conservation audit** as a separate follow-up round, if desired. This round's connector sweep (Findings 9/10) covered every *margin/reaction-time* race in the Bitcoin transaction graph; it deliberately did not check whether the constructed transactions' actual wiring (inputs/outputs/amounts) matches the intended graph, whether fees/amounts conserve correctly end-to-end, or whether the Taproot leaf scripts encode the intended authorization — those are different classes of correctness property, better suited to direct code audit or Rust property tests than to TLA+'s margin-arithmetic style. + +--- + +## Spec inventory + +See root `README.md`'s "Formal verification (TLA+)" section for the authoritative, up-to-date table (spec, config, what it models, expected result) and exact run commands. Summary: + +| Spec | Findings it covers | Status | +|---|---|---| +| `GraphLifecycle.tla` | Finding 1 | Fixed in `991faaa`; bug config kept as permanent historical regression check | +| `GraphLifecycleFineGrained.tla` / `GraphLifecycleFineGrainedFixed.tla` | Finding 1b | Fixed in `991faaa`; bug config kept as permanent historical regression check | +| `InstancePresigned.tla` | Finding 2 | Fixed in `991faaa`; bug config kept as permanent historical regression check | +| `Take2DisproveRace.tla` | Finding 4 | Fixed in `991faaa`; `ShippedTimelocks.tla` updated to the real new numbers, re-verified via TLC | +| `MultiActorRace.tla` | Finding 5 | Verification (no bug); still holds under the real new `991faaa` numbers | +| `InstanceBridgeOutRace.tla` | Finding 6 | Fixed in `991faaa`; bug config kept as permanent historical regression check | +| `MessageStateRace.tla` | Finding 7 | Fixed in `991faaa`; bug config kept as permanent historical regression check | +| `Take1ChallengeRace.tla` | Finding 9 | Fixed in `991faaa`; `ConnectorA` kept as historical pre-fix value, `ConnectorAFixed` matches the real shipped value | +| `MultiActorRace.tla` (extended) | Finding 10 | Verification (no bug); still holds under the real new `991faaa` numbers | + +All specs are runnable today: `cd node/tla && java -jar ~/.local/share/tlaplus/tla2tools.jar -config .cfg .tla`. CI (`.github/workflows/ci.yml`, job `tla-plus`) runs the full set on every push/PR against `gc-v2`, and is *expected* to show the bug-reproduction configs still failing — they model the pre-fix code on purpose, as a permanent regression check, not a claim that the bug is still live in shipped code. See that job's own comments and the note at the top of this report. diff --git a/circuits/commit-chain-proof/host/src/lib.rs b/circuits/commit-chain-proof/host/src/lib.rs index 400281e82..e5fce4076 100644 --- a/circuits/commit-chain-proof/host/src/lib.rs +++ b/circuits/commit-chain-proof/host/src/lib.rs @@ -82,14 +82,14 @@ pub async fn fetch_commit_chain( commits_file: &str, network: Network, ) -> anyhow::Result> { - let btc_client = BTCClient::new(network, Some(&esplora_url)); + let btc_client = BTCClient::new(network, Some(esplora_url)); tracing::info!( "Fetching commit chain, commit_info_file: {}, commits_file: {}", commit_info_file, commits_file ); - let rdr = std::fs::File::open(commit_info_file).context(&("read error"))?; + let rdr = std::fs::File::open(commit_info_file).context("read error")?; let ci: CommitInfo = serde_json::from_reader(rdr)?; let mut commits: Vec = vec![]; let txid = Txid::from_str(&ci.txid)?; @@ -131,8 +131,8 @@ pub async fn fetch_commit_chain( block_height, }; commits.push(commit); - std::fs::write(&commits_file, serde_json::to_vec(&commits)?) - .expect(&format!("write {commits_file} error")); + std::fs::write(commits_file, serde_json::to_vec(&commits)?) + .with_context(|| format!("write {commits_file} error"))?; Ok(commits) } @@ -144,6 +144,7 @@ pub struct CommitChainProofBuilder { } impl CommitChainProofBuilder { + #[allow(clippy::new_without_default)] pub fn new() -> Self { let client = ProverClient::new(); let (proving_key, verifying_key) = client.setup(COMMIT_CHAIN); @@ -183,7 +184,7 @@ impl ProofBuilder for CommitChainProofBuilder { let prev_receipt = if *init_input { None } else { - let public_inputs = fs::read(&format!("{}.public_inputs.bin", input_proof)) + let public_inputs = fs::read(format!("{}.public_inputs.bin", input_proof)) .context("Read public input")?; //let prev: CommitChainCircuitOutput = serde_json::from_slice(&public_inputs).unwrap(); Some(public_inputs) @@ -193,8 +194,8 @@ impl ProofBuilder for CommitChainProofBuilder { Some(public_inputs) => { let proof_bytes = fs::read(input_proof).context("Failed to read input proof file")?; - let zkm_vk_hash = fs::read(&format!("{}.vk_hash.bin", input_proof)) - .context("Read vk hash")?; + let zkm_vk_hash = + fs::read(format!("{}.vk_hash.bin", input_proof)).context("Read vk hash")?; let version_path = format!("{input_proof}.zkm_version.bin"); let zkm_version = fs::read(&version_path) .with_context(|| { @@ -290,16 +291,16 @@ impl ProofBuilder for CommitChainProofBuilder { //tracing::info!("Generate proof successfully, proof: {:?}", proof); //Ok((public_value_hex, proof_size)) - std::fs::write(&format!("{}", output_proof), proof.bytes())?; + std::fs::write(output_proof, proof.bytes())?; let public_value_hex = hex::encode(proof.public_values.to_vec()); let proof_size = proof.bytes().len(); let zkm_version = proof.zkm_version.clone(); std::fs::write( - &format!("{}.public_inputs.bin", output_proof), + format!("{}.public_inputs.bin", output_proof), proof.public_values.to_vec(), )?; - std::fs::write(&format!("{}.vk_hash.bin", output_proof), self.verifying_key.bytes32())?; - std::fs::write(&format!("{}.zkm_version.bin", output_proof), zkm_version)?; + std::fs::write(format!("{}.vk_hash.bin", output_proof), self.verifying_key.bytes32())?; + std::fs::write(format!("{}.zkm_version.bin", output_proof), zkm_version)?; Ok((public_value_hex, proof_size)) } } diff --git a/circuits/header-chain-proof/host/src/lib.rs b/circuits/header-chain-proof/host/src/lib.rs index 9710a5d6b..e107db823 100644 --- a/circuits/header-chain-proof/host/src/lib.rs +++ b/circuits/header-chain-proof/host/src/lib.rs @@ -80,8 +80,9 @@ pub async fn fetch_header_chain( .read(true) .write(true) .create(true) + .truncate(false) .open(block_header_file) - .expect(&format!("Open {block_header_file} error")); + .with_context(|| format!("Open {block_header_file} error"))?; let mut headers: Vec = Vec::new(); writer.read_to_end(&mut headers)?; @@ -151,6 +152,7 @@ pub struct HeaderChainProofBuilder { } impl HeaderChainProofBuilder { + #[allow(clippy::new_without_default)] pub fn new() -> Self { let client = ProverClient::new(); let (proving_key, verifying_key) = client.setup(HEADER_CHAIN); @@ -197,9 +199,10 @@ impl ProofBuilder for HeaderChainProofBuilder { let prev_receipt = if *init_input { None } else { - let public_inputs = fs::read(&format!("{}.public_inputs.bin", input_proof)).expect( - &format!("Failed to read public inputs from {}.public_inputs.bin", input_proof), - ); + let public_inputs = fs::read(format!("{}.public_inputs.bin", input_proof)) + .with_context(|| { + format!("Failed to read public inputs from {}.public_inputs.bin", input_proof) + })?; Some(public_inputs) }; @@ -208,7 +211,7 @@ impl ProofBuilder for HeaderChainProofBuilder { Some(public_inputs) => { let proof_bytes = fs::read(input_proof).context("Failed to read input proof file").unwrap(); - let zkm_vk_hash = fs::read(&format!("{}.vk_hash.bin", input_proof)).unwrap(); + let zkm_vk_hash = fs::read(format!("{}.vk_hash.bin", input_proof)).unwrap(); let version_path = format!("{input_proof}.zkm_version.bin"); let zkm_version = fs::read(&version_path) .with_context(|| { @@ -244,7 +247,7 @@ impl ProofBuilder for HeaderChainProofBuilder { batch_size ); - let block_headers = (&total_block_headers[*start..*start + *batch_size]).to_vec(); + let block_headers = total_block_headers[*start..*start + *batch_size].to_vec(); let input: HeaderChainCircuitInput = HeaderChainCircuitInput { prev_proof, zkm_proof, @@ -306,16 +309,16 @@ impl ProofBuilder for HeaderChainProofBuilder { //fs::write(&format!("{}.vk", output_proof), bincode::serialize(&self.verifying_key)?)?; //fs::write(&format!("{}.in", output_proof), input)?; - std::fs::write(&format!("{}", output_proof), proof.bytes())?; + std::fs::write(output_proof, proof.bytes())?; let public_value_hex = hex::encode(proof.public_values.to_vec()); let proof_size = proof.bytes().len(); let zkm_version = proof.zkm_version.clone(); std::fs::write( - &format!("{}.public_inputs.bin", output_proof), + format!("{}.public_inputs.bin", output_proof), proof.public_values.to_vec(), )?; - std::fs::write(&format!("{}.vk_hash.bin", output_proof), self.verifying_key.bytes32())?; - std::fs::write(&format!("{}.zkm_version.bin", output_proof), zkm_version)?; + std::fs::write(format!("{}.vk_hash.bin", output_proof), self.verifying_key.bytes32())?; + std::fs::write(format!("{}.zkm_version.bin", output_proof), zkm_version)?; tracing::info!("Generate proof successfully, proof: {:?}", proof); Ok((public_value_hex, proof_size)) diff --git a/circuits/operator-proof/host/src/lib.rs b/circuits/operator-proof/host/src/lib.rs index 73231f637..5c62a4ae8 100644 --- a/circuits/operator-proof/host/src/lib.rs +++ b/circuits/operator-proof/host/src/lib.rs @@ -89,11 +89,14 @@ use sha2::{Digest, Sha256}; use std::sync::OnceLock; static ELF_ID: OnceLock = OnceLock::new(); +type IndexedWatchtowerInputs = Vec<(u16, Txid, PublicKey)>; +type GraphWatchtowerXOnlyPublicKeys = Vec<[u8; 32]>; + /// Parses the full graph key list and keeps each included challenge's original graph index. fn parse_indexed_watchtower_inputs( watchtower_challenge_txids: &str, watchtower_public_keys: &str, -) -> anyhow::Result<(Vec<(u16, Txid, PublicKey)>, Vec<[u8; 32]>)> { +) -> anyhow::Result<(IndexedWatchtowerInputs, GraphWatchtowerXOnlyPublicKeys)> { let public_keys = watchtower_public_keys .split(',') .map(PublicKey::from_str) @@ -143,9 +146,9 @@ pub async fn fetch_target_block_and_watchtower_tx( )> { let (indexed_watchtower_inputs, graph_watchtower_xonly_public_keys) = parse_indexed_watchtower_inputs(watchtower_challenge_txids, watchtower_public_keys)?; - let btc_client = BTCClient::new(bitcoin_network, Some(&esplora_url)); + let btc_client = BTCClient::new(bitcoin_network, Some(esplora_url)); - let latest_sequencer_commit_txid = Txid::from_str(&latest_sequencer_commit_txid)?; + let latest_sequencer_commit_txid = Txid::from_str(latest_sequencer_commit_txid)?; let operator_latest_sequencer_commit_txn = match btc_client.get_tx(&latest_sequencer_commit_txid).await? { Some(tx) => tx, @@ -156,7 +159,7 @@ pub async fn fetch_target_block_and_watchtower_tx( }; let tx_status = btc_client.get_tx_status(&latest_sequencer_commit_txid).await?; let block_pos_ss_commit = match tx_status.block_height { - Some(height) => height as u32, + Some(height) => height, None => anyhow::bail!( "Latest sequencer commit txn is not confirmed yet: {}", latest_sequencer_commit_txid @@ -258,6 +261,7 @@ pub struct OperatorProofBuilder { } impl OperatorProofBuilder { + #[allow(clippy::new_without_default)] pub fn new() -> Self { let client = ProverClient::new(); let (proving_key, verifying_key) = client.setup(OPERATOR); @@ -316,12 +320,12 @@ impl ProofBuilder for OperatorProofBuilder { // --- header chain --- // let header_chain_input = { let zkm_public_values = - fs::read(&format!("{}.public_inputs.bin", header_chain_input_proof)).unwrap(); + fs::read(format!("{}.public_inputs.bin", header_chain_input_proof)).unwrap(); let zkm_proof = fs::read(header_chain_input_proof) .context("Failed to read input proof file") .unwrap(); let zkm_vk_hash = - fs::read(&format!("{}.vk_hash.bin", header_chain_input_proof)).unwrap(); + fs::read(format!("{}.vk_hash.bin", header_chain_input_proof)).unwrap(); let version_path = format!("{header_chain_input_proof}.zkm_version.bin"); let zkm_version = fs::read(&version_path) .with_context(|| format!("failed to read zkm_version file '{version_path}'")) @@ -343,12 +347,12 @@ impl ProofBuilder for OperatorProofBuilder { // --- commit chain --- // let commit_chain_input = { let zkm_public_values = - fs::read(&format!("{}.public_inputs.bin", commit_chain_input_proof)).unwrap(); + fs::read(format!("{}.public_inputs.bin", commit_chain_input_proof)).unwrap(); let zkm_proof = fs::read(commit_chain_input_proof) .context("Failed to read input proof file") .unwrap(); let zkm_vk_hash = - fs::read(&format!("{}.vk_hash.bin", commit_chain_input_proof)).unwrap(); + fs::read(format!("{}.vk_hash.bin", commit_chain_input_proof)).unwrap(); let version_path = format!("{commit_chain_input_proof}.zkm_version.bin"); let zkm_version = fs::read(&version_path) .with_context(|| format!("failed to read zkm_version file '{version_path}'")) @@ -373,9 +377,8 @@ impl ProofBuilder for OperatorProofBuilder { .context("Failed to read input proof file") .unwrap(); let zkm_public_values = - fs::read(&format!("{}.public_inputs.bin", state_chain_input_proof)).unwrap(); - let zkm_vk_hash = - fs::read(&format!("{}.vk_hash.bin", state_chain_input_proof)).unwrap(); + fs::read(format!("{}.public_inputs.bin", state_chain_input_proof)).unwrap(); + let zkm_vk_hash = fs::read(format!("{}.vk_hash.bin", state_chain_input_proof)).unwrap(); let version_path = format!("{state_chain_input_proof}.zkm_version.bin"); let zkm_version = fs::read(&version_path) .with_context(|| format!("failed to read zkm_version file '{version_path}'")) @@ -398,11 +401,10 @@ impl ProofBuilder for OperatorProofBuilder { // --- spv --- // //let latest_sequencer_commit_txid = Txid::from_str(&latest_sequencer_commit_txid).unwrap(); - let operator_genesis_sequencer_commit_txid = - Txid::from_str(&genesis_sequencer_commit_txid)?; + let operator_genesis_sequencer_commit_txid = Txid::from_str(genesis_sequencer_commit_txid)?; let bitcoin_block_headers = { - let headers: Vec = std::fs::read(&format!("{header_chain_input_proof}.blocks")) + let headers: Vec = std::fs::read(format!("{header_chain_input_proof}.blocks")) .context("read header chain blocks error")?; headers .chunks(80) @@ -426,7 +428,7 @@ impl ProofBuilder for OperatorProofBuilder { operator_latest_sequencer_commit_txn.compute_txid() ); let spv_ss_commit = build_spv( - &operator_latest_sequencer_commit_txn, + operator_latest_sequencer_commit_txn, *block_pos_ss_commit, target_block_ss_commit.clone(), &bitcoin_block_headers, @@ -437,7 +439,7 @@ impl ProofBuilder for OperatorProofBuilder { || -> anyhow::Result<(ZKMProofWithPublicValues, u64, f32)> { let mut stdin = ZKMStdin::new(); - let included_watchtowers: U256 = U256::from_str(&included_watchtowers).unwrap(); + let included_watchtowers: U256 = U256::from_str(included_watchtowers).unwrap(); stdin.write(&included_watchtowers); stdin.write(&graph_id); @@ -495,11 +497,11 @@ impl ProofBuilder for OperatorProofBuilder { let public_value_hex = hex::encode(proof.public_values.to_vec()); let proof_size = proof.bytes().len(); let zkm_version = proof.zkm_version.clone(); - std::fs::write(&format!("{}.public_inputs.bin", output), proof.public_values.to_vec())?; - std::fs::write(&format!("{}.vk_hash.bin", output), self.verifying_key.bytes32())?; - std::fs::write(&format!("{}.zkm_version.bin", output), zkm_version)?; + std::fs::write(format!("{}.public_inputs.bin", output), proof.public_values.to_vec())?; + std::fs::write(format!("{}.vk_hash.bin", output), self.verifying_key.bytes32())?; + std::fs::write(format!("{}.zkm_version.bin", output), zkm_version)?; let proof = bincode::serialize(&proof)?; - std::fs::write(&format!("{}", output), proof)?; + std::fs::write(output, proof)?; Ok((public_value_hex, proof_size)) } } diff --git a/circuits/state-chain-proof/host/src/lib.rs b/circuits/state-chain-proof/host/src/lib.rs index e908c95aa..b29a6a360 100644 --- a/circuits/state-chain-proof/host/src/lib.rs +++ b/circuits/state-chain-proof/host/src/lib.rs @@ -98,7 +98,7 @@ async fn fetch_withdrawal_events( start: u64, batch_size: u64, ) -> anyhow::Result> { - let rpc_url = Url::parse(&execution_layer_rpc)?; + let rpc_url = Url::parse(execution_layer_rpc)?; let provider = RootProvider::::new_http(rpc_url); let mut withdrawals = vec![]; for i in start..start + batch_size { @@ -150,7 +150,7 @@ async fn fetch_exection_layer_block( genesis: &Genesis, ) -> anyhow::Result { // Setup the provider. - let rpc_url = Url::parse(&execution_layer_rpc)?; + let rpc_url = Url::parse(execution_layer_rpc)?; let provider = RootProvider::::new_http(rpc_url); //let rpc_db = RpcDb::new(provider.clone(), provider.clone(), execution_layer_block_number - 1); let chain_spec: Arc = Arc::new(genesis.try_into().unwrap()); @@ -170,6 +170,7 @@ async fn fetch_exection_layer_block( Ok(client_input) } +#[allow(clippy::too_many_arguments)] pub async fn fetch_state_chain( l2_contract_addresses: &str, proceed_withdraw_method_ids: &str, @@ -194,7 +195,7 @@ pub async fn fetch_state_chain( let genesis = if genesis == "goattest" { Genesis::GoatTestnet } else { Genesis::GOAT }; assert!(start > 0, "Don't get genesis block from the consensus layer."); let mut blocks: Vec<_> = Vec::new(); - let base_slot: [u8; 32] = U256::from(16).to_be_bytes().try_into()?; + let base_slot: [u8; 32] = U256::from(16).to_be_bytes(); // fetch graph_block_numbers and graph_ids between in goat block(start, start + batch_size) let withdrawal_events = fetch_withdrawal_events( execution_layer_rpc, @@ -207,7 +208,7 @@ pub async fn fetch_state_chain( for i in start..(start + batch_size) { let evm_block = - fetch_exection_layer_block(&execution_layer_rpc, i, &genesis).await.map_err(|e| { + fetch_exection_layer_block(execution_layer_rpc, i, &genesis).await.map_err(|e| { tracing::error!("fetch_exection_layer_block: {e:?}"); proof_builder::ProofError::InputNotReady((batch_size + start - i) * 3) })?; @@ -216,10 +217,9 @@ pub async fn fetch_state_chain( evm_block.current_block.header.parent_beacon_block_root.unwrap(); let parent_cosmos_block_height = - match get_cosmos_block_height_at(cosmos_rpc_url, *parent_beacon_block_root).await { - Ok(d) => d, - Err(_) => None, - }; + get_cosmos_block_height_at(cosmos_rpc_url, *parent_beacon_block_root) + .await + .unwrap_or_default(); let (_, cl_block_number) = fetch_cbft_validator_info(cosmos_rpc_url, i, parent_cosmos_block_height, 1000) @@ -265,7 +265,7 @@ pub async fn fetch_state_chain( blocks.push(CircuitStateBlock { cosmos_txns, cosmos_block, evm_block, withdrawals }); } let block_bytes = serde_json::to_vec(&blocks)?; - std::fs::write(&blocks_file, block_bytes)?; + std::fs::write(blocks_file, block_bytes)?; Ok(blocks) } @@ -276,6 +276,7 @@ pub struct StateChainProofBuilder { } impl StateChainProofBuilder { + #[allow(clippy::new_without_default)] pub fn new() -> Self { let client = ProverClient::new(); let (proving_key, verifying_key) = client.setup(STATE_CHAIN); @@ -313,7 +314,7 @@ impl ProofBuilder for StateChainProofBuilder { let prev_receipt = if *init_input { None } else { - let public_inputs = fs::read(&format!("{}.public_inputs.bin", input_proof)) + let public_inputs = fs::read(format!("{}.public_inputs.bin", input_proof)) .context("Read public input")?; Some(public_inputs) }; @@ -323,8 +324,8 @@ impl ProofBuilder for StateChainProofBuilder { Some(public_inputs) => { let proof_bytes = fs::read(input_proof).context("Failed to read input proof file")?; - let zkm_vk_hash = fs::read(&format!("{}.vk_hash.bin", input_proof)) - .context("Read vk_hash")?; + let zkm_vk_hash = + fs::read(format!("{}.vk_hash.bin", input_proof)).context("Read vk_hash")?; let version_path = format!("{input_proof}.zkm_version.bin"); let zkm_version = fs::read(&version_path) .with_context(|| { @@ -407,16 +408,16 @@ impl ProofBuilder for StateChainProofBuilder { let ProofRequest::StateChainProofRequest { output_proof, .. } = ctx else { anyhow::bail!("Invalid state chain inputs"); }; - std::fs::write(&format!("{}", output_proof), proof.bytes())?; + std::fs::write(output_proof, proof.bytes())?; let public_value_hex = hex::encode(proof.public_values.to_vec()); let proof_size = proof.bytes().len(); let zkm_version = proof.zkm_version.clone(); std::fs::write( - &format!("{}.public_inputs.bin", output_proof), + format!("{}.public_inputs.bin", output_proof), proof.public_values.to_vec(), )?; - std::fs::write(&format!("{}.vk_hash.bin", output_proof), self.verifying_key.bytes32())?; - std::fs::write(&format!("{}.zkm_version.bin", output_proof), zkm_version)?; + std::fs::write(format!("{}.vk_hash.bin", output_proof), self.verifying_key.bytes32())?; + std::fs::write(format!("{}.zkm_version.bin", output_proof), zkm_version)?; tracing::info!("Generate proof successfully, proof: {:?}", proof); Ok((public_value_hex, proof_size)) diff --git a/circuits/watchtower-proof/host/src/lib.rs b/circuits/watchtower-proof/host/src/lib.rs index d7aa2cccf..e0023bc0e 100644 --- a/circuits/watchtower-proof/host/src/lib.rs +++ b/circuits/watchtower-proof/host/src/lib.rs @@ -64,7 +64,7 @@ pub async fn fetch_target_block( latest_sequencer_commit_txid: &str, bitcoin_network: Network, ) -> anyhow::Result<(u32, Block, Transaction)> { - let btc_client = client::btc_chain::BTCClient::new(bitcoin_network, Some(&esplora_url)); + let btc_client = client::btc_chain::BTCClient::new(bitcoin_network, Some(esplora_url)); let latest_sequencer_commit_txid = Txid::from_str(latest_sequencer_commit_txid).unwrap(); let latest_sequencer_commit_tx = @@ -88,6 +88,7 @@ pub struct WatchtowerProofBuilder { } impl WatchtowerProofBuilder { + #[allow(clippy::new_without_default)] pub fn new() -> Self { let client = ProverClient::new(); let (proving_key, verifying_key) = client.setup(WATCHTOWER); @@ -135,12 +136,12 @@ impl ProofBuilder for WatchtowerProofBuilder { // --- header chain --- // let header_chain_input = { let zkm_public_values = - fs::read(&format!("{}.public_inputs.bin", header_chain_input_proof)).unwrap(); + fs::read(format!("{}.public_inputs.bin", header_chain_input_proof)).unwrap(); let zkm_proof = fs::read(header_chain_input_proof) .context("Failed to read input proof file") .unwrap(); let zkm_vk_hash = - fs::read(&format!("{}.vk_hash.bin", header_chain_input_proof)).unwrap(); + fs::read(format!("{}.vk_hash.bin", header_chain_input_proof)).unwrap(); let version_path = format!("{header_chain_input_proof}.zkm_version.bin"); let zkm_version = fs::read(&version_path) .with_context(|| format!("failed to read zkm_version file '{version_path}'")) @@ -163,12 +164,12 @@ impl ProofBuilder for WatchtowerProofBuilder { // --- commit chain --- // let commit_chain_input = { let zkm_public_values = - fs::read(&format!("{}.public_inputs.bin", commit_chain_input_proof)).unwrap(); + fs::read(format!("{}.public_inputs.bin", commit_chain_input_proof)).unwrap(); let zkm_proof = fs::read(commit_chain_input_proof) .context("Failed to read input proof file") .unwrap(); let zkm_vk_hash = - fs::read(&format!("{}.vk_hash.bin", commit_chain_input_proof)).unwrap(); + fs::read(format!("{}.vk_hash.bin", commit_chain_input_proof)).unwrap(); let version_path = format!("{commit_chain_input_proof}.zkm_version.bin"); let zkm_version = fs::read(&version_path) .with_context(|| format!("failed to read zkm_version file '{version_path}'")) @@ -193,9 +194,8 @@ impl ProofBuilder for WatchtowerProofBuilder { .context("Failed to read input proof file") .unwrap(); let zkm_public_values = - fs::read(&format!("{}.public_inputs.bin", state_chain_input_proof)).unwrap(); - let zkm_vk_hash = - fs::read(&format!("{}.vk_hash.bin", state_chain_input_proof)).unwrap(); + fs::read(format!("{}.public_inputs.bin", state_chain_input_proof)).unwrap(); + let zkm_vk_hash = fs::read(format!("{}.vk_hash.bin", state_chain_input_proof)).unwrap(); let version_path = format!("{state_chain_input_proof}.zkm_version.bin"); let zkm_version = fs::read(&version_path) .with_context(|| format!("failed to read zkm_version file '{version_path}'")) @@ -214,10 +214,10 @@ impl ProofBuilder for WatchtowerProofBuilder { } }; // --- spv --- // - let genesis_sequencer_commit_txid = Txid::from_str(&genesis_sequencer_commit_txid)?; - let latest_sequencer_commit_txid = Txid::from_str(&latest_sequencer_commit_txid)?; + let genesis_sequencer_commit_txid = Txid::from_str(genesis_sequencer_commit_txid)?; + let latest_sequencer_commit_txid = Txid::from_str(latest_sequencer_commit_txid)?; let bitcoin_block_headers = { - let headers: Vec = std::fs::read(&format!("{header_chain_input_proof}.blocks"))?; + let headers: Vec = std::fs::read(format!("{header_chain_input_proof}.blocks"))?; headers .chunks(80) .map(|header| CircuitBlockHeader::try_from_slice(header).unwrap()) @@ -287,13 +287,13 @@ impl ProofBuilder for WatchtowerProofBuilder { let ProofRequest::WatchtowerProofRequest { output, .. } = ctx else { anyhow::bail!("invalid context"); }; - std::fs::write(&format!("{}", output), proof.bytes())?; + std::fs::write(output, proof.bytes())?; let public_value_hex = hex::encode(proof.public_values.to_vec()); let proof_size = proof.bytes().len(); let zkm_version = proof.zkm_version.clone(); - std::fs::write(&format!("{}.public_inputs.bin", output), proof.public_values.to_vec())?; - std::fs::write(&format!("{}.vk_hash.bin", output), self.verifying_key.bytes32())?; - std::fs::write(&format!("{}.zkm_version.bin", output), zkm_version)?; + std::fs::write(format!("{}.public_inputs.bin", output), proof.public_values.to_vec())?; + std::fs::write(format!("{}.vk_hash.bin", output), self.verifying_key.bytes32())?; + std::fs::write(format!("{}.zkm_version.bin", output), zkm_version)?; Ok((public_value_hex, proof_size)) } } diff --git a/crates/bitcoin-light-client-circuit/src/signature.rs b/crates/bitcoin-light-client-circuit/src/signature.rs index 43f145885..49510880a 100644 --- a/crates/bitcoin-light-client-circuit/src/signature.rs +++ b/crates/bitcoin-light-client-circuit/src/signature.rs @@ -127,14 +127,14 @@ mod tests { }], output: vec![TxOut { value: Amount::from_sat(49_000), - script_pubkey: ScriptBuf::new_op_return(&[0x6a]), + script_pubkey: ScriptBuf::new_op_return([0x6a]), }], }; // 6. Generate Schnorr signature let sig = generate_taproot_leaf_schnorr_signature( &mut spending_tx, - &[prev_out.clone()], + std::slice::from_ref(&prev_out), 0, TapSighashType::AllPlusAnyoneCanPay, &script, diff --git a/crates/bitvm-gc/src/babe_adapter.rs b/crates/bitvm-gc/src/babe_adapter.rs index 3ee682b6d..360c74892 100644 --- a/crates/bitvm-gc/src/babe_adapter.rs +++ b/crates/bitvm-gc/src/babe_adapter.rs @@ -206,6 +206,7 @@ pub fn open_real_setup_and_solder( } /// Verifies real CAC openings, commitments, and the native Ziren soldering proof. +#[allow(clippy::too_many_arguments)] pub fn verify_real_setup( soldering_builder: &BabeBundleBuilder, package: &CACSetupPackage, @@ -223,7 +224,7 @@ pub fn verify_real_setup( }; soldering_builder .babe_prover_verify_setup( - &package, + package, &bundle, vk, static_public_inputs, @@ -512,12 +513,8 @@ fn restore_real_verifier( vk: &Groth16VerifyingKey, static_inputs: Fr, ) -> Result { - let verifier = BABEVerifier::from_state(&state.instance_seeds, package, vk, static_inputs); - if verifier.is_none() { - Err(anyhow::anyhow!("Cannot restore real verifier")) - } else { - Ok(verifier.unwrap()) - } + BABEVerifier::from_state(&state.instance_seeds, package, vk, static_inputs) + .ok_or_else(|| anyhow::anyhow!("Cannot restore real verifier")) } fn validate_finalized_indices( diff --git a/crates/bitvm-gc/src/timelocks.rs b/crates/bitvm-gc/src/timelocks.rs index a65703631..835ab4875 100644 --- a/crates/bitvm-gc/src/timelocks.rs +++ b/crates/bitvm-gc/src/timelocks.rs @@ -20,12 +20,12 @@ pub const NODE_BITCOIN_TIMELOCK_CONFIG: TimelockConfig = TimelockConfig { pub const NODE_TESTNET_TIMELOCK_CONFIG: TimelockConfig = TimelockConfig { connector_z: 100, connector_a: 16, - prover_connector: 22, - connector_d: 34, - watchtower_challenge: 34, - operator_ack: 46, - operator_commit: 58, - connector_f: 70, + prover_connector: 20, + connector_d: 40, + watchtower_challenge: 20, + operator_ack: 32, + operator_commit: 40, + connector_f: 52, }; pub const NODE_SIGNET_TIMELOCK_CONFIG: TimelockConfig = TimelockConfig { connector_z: 6, @@ -39,7 +39,7 @@ pub const NODE_SIGNET_TIMELOCK_CONFIG: TimelockConfig = TimelockConfig { }; pub const NODE_REGTEST_TIMELOCK_CONFIG: TimelockConfig = TimelockConfig { connector_z: 1, - connector_a: 1, + connector_a: 2, prover_connector: 1, connector_d: 3, watchtower_challenge: 1, @@ -66,7 +66,20 @@ pub fn estimated_block_interval_secs(network: Network) -> i64 { } } +const MIN_REACTION_SECS: i64 = 3600; + +fn min_reaction_blocks(network: Network) -> u32 { + match network { + Network::Bitcoin | Network::Testnet | Network::Testnet4 => { + let interval = estimated_block_interval_secs(network); + ((MIN_REACTION_SECS + interval - 1) / interval) as u32 + } + Network::Signet | Network::Regtest => 1, + } +} + pub fn validate_timelock_config(network: Network, config: &TimelockConfig) -> Result<()> { + let min_blocks = min_reaction_blocks(network); for (name, value) in [ ("connector_z", config.connector_z), ("connector_a", config.connector_a), @@ -77,8 +90,11 @@ pub fn validate_timelock_config(network: Network, config: &TimelockConfig) -> Re ("operator_commit", config.operator_commit), ("connector_f", config.connector_f), ] { - if value == 0 { - bail!("timelock_config.{name} must be greater than 0"); + if value < min_blocks { + bail!( + "timelock_config.{name} must be at least {min_blocks} blocks \ + (~{MIN_REACTION_SECS}s reaction window), got {value}" + ); } } let default_connector_z = default_timelock_config(network).connector_z; @@ -89,7 +105,14 @@ pub fn validate_timelock_config(network: Network, config: &TimelockConfig) -> Re ); } - ensure_lte("prover_connector", config.prover_connector, "connector_d", config.connector_d)?; + ensure_reaction_margin( + "prover_connector", + config.prover_connector, + "connector_d", + config.connector_d, + min_blocks, + )?; + ensure_gt("connector_a", config.connector_a, "min_reaction_blocks", min_blocks)?; ensure_lt( "watchtower_challenge", config.watchtower_challenge, @@ -102,9 +125,18 @@ pub fn validate_timelock_config(network: Network, config: &TimelockConfig) -> Re Ok(()) } -fn ensure_lte(left_name: &str, left: u32, right_name: &str, right: u32) -> Result<()> { - if left > right { - bail!("timelock_config.{left_name} must be <= timelock_config.{right_name}"); +fn ensure_reaction_margin( + left_name: &str, + left: u32, + right_name: &str, + right: u32, + min_margin: u32, +) -> Result<()> { + if left.saturating_add(min_margin) >= right { + bail!( + "timelock_config.{left_name} must be more than {min_margin} blocks less than \ + timelock_config.{right_name}" + ); } Ok(()) } @@ -116,6 +148,13 @@ fn ensure_lt(left_name: &str, left: u32, right_name: &str, right: u32) -> Result Ok(()) } +fn ensure_gt(left_name: &str, left: u32, right_name: &str, right: u32) -> Result<()> { + if left <= right { + bail!("timelock_config.{left_name} must be > {right_name} ({right})"); + } + Ok(()) +} + pub fn timelock_blocks(_network: Network, blocks: u32) -> u32 { blocks } @@ -155,3 +194,22 @@ pub fn operator_ack_timelock_blocks(network: Network, config: &TimelockConfig) - pub fn operator_commit_timelock_blocks(network: Network, config: &TimelockConfig) -> u32 { timelock_blocks(network, config.operator_commit) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn all_default_timelock_configs_validate() { + for (network, config) in [ + (Network::Bitcoin, NODE_BITCOIN_TIMELOCK_CONFIG), + (Network::Testnet, NODE_TESTNET_TIMELOCK_CONFIG), + (Network::Testnet4, NODE_TESTNET_TIMELOCK_CONFIG), + (Network::Signet, NODE_SIGNET_TIMELOCK_CONFIG), + (Network::Regtest, NODE_REGTEST_TIMELOCK_CONFIG), + ] { + assert_eq!(default_timelock_config(network), config); + validate_timelock_config(network, &config).unwrap(); + } + } +} diff --git a/crates/bitvm-gc/src/types.rs b/crates/bitvm-gc/src/types.rs index abb2c1d50..65e27178f 100644 --- a/crates/bitvm-gc/src/types.rs +++ b/crates/bitvm-gc/src/types.rs @@ -718,7 +718,7 @@ impl BitvmGcInstanceParameters { impl BitvmGcGraphParameters { pub fn canonical_graph_params_hash(&self) -> Result<[u8; 32]> { - let encoded = bincode::serialize(self)?; + let encoded = serde_json::to_vec(self)?; let mut hasher = Sha256::new(); hasher.update(b"GOAT_BITVM_GC_GRAPH_PARAMS_V1"); hasher.update((encoded.len() as u64).to_be_bytes()); diff --git a/crates/commit-chain/src/commit_chain.rs b/crates/commit-chain/src/commit_chain.rs index bb51246e2..4b8c5a087 100644 --- a/crates/commit-chain/src/commit_chain.rs +++ b/crates/commit-chain/src/commit_chain.rs @@ -370,7 +370,7 @@ mod tests { fn test_extract_op_return() { // Example: construct a fake tx with OP_RETURN let expected_op_data = [12, 3, 4, 45]; - let script = ScriptBuf::new_op_return(&expected_op_data); + let script = ScriptBuf::new_op_return(expected_op_data); let tx = Transaction { version: bitcoin::transaction::Version::TWO, lock_time: bitcoin::absolute::LockTime::ZERO, diff --git a/crates/commit-chain/src/publisher.rs b/crates/commit-chain/src/publisher.rs index 193a3d860..5e25fedc0 100644 --- a/crates/commit-chain/src/publisher.rs +++ b/crates/commit-chain/src/publisher.rs @@ -219,8 +219,7 @@ mod tests { let prevout = TxOut { value: prev_value, script_pubkey }; // Fake OutPoint - let prev_outpoint = - OutPoint { txid: bitcoin::Txid::from_byte_array([0u8; 32].into()), vout: 0 }; + let prev_outpoint = OutPoint { txid: bitcoin::Txid::from_byte_array([0u8; 32]), vout: 0 }; // === Step 4: construct spending tx === let mut tx = Transaction { @@ -236,7 +235,7 @@ mod tests { value: Amount::from_sat(99_000), script_pubkey: { let btc_pk0 = bitcoin::PublicKey::from(pubkeys[0]); - Address::p2pkh(&btc_pk0, Network::Testnet).script_pubkey() + Address::p2pkh(btc_pk0, Network::Testnet).script_pubkey() }, }], }; @@ -254,15 +253,9 @@ mod tests { finalize(&mut tx, vec![sig1, sig2], &redeem_script).unwrap(); // === Step 7: verify === - let ok = verify_p2wsh_multisig_witness( - &tx, - 0, - &prevout, - &redeem_script, - &pubkeys, - threshold as usize, - ) - .unwrap(); + let ok = + verify_p2wsh_multisig_witness(&tx, 0, &prevout, &redeem_script, &pubkeys, threshold) + .unwrap(); assert!(ok, "2-of-3 multisig witness should verify"); } @@ -278,8 +271,7 @@ mod tests { value: prev_value, script_pubkey: ScriptBuf::new_p2wsh(&ScriptBuf::new().wscript_hash()), }; - let prev_outpoint = - OutPoint { txid: bitcoin::Txid::from_byte_array([0u8; 32].into()), vout: 0 }; + let prev_outpoint = OutPoint { txid: bitcoin::Txid::from_byte_array([0u8; 32]), vout: 0 }; let mut tx = Transaction { version: Version::TWO, @@ -294,7 +286,7 @@ mod tests { value: Amount::from_sat(99_000), script_pubkey: { let btc_pk0 = bitcoin::PublicKey::from(pubkeys[0]); - Address::p2pkh(&btc_pk0, Network::Testnet).script_pubkey() + Address::p2pkh(btc_pk0, Network::Testnet).script_pubkey() }, }], }; @@ -308,15 +300,8 @@ mod tests { finalize(&mut tx, vec![sig1, sig2], &redeem_script).unwrap(); assert!( - verify_p2wsh_multisig_witness( - &tx, - 0, - &prevout, - &redeem_script, - &pubkeys, - threshold as usize, - ) - .is_err() + verify_p2wsh_multisig_witness(&tx, 0, &prevout, &redeem_script, &pubkeys, threshold,) + .is_err() ); } } diff --git a/crates/header-chain/src/header_chain.rs b/crates/header-chain/src/header_chain.rs index 9bfd2a681..a36e91824 100644 --- a/crates/header-chain/src/header_chain.rs +++ b/crates/header-chain/src/header_chain.rs @@ -921,10 +921,10 @@ mod tests { block_headers[..11].iter().map(|header| header.time).collect::>(); // The validation is expected to return false - assert_eq!( - validate_timestamp(block_headers[1].time, first_11_timestamps.try_into().unwrap(),), - false - ); + assert!(!validate_timestamp( + block_headers[1].time, + first_11_timestamps.try_into().unwrap(), + )); } #[test] @@ -937,13 +937,10 @@ mod tests { let first_11_timestamps = block_headers[..11].iter().map(|header| header.time).collect::>(); - assert_eq!( - validate_timestamp( - block_headers[11].time, - first_11_timestamps.clone().try_into().unwrap(), - ), - true - ); + assert!(validate_timestamp( + block_headers[11].time, + first_11_timestamps.clone().try_into().unwrap(), + )); } #[test] @@ -1028,7 +1025,7 @@ mod tests { nonce: 2083236893, }; - let bridge_header: CircuitBlockHeader = header.clone().into(); + let bridge_header: CircuitBlockHeader = header.into(); assert_eq!(bridge_header.version, header.version.to_consensus()); assert_eq!(bridge_header.prev_block_hash, *header.prev_blockhash.as_byte_array()); @@ -1072,7 +1069,7 @@ mod tests { nonce: 2083236893, }; - let bridge_header: CircuitBlockHeader = original_header.clone().into(); + let bridge_header: CircuitBlockHeader = original_header.into(); let converted_header: Header = bridge_header.into(); assert_eq!(original_header, converted_header); diff --git a/crates/state-chain/src/cbft.rs b/crates/state-chain/src/cbft.rs index 36962fd70..6c4a5f9d6 100644 --- a/crates/state-chain/src/cbft.rs +++ b/crates/state-chain/src/cbft.rs @@ -153,7 +153,7 @@ mod tests { pub fn test_verify_goat_block() { // https://explorer.goat.network/block/5756298 // curl "http://127.0.0.1:26657/block?height=5756784" | jq .result.block.data - let cosmos_txns: Vec = serde_json::from_str(&LB_1_JSON_TXNS).unwrap(); + let cosmos_txns: Vec = serde_json::from_str(LB_1_JSON_TXNS).unwrap(); let cosmos_txns = cosmos_txns.into_iter().map(|s| BASE64.decode(s).unwrap()).collect::>(); // loght block 5756784 @@ -177,7 +177,7 @@ mod tests { let light_block_2 = serde_json::from_str::(LB_2_JSON).unwrap(); // curl "http://127.0.0.1:26657/block?height=5756785" | jq .result.block.data // https://explorer.goat.network/block/5756299 - let cosmos_txns: Vec = serde_json::from_str(&LB_2_JSON_TXNS).unwrap(); + let cosmos_txns: Vec = serde_json::from_str(LB_2_JSON_TXNS).unwrap(); let cosmos_txns = cosmos_txns.into_iter().map(|s| BASE64.decode(s).unwrap()).collect::>(); check_el_block_from_payload( diff --git a/crates/store/.sqlx/query-ede88a2e872505ac5e58b2e71280dc8081e776358af4302a88f2f683c8d97ad4.json b/crates/store/.sqlx/query-ede88a2e872505ac5e58b2e71280dc8081e776358af4302a88f2f683c8d97ad4.json new file mode 100644 index 000000000..73cd38143 --- /dev/null +++ b/crates/store/.sqlx/query-ede88a2e872505ac5e58b2e71280dc8081e776358af4302a88f2f683c8d97ad4.json @@ -0,0 +1,12 @@ +{ + "db_name": "SQLite", + "query": "UPDATE instance SET status = ?, status_updated_at = ?, updated_at = ? WHERE instance_id = ? AND status = ?", + "describe": { + "columns": [], + "parameters": { + "Right": 5 + }, + "nullable": [] + }, + "hash": "ede88a2e872505ac5e58b2e71280dc8081e776358af4302a88f2f683c8d97ad4" +} diff --git a/crates/store/migrations/20260720000000_add_graph_definition_hash.sql b/crates/store/migrations/20260720000000_add_graph_definition_hash.sql new file mode 100644 index 000000000..4240e96ed --- /dev/null +++ b/crates/store/migrations/20260720000000_add_graph_definition_hash.sql @@ -0,0 +1,2 @@ +ALTER TABLE graph + ADD COLUMN `definition_hash` TEXT NOT NULL DEFAULT ''; diff --git a/crates/store/migrations/20260720000001_create_graph_compensation_marker.sql b/crates/store/migrations/20260720000001_create_graph_compensation_marker.sql new file mode 100644 index 000000000..6f0a73147 --- /dev/null +++ b/crates/store/migrations/20260720000001_create_graph_compensation_marker.sql @@ -0,0 +1,6 @@ +CREATE TABLE graph_compensation_marker ( + graph_id BLOB NOT NULL, + message_id TEXT NOT NULL, + created_at INTEGER NOT NULL, + PRIMARY KEY (graph_id, message_id) +); diff --git a/crates/store/src/localdb.rs b/crates/store/src/localdb.rs index bdd557f9d..422dfe340 100644 --- a/crates/store/src/localdb.rs +++ b/crates/store/src/localdb.rs @@ -1,17 +1,19 @@ use crate::utils::{QueryBuilder, QueryParam, create_place_holders}; use crate::{ - BridgeOutGlobalStats, GoatTxRecord, Graph, GraphBtcTxVoutMonitor, GraphRawData, Instance, - LongRunningTaskProof, Message, Node, NodesOverview, OperatorProof, PeginGraphProcessData, - PeginInstanceProcessData, PendingGraphInit, SequencerSetHashChange, SequencerSetScanState, - SerializableTxid, WatchContract, WatchtowerProof, + BridgeOutGlobalStats, GoatTxRecord, Graph, GraphBtcTxVoutMonitor, GraphRawData, GraphStatus, + GraphStatusSource, GraphStatusTransitionOutcome, Instance, LongRunningTaskProof, Message, Node, + NodesOverview, OperatorProof, PeginGraphProcessData, PeginInstanceProcessData, + PendingGraphInit, SequencerSetHashChange, SequencerSetScanState, SerializableTxid, + WatchContract, WatchtowerProof, }; use indexmap::IndexMap; use sqlx::migrate::Migrator; use sqlx::pool::PoolConnection; +use sqlx::sqlite::SqliteRow; use sqlx::types::Uuid; use sqlx::{Row, Sqlite, SqliteConnection, SqlitePool, Transaction, migrate::MigrateDatabase}; -use std::collections::HashMap; +use std::str::FromStr; use std::time::{SystemTime, UNIX_EPOCH}; use tracing::warn; @@ -19,8 +21,20 @@ fn get_current_timestamp_secs() -> i64 { SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs() as i64 } -fn get_current_timestamp_millis() -> i64 { - SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_millis() as i64 +fn message_from_row(row: &SqliteRow) -> Result { + Ok(Message { + message_id: row.try_get("message_id")?, + business_id: row.try_get("business_id")?, + actor: row.try_get("actor")?, + from_peer: row.try_get("from_peer")?, + msg_type: row.try_get("msg_type")?, + content: row.try_get("content")?, + state: row.try_get("state")?, + message_version: row.try_get("message_version")?, + weight: row.try_get("weight")?, + lock_time_until: row.try_get("lock_time_until")?, + created_at: row.try_get("created_at")?, + }) } #[derive(Clone, Debug)] @@ -43,13 +57,25 @@ pub struct StorageProcessor<'a> { pub in_transaction: bool, } +#[derive(Clone, Debug, Default)] +pub struct MessageQueueStats { + pub pending_ready: i64, + pub pending_locked: i64, + pub failed: i64, + pub oldest_pending_at: Option, +} + static MIGRATOR: Migrator = sqlx::migrate!("./migrations"); impl LocalDB { pub async fn new(path: &str, is_mem: bool) -> LocalDB { if !Sqlite::database_exists(path).await.unwrap_or(false) { tracing::info!("Creating database {}", path); match Sqlite::create_database(path).await { - Ok(_) => println!("Create db success"), + Ok(_) => tracing::info!( + event = "database_lifecycle", + outcome = "created", + "local database created" + ), Err(error) => panic!("error: {error}"), } } else { @@ -81,6 +107,16 @@ impl LocalDB { in_transaction: true, }) } + + /// Start a short write transaction before reading state that will be + /// immediately reconciled. This prevents a stale snapshot from dropping + /// a concurrent update between the read and the conditional write. + pub async fn start_immediate_transaction<'a>(&self) -> anyhow::Result> { + Ok(StorageProcessor { + conn: ConnectionHolder::Transaction(self.conn.begin_with("BEGIN IMMEDIATE").await?), + in_transaction: true, + }) + } } #[derive(Clone, Debug, Default)] @@ -211,44 +247,54 @@ pub struct InstanceUpdate { pub to_addr: Option, pub btc_txid: Option, pub status: Option, - pub pegin_confirm_txid: Option, + pub pegin_confirm_txid: Option, + pub pegin_cancel_txid: Option, pub post_pegin_txhash: Option, pub btc_height: Option, - pub committees_answers: Option>>, + pub committees_answers: Option>>, pub bridge_out_lock_time: Option, + pub bridge_out_amount: Option, + pub goat_tx_hash: Option, + pub goat_tx_height: Option, + pub user_change_addr: Option, + pub user_refund_addr: Option, + pub only_if_status_in: Option>, + pub only_if_is_bridge_in: Option, + pub only_if_goat_tx_hash: Option, } impl InstanceUpdate { - /// Create new update parameters - pub fn new_with_instance_id(instance_id: Uuid) -> Self { + fn empty() -> Self { Self { - instance_id: Some(instance_id), + instance_id: None, escrow_hash: None, from_addr: None, to_addr: None, btc_txid: None, status: None, pegin_confirm_txid: None, + pegin_cancel_txid: None, post_pegin_txhash: None, btc_height: None, committees_answers: None, bridge_out_lock_time: None, + bridge_out_amount: None, + goat_tx_hash: None, + goat_tx_height: None, + user_change_addr: None, + user_refund_addr: None, + only_if_status_in: None, + only_if_is_bridge_in: None, + only_if_goat_tx_hash: None, } } + + /// Create new update parameters + pub fn new_with_instance_id(instance_id: Uuid) -> Self { + Self { instance_id: Some(instance_id), ..Self::empty() } + } pub fn new_with_escrow_hash(escrow_hash: String) -> Self { - Self { - instance_id: None, - escrow_hash: Some(escrow_hash), - to_addr: None, - from_addr: None, - btc_txid: None, - status: None, - pegin_confirm_txid: None, - post_pegin_txhash: None, - btc_height: None, - committees_answers: None, - bridge_out_lock_time: None, - } + Self { escrow_hash: Some(escrow_hash), ..Self::empty() } } /// Set from_addr @@ -274,12 +320,18 @@ impl InstanceUpdate { self } - /// Set pegin confirmation information - pub fn with_pegin_confirm(mut self, txid: String, _fee: i64) -> Self { + /// Set pegin confirmation transaction ID. + pub fn with_pegin_confirm_txid(mut self, txid: SerializableTxid) -> Self { self.pegin_confirm_txid = Some(txid); self } + /// Set pegin cancellation transaction ID. + pub fn with_pegin_cancel_txid(mut self, txid: SerializableTxid) -> Self { + self.pegin_cancel_txid = Some(txid); + self + } + /// Set post pegin information pub fn with_post_pegin(mut self, txid: String) -> Self { self.post_pegin_txhash = Some(txid); @@ -287,7 +339,10 @@ impl InstanceUpdate { } /// Set committees answers - pub fn with_committees_answers(mut self, committees_answers: HashMap>) -> Self { + pub fn with_committees_answers( + mut self, + committees_answers: IndexMap>, + ) -> Self { self.committees_answers = Some(committees_answers); self } @@ -303,6 +358,52 @@ impl InstanceUpdate { self.bridge_out_lock_time = Some(bridge_out_lock_time); self } + + pub fn with_bridge_out_amount(mut self, bridge_out_amount: String) -> Self { + self.bridge_out_amount = Some(bridge_out_amount); + self + } + + pub fn with_goat_tx_hash(mut self, goat_tx_hash: String) -> Self { + self.goat_tx_hash = Some(goat_tx_hash); + self + } + + pub fn with_goat_tx_height(mut self, goat_tx_height: i64) -> Self { + self.goat_tx_height = Some(goat_tx_height); + self + } + + pub fn with_user_change_addr(mut self, user_change_addr: String) -> Self { + self.user_change_addr = Some(user_change_addr); + self + } + + pub fn with_user_refund_addr(mut self, user_refund_addr: String) -> Self { + self.user_refund_addr = Some(user_refund_addr); + self + } + + /// Apply this update only while the instance is still in one of the + /// expected states. The condition is folded into the UPDATE statement. + pub fn with_only_if_status_in(mut self, statuses: Vec) -> Self { + self.only_if_status_in = Some(statuses); + self + } + + /// Apply this update only to the expected bridge direction. + pub fn with_only_if_is_bridge_in(mut self, is_bridge_in: bool) -> Self { + self.only_if_is_bridge_in = Some(is_bridge_in); + self + } + + /// Apply this update only when the existing Goat transaction hash is the + /// expected value. Used to make swap initialization idempotent. + pub fn with_only_if_goat_tx_hash(mut self, goat_tx_hash: String) -> Self { + self.only_if_goat_tx_hash = Some(goat_tx_hash); + self + } + /// Check if any fields need to be updated pub fn has_updates(&self) -> bool { self.escrow_hash.is_some() @@ -311,10 +412,16 @@ impl InstanceUpdate { || self.btc_txid.is_some() || self.status.is_some() || self.pegin_confirm_txid.is_some() + || self.pegin_cancel_txid.is_some() || self.post_pegin_txhash.is_some() || self.btc_height.is_some() || self.committees_answers.is_some() || self.bridge_out_lock_time.is_some() + || self.bridge_out_amount.is_some() + || self.goat_tx_hash.is_some() + || self.goat_tx_height.is_some() + || self.user_change_addr.is_some() + || self.user_refund_addr.is_some() } pub fn get_query_builder(&self, base_sql: &str) -> QueryBuilder { @@ -327,7 +434,11 @@ impl InstanceUpdate { } if let Some(ref txid) = self.pegin_confirm_txid { - query_builder.set_field("pegin_confirm_txid", QueryParam::Text(txid.clone())); + query_builder.set_field("pegin_confirm_txid", QueryParam::BTCTxid(txid.clone())); + } + + if let Some(ref txid) = self.pegin_cancel_txid { + query_builder.set_field("pegin_cancel_txid", QueryParam::BTCTxid(txid.clone())); } if let Some(ref txid) = self.post_pegin_txhash { @@ -354,6 +465,33 @@ impl InstanceUpdate { query_builder.set_field("bridge_out_lock_time", QueryParam::Int(bridge_out_lock_time)); } + if let Some(ref bridge_out_amount) = self.bridge_out_amount { + query_builder + .set_field("bridge_out_amount", QueryParam::Text(bridge_out_amount.clone())); + } + + if let Some(ref goat_tx_hash) = self.goat_tx_hash { + query_builder.set_field("goat_tx_hash", QueryParam::Text(goat_tx_hash.clone())); + } + + if let Some(goat_tx_height) = self.goat_tx_height { + query_builder.set_field("goat_tx_height", QueryParam::Int(goat_tx_height)); + } + + if let Some(ref user_change_addr) = self.user_change_addr { + query_builder.set_field("user_change_addr", QueryParam::Text(user_change_addr.clone())); + } + + if let Some(ref user_refund_addr) = self.user_refund_addr { + query_builder.set_field("user_refund_addr", QueryParam::Text(user_refund_addr.clone())); + } + + if let Some(ref committees_answers) = self.committees_answers { + let committees_answers = serde_json::to_string(committees_answers) + .expect("IndexMap> serialization is infallible"); + query_builder.set_field("committees_answers", QueryParam::Text(committees_answers)); + } + // Add update time let current_time = get_current_timestamp_secs(); query_builder.set_field("updated_at", QueryParam::Int(current_time)); @@ -370,6 +508,25 @@ impl InstanceUpdate { .and_where("escrow_hash = ? ", Some(QueryParam::Text(escrow_hash.clone()))); } + if let Some(ref statuses) = self.only_if_status_in { + if statuses.is_empty() { + // An empty allow-list must reject the update rather than + // silently dropping the compare-and-swap guard. + query_builder.and_where("1 = 0", None); + } else { + query_builder.and_where_in("status", statuses, false); + } + } + + if let Some(is_bridge_in) = self.only_if_is_bridge_in { + query_builder.and_where("is_bridge_in = ?", Some(QueryParam::Bool(is_bridge_in))); + } + + if let Some(ref goat_tx_hash) = self.only_if_goat_tx_hash { + query_builder + .and_where("goat_tx_hash = ?", Some(QueryParam::Text(goat_tx_hash.clone()))); + } + query_builder } } @@ -497,89 +654,40 @@ impl GraphQuery { } } +/// Runtime graph fields that are not part of the signed graph definition. +/// +/// Status is intentionally absent. All graph status changes must go through +/// `StorageProcessor::transition_graph_status` so a stale event cannot replace +/// a later chain-observed state. #[derive(Clone, Debug)] -pub struct GraphUpdate { +pub struct GraphRuntimeUpdate { + pub instance_id: Uuid, pub graph_id: Uuid, - pub status: Option, - pub sub_status: Option, pub challenge_txid: Option, - pub disprove_txids: Option>, - pub watchtower_challenge_timeout_txids: Option>, - pub operator_challenge_nack_txids: Option>, - pub operator_commit_timeout_txid: Option>, pub bridge_out_start_at: Option, pub init_withdraw_tx_hash: Option, pub proceed_withdraw_height: Option, } -impl GraphUpdate { +impl GraphRuntimeUpdate { /// Create new update parameters - pub fn new(graph_id: Uuid) -> Self { + pub fn new(instance_id: Uuid, graph_id: Uuid) -> Self { Self { + instance_id, graph_id, - status: None, - sub_status: None, challenge_txid: None, - disprove_txids: None, - watchtower_challenge_timeout_txids: None, - operator_challenge_nack_txids: None, - operator_commit_timeout_txid: None, bridge_out_start_at: None, init_withdraw_tx_hash: None, proceed_withdraw_height: None, } } - /// Set status - pub fn with_status(mut self, status: String) -> Self { - self.status = Some(status); - self - } - /// Set sub_status - pub fn with_sub_status(mut self, sub_status: String) -> Self { - self.sub_status = Some(sub_status); - self - } - /// Set challenge transaction ID pub fn with_challenge_txid(mut self, challenge_txid: SerializableTxid) -> Self { self.challenge_txid = Some(challenge_txid); self } - /// Set disprove transaction IDs - pub fn with_disprove_txids(mut self, disprove_txids: Vec) -> Self { - self.disprove_txids = Some(disprove_txids); - self - } - - /// Set watchtower challenge timeout transaction IDs - pub fn with_watchtower_challenge_timeout_txids( - mut self, - watchtower_challenge_timeout_txids: Vec, - ) -> Self { - self.watchtower_challenge_timeout_txids = Some(watchtower_challenge_timeout_txids); - self - } - - /// Set operator challenge NACK transaction IDs - pub fn with_operator_challenge_nack_txids( - mut self, - operator_challenge_nack_txids: Vec, - ) -> Self { - self.operator_challenge_nack_txids = Some(operator_challenge_nack_txids); - self - } - - /// Set operator commit timeout transaction ID - pub fn with_operator_commit_timeout_txid( - mut self, - operator_commit_timeout_txid: Option, - ) -> Self { - self.operator_commit_timeout_txid = Some(operator_commit_timeout_txid); - self - } - /// Set bridge out start time pub fn with_bridge_out_start_at(mut self, bridge_out_start_at: i64) -> Self { self.bridge_out_start_at = Some(bridge_out_start_at); @@ -600,13 +708,7 @@ impl GraphUpdate { /// Check if any fields need to be updated pub fn has_updates(&self) -> bool { - self.status.is_some() - || self.sub_status.is_some() - || self.challenge_txid.is_some() - || self.disprove_txids.is_some() - || self.watchtower_challenge_timeout_txids.is_some() - || self.operator_challenge_nack_txids.is_some() - || self.operator_commit_timeout_txid.is_some() + self.challenge_txid.is_some() || self.bridge_out_start_at.is_some() || self.init_withdraw_tx_hash.is_some() || self.proceed_withdraw_height.is_some() @@ -615,51 +717,9 @@ impl GraphUpdate { pub fn get_query_builder(&self, base_sql: &str) -> QueryBuilder { let mut query_builder = QueryBuilder::update(base_sql); // Add SET fields - if let Some(ref status) = self.status { - query_builder.set_field("status", QueryParam::Text(status.clone())); - query_builder - .set_field("status_updated_at", QueryParam::Int(get_current_timestamp_secs())); - } - if let Some(ref sub_status) = self.sub_status { - query_builder.set_field("sub_status", QueryParam::Text(sub_status.clone())); - } if let Some(ref challenge_txid) = self.challenge_txid { query_builder.set_field("challenge_txid", QueryParam::BTCTxid(challenge_txid.clone())); } - if let Some(ref disprove_txids) = self.disprove_txids { - let disprove_txids_json = - serde_json::to_string(disprove_txids).unwrap_or_else(|_| "[]".into()); - query_builder.set_field("disprove_txids", QueryParam::Text(disprove_txids_json)); - } - if let Some(ref watchtower_challenge_timeout_txids) = - self.watchtower_challenge_timeout_txids - { - let watchtower_challenge_timeout_txids_json = - serde_json::to_string(watchtower_challenge_timeout_txids) - .unwrap_or_else(|_| "[]".into()); - query_builder.set_field( - "watchtower_challenge_timeout_txids", - QueryParam::Text(watchtower_challenge_timeout_txids_json), - ); - } - if let Some(ref operator_challenge_nack_txids) = self.operator_challenge_nack_txids { - let operator_challenge_nack_txids_json = - serde_json::to_string(operator_challenge_nack_txids) - .unwrap_or_else(|_| "[]".into()); - query_builder.set_field( - "operator_challenge_nack_txids", - QueryParam::Text(operator_challenge_nack_txids_json), - ); - } - if let Some(ref operator_commit_timeout_txid) = self.operator_commit_timeout_txid { - match operator_commit_timeout_txid { - Some(operator_commit_timeout_txid) => query_builder.set_field( - "operator_commit_timeout_txid", - QueryParam::BTCTxid(operator_commit_timeout_txid.clone()), - ), - None => query_builder.set_field_null("operator_commit_timeout_txid"), - } - } if let Some(bridge_out_start_at) = self.bridge_out_start_at { query_builder.set_field("bridge_out_start_at", QueryParam::Int(bridge_out_start_at)); } @@ -687,6 +747,10 @@ impl GraphUpdate { "hex(graph_id) = ? COLLATE NOCASE", Some(QueryParam::Text(hex::encode(self.graph_id))), ); + query_builder.and_where( + "hex(instance_id) = ? COLLATE NOCASE", + Some(QueryParam::Text(hex::encode(self.instance_id))), + ); query_builder } @@ -845,6 +909,56 @@ impl<'a> StorageProcessor<'a> { Ok(res.rows_affected() > 0) } + /// Insert an instance only when its ID is not already present. + /// + /// Creation paths that race with status transitions must use this instead + /// of `upsert_instance`, whose `INSERT OR REPLACE` semantics can restore + /// a stale full row over a newer terminal status. + pub async fn insert_instance_if_absent(&mut self, instance: &Instance) -> anyhow::Result { + let committees_answers_json = serde_json::to_string(&instance.committees_answers)?; + let res = sqlx::query( + "INSERT INTO instance \ + (instance_id, is_bridge_in, network, from_addr, to_addr, amount, fees, input_utxos, \ + status, goat_tx_hash, goat_tx_height, user_xonly_pubkey, user_change_addr, \ + user_refund_addr, btc_txid, pegin_confirm_txid, pegin_cancel_txid, committees_answers, \ + pegin_data_tx_hash, btc_height, parameters, status_updated_at, escrow_hash, \ + bridge_out_lock_time, post_pegin_txhash, bridge_out_amount, created_at, updated_at) \ + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) \ + ON CONFLICT(instance_id) DO NOTHING", + ) + .bind(instance.instance_id) + .bind(instance.is_bridge_in) + .bind(&instance.network) + .bind(&instance.from_addr) + .bind(&instance.to_addr) + .bind(instance.amount) + .bind(instance.fees) + .bind(&instance.input_utxos) + .bind(&instance.status) + .bind(&instance.goat_tx_hash) + .bind(instance.goat_tx_height) + .bind(instance.user_xonly_pubkey) + .bind(&instance.user_change_addr) + .bind(&instance.user_refund_addr) + .bind(&instance.btc_txid) + .bind(&instance.pegin_confirm_txid) + .bind(&instance.pegin_cancel_txid) + .bind(committees_answers_json) + .bind(&instance.pegin_data_tx_hash) + .bind(instance.btc_height) + .bind(&instance.parameters) + .bind(instance.status_updated_at) + .bind(&instance.escrow_hash) + .bind(instance.bridge_out_lock_time) + .bind(&instance.post_pegin_txhash) + .bind(&instance.bridge_out_amount) + .bind(instance.created_at) + .bind(instance.updated_at) + .execute(self.conn()) + .await?; + Ok(res.rows_affected() > 0) + } + /// Find a single instance by its ID /// /// Retrieves an instance from the database using its unique instance_id. @@ -976,6 +1090,29 @@ impl<'a> StorageProcessor<'a> { Ok(result.rows_affected() > 0) } + /// Transition an instance only when it is still in the expected status. + pub async fn update_instance_status_if_current( + &mut self, + instance_id: &Uuid, + current_status: &str, + new_status: &str, + ) -> anyhow::Result { + let current_time = get_current_timestamp_secs(); + let result = sqlx::query!( + "UPDATE instance SET status = ?, status_updated_at = ?, updated_at = ? \ + WHERE instance_id = ? AND status = ?", + new_status, + current_time, + current_time, + instance_id, + current_status, + ) + .execute(self.conn()) + .await?; + + Ok(result.rows_affected() > 0) + } + /// Update instance pegin confirmation information /// /// Method specifically for updating pegin confirmation transaction ID and fee @@ -1026,6 +1163,9 @@ impl<'a> StorageProcessor<'a> { if !params.has_updates() { return Ok(false); } + if params.instance_id.is_none() && params.escrow_hash.is_none() { + anyhow::bail!("instance update requires instance_id or escrow_hash"); + } let query_builder = params.get_query_builder("instance"); // Get SQL and parameters let update_sql = query_builder.get_sql(); @@ -1038,28 +1178,28 @@ impl<'a> StorageProcessor<'a> { /// Add or update a single committee answer for an instance /// - /// This method allows adding or updating a single committee's answer - /// without needing to provide the entire committees_answers HashMap. + /// This method merges one committee answer atomically so concurrent event + /// handlers cannot overwrite each other's answers with stale full maps. pub async fn update_instance_committee_answer( &mut self, instance_id: &Uuid, committee_addr: &str, pubkey: Vec, ) -> anyhow::Result { - // First, get the current committees_answers - let current_instance = self.find_instance(instance_id).await?; - if current_instance.is_none() { - return Ok(false); - } - - let mut committees_answers = current_instance.unwrap().committees_answers; - committees_answers - .entry(committee_addr.to_string()) - .and_modify(|existing| { - *existing = pubkey.clone(); - }) - .or_insert_with(|| pubkey); - self.update_instance_committees_answers_map(instance_id, &committees_answers).await + let committee_patch = serde_json::json!({ (committee_addr): pubkey }).to_string(); + let current_time = get_current_timestamp_secs(); + let result = sqlx::query( + "UPDATE instance \ + SET committees_answers = json_patch(COALESCE(committees_answers, '{}'), json(?)), \ + updated_at = ? \ + WHERE instance_id = ?", + ) + .bind(committee_patch) + .bind(current_time) + .bind(instance_id) + .execute(self.conn()) + .await?; + Ok(result.rows_affected() > 0) } /// Remove a committee answer from an instance @@ -1070,16 +1210,23 @@ impl<'a> StorageProcessor<'a> { instance_id: &Uuid, committee: &str, ) -> anyhow::Result { - // First, get the current committees_answers - let current_instance = self.find_instance(instance_id).await?; - if current_instance.is_none() { - return Ok(false); - } - let mut committees_answers = current_instance.unwrap().committees_answers; - // Remove the committee answer - committees_answers.shift_remove(committee); - // Update the instance with the new committees_answers - self.update_instance_committees_answers_map(instance_id, &committees_answers).await + // JSON merge-patch removes object members with a null value, so this + // stays atomic with concurrent single-answer additions. + let committee_patch = + serde_json::json!({ (committee): serde_json::Value::Null }).to_string(); + let current_time = get_current_timestamp_secs(); + let result = sqlx::query( + "UPDATE instance \ + SET committees_answers = json_patch(COALESCE(committees_answers, '{}'), json(?)), \ + updated_at = ? \ + WHERE instance_id = ?", + ) + .bind(committee_patch) + .bind(current_time) + .bind(instance_id) + .execute(self.conn()) + .await?; + Ok(result.rows_affected() > 0) } /// Get committees answers for an instance @@ -1097,10 +1244,10 @@ impl<'a> StorageProcessor<'a> { } } - /// Update instance committees answers with HashMap (convenience method) + /// Replace the complete committee-answer map. /// - /// This is a convenience method that accepts a HashMap directly. - /// Internally converts it to arrays and calls the main update method. + /// Callers that add a single answer should use + /// `update_instance_committee_answer` instead, which merges atomically. pub async fn update_instance_committees_answers_map( &mut self, instance_id: &Uuid, @@ -1219,81 +1366,238 @@ impl<'a> StorageProcessor<'a> { Ok(result.rows_affected()) } - /// Insert or update a graph - /// - /// Performs an INSERT OR REPLACE operation on the graph table. - /// If a graph with the same graph_id exists, it will be updated. - /// If no graph exists, a new one will be created. + /// Insert a graph definition or verify a compatible replay. /// - /// Parameters: - /// - graph: The complete graph data to insert or update - /// - /// Returns: - /// - Ok(affected_rows) number of rows affected by the operation - /// - Err if the operation failed - pub async fn upsert_graph(&mut self, graph: &Graph) -> anyhow::Result { + /// The canonical definition hash is an identity fence: a new graph may + /// never reuse an existing graph id and inherit its runtime projection. + /// Compatible replays may only fill missing non-identity metadata; they + /// never replace status, observed transaction ids, or withdraw metadata. + pub async fn upsert_graph_definition(&mut self, graph: &Graph) -> anyhow::Result { + if graph.definition_hash.is_empty() { + anyhow::bail!("graph {} is missing its definition hash", graph.graph_id); + } + if graph.status != GraphStatus::OperatorPresigned.to_string() + || !graph.sub_status.is_empty() + || graph.challenge_txid.is_some() + || graph.init_withdraw_tx_hash.is_some() + || graph.bridge_out_start_at != 0 + || graph.proceed_withdraw_height != 0 + { + anyhow::bail!( + "graph {} definition writes must use the OperatorPresigned baseline without runtime data", + graph.graph_id + ); + } let verifier_assert_txids_json = serde_json::to_string(&graph.verifier_assert_txids)?; let disprove_txids_json = serde_json::to_string(&graph.disprove_txids)?; let watchtower_challenge_timeout_txids_json = serde_json::to_string(&graph.watchtower_challenge_timeout_txids)?; let operator_challenge_nack_txids_json = serde_json::to_string(&graph.operator_challenge_nack_txids)?; - let res = sqlx::query!( - "INSERT OR - REPLACE INTO graph (graph_id, instance_id, kickoff_index, from_addr, to_addr, amount, challenge_amount, - status, sub_status, operator_pubkey, cur_prekickoff_txid, next_prekickoff, force_skip_kickoff_txid, + let res = sqlx::query( + "INSERT INTO graph (graph_id, instance_id, kickoff_index, from_addr, to_addr, amount, challenge_amount, + status, sub_status, operator_pubkey, definition_hash, cur_prekickoff_txid, next_prekickoff, force_skip_kickoff_txid, quick_challenge_txid, challenge_incomplete_kickoff_txid, pegin_txid, kickoff_txid, take1_txid, challenge_txid, take2_txid, watchtower_challenge_init_txid, operator_assert_txid, verifier_assert_txids, disprove_txids, watchtower_challenge_timeout_txids, operator_challenge_nack_txids, operator_commit_timeout_txid, - init_withdraw_tx_hash, - bridge_out_start_at, status_updated_at, proceed_withdraw_height, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", - graph.graph_id, - graph.instance_id, - graph.kickoff_index, - graph.from_addr, - graph.to_addr, - graph.amount, - graph.challenge_amount, - graph.status, - graph.sub_status, - graph.operator_pubkey, - graph.cur_prekickoff_txid, - graph.next_prekickoff, - graph.force_skip_kickoff_txid, - graph.quick_challenge_txid, - graph.challenge_incomplete_kickoff_txid, - graph.pegin_txid, - graph.kickoff_txid, - graph.take1_txid, - graph.challenge_txid, - graph.take2_txid, - graph.watchtower_challenge_init_txid, - graph.operator_assert_txid, - verifier_assert_txids_json, - disprove_txids_json, - watchtower_challenge_timeout_txids_json, - operator_challenge_nack_txids_json, - graph.operator_commit_timeout_txid, - graph.init_withdraw_tx_hash, - graph.bridge_out_start_at, - graph.status_updated_at, - graph.proceed_withdraw_height, - graph.created_at, - graph.updated_at, - ) + init_withdraw_tx_hash, bridge_out_start_at, status_updated_at, proceed_withdraw_height, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(graph_id) DO UPDATE SET + from_addr = CASE + WHEN graph.from_addr = '' AND excluded.from_addr <> '' THEN excluded.from_addr + ELSE graph.from_addr + END, + to_addr = CASE + WHEN graph.to_addr = '' AND excluded.to_addr <> '' THEN excluded.to_addr + ELSE graph.to_addr + END + WHERE graph.definition_hash = excluded.definition_hash + AND graph.instance_id = excluded.instance_id", + ) + .bind(graph.graph_id) + .bind(graph.instance_id) + .bind(graph.kickoff_index) + .bind(&graph.from_addr) + .bind(&graph.to_addr) + .bind(graph.amount) + .bind(graph.challenge_amount) + .bind(GraphStatus::OperatorPresigned.to_string()) + .bind("") + .bind(&graph.operator_pubkey) + .bind(&graph.definition_hash) + .bind(graph.cur_prekickoff_txid.clone()) + .bind(graph.next_prekickoff.clone()) + .bind(graph.force_skip_kickoff_txid.clone()) + .bind(graph.quick_challenge_txid.clone()) + .bind(graph.challenge_incomplete_kickoff_txid.clone()) + .bind(graph.pegin_txid.clone()) + .bind(graph.kickoff_txid.clone()) + .bind(graph.take1_txid.clone()) + .bind(Option::::None) + .bind(graph.take2_txid.clone()) + .bind(graph.watchtower_challenge_init_txid.clone()) + .bind(graph.operator_assert_txid.clone()) + .bind(verifier_assert_txids_json) + .bind(disprove_txids_json) + .bind(watchtower_challenge_timeout_txids_json) + .bind(operator_challenge_nack_txids_json) + .bind(graph.operator_commit_timeout_txid.clone()) + .bind(Option::::None) + .bind(0_i64) + .bind(graph.status_updated_at) + .bind(0_i64) + .bind(graph.created_at) + .bind(graph.updated_at) .execute(self.conn()) .await?; + if res.rows_affected() == 0 { + let Some(existing) = self.find_graph(&graph.graph_id).await? else { + anyhow::bail!("graph {} disappeared while storing its definition", graph.graph_id); + }; + if existing.definition_hash != graph.definition_hash { + anyhow::bail!( + "conflicting graph definition for graph_id {}: existing={}, incoming={}", + graph.graph_id, + existing.definition_hash, + graph.definition_hash + ); + } + if existing.instance_id != graph.instance_id { + anyhow::bail!( + "graph definition instance mismatch for graph_id {}: existing={}, incoming={}", + graph.graph_id, + existing.instance_id, + graph.instance_id + ); + } + } Ok(res.rows_affected()) } - pub async fn update_graph(&mut self, params: &GraphUpdate) -> anyhow::Result<()> { + pub async fn update_graph_runtime( + &mut self, + params: &GraphRuntimeUpdate, + ) -> anyhow::Result { + if !params.has_updates() { + return Ok(false); + } let query_builder = params.get_query_builder("graph"); let update_sql = query_builder.get_sql(); let query = sqlx::query(&update_sql); let query = query_builder.query(query); - let _ = query.execute(self.conn()).await?; - Ok(()) + let result = query.execute(self.conn()).await?; + Ok(result.rows_affected() > 0) + } + + /// Atomically advance a graph status according to its evidence source. + /// + /// The conditional UPDATE is the authority check. The follow-up read only + /// distinguishes an idempotent replay from a stale event; it never decides + /// whether a write is permitted. + pub async fn transition_graph_status( + &mut self, + instance_id: Uuid, + graph_id: Uuid, + target: GraphStatus, + source: GraphStatusSource, + sub_status: Option, + ) -> anyhow::Result { + if !target.is_protocol_status() { + anyhow::bail!("frontend-only graph status {target} cannot be persisted"); + } + + let allowed_from = target.allowed_transition_from(source); + if allowed_from.is_empty() { + // Some verified scans merely observe a baseline state (for + // example, an operator-pre-signed graph). There is no authorized + // predecessor edge to write in that case; return the current + // projection without mutating it. + return self.graph_status_transition_outcome(instance_id, graph_id, target).await; + } + + let allowed_from: Vec = allowed_from.iter().map(ToString::to_string).collect(); + let current_time = get_current_timestamp_secs(); + let mut query_builder = QueryBuilder::update("graph"); + query_builder.set_field("status", QueryParam::Text(target.to_string())); + if let Some(sub_status) = sub_status.as_ref() { + query_builder.set_field("sub_status", QueryParam::Text(sub_status.clone())); + } + query_builder.set_field("status_updated_at", QueryParam::Int(current_time)); + query_builder.set_field("updated_at", QueryParam::Int(current_time)); + query_builder.and_where( + "hex(graph_id) = ? COLLATE NOCASE", + Some(QueryParam::Text(hex::encode(graph_id))), + ); + query_builder.and_where( + "hex(instance_id) = ? COLLATE NOCASE", + Some(QueryParam::Text(hex::encode(instance_id))), + ); + query_builder.and_where_in("status", &allowed_from, false); + + let update_sql = query_builder.get_sql(); + let query = query_builder.query(sqlx::query(&update_sql)); + if query.execute(self.conn()).await?.rows_affected() > 0 { + return Ok(GraphStatusTransitionOutcome::Applied); + } + + // A zero-row conditional update has two distinct meanings: this can + // be an idempotent replay, or another writer may have moved the row to + // a state which rejects this transition. The follow-up read reports + // that distinction; it never authorizes a write. + let outcome = self.graph_status_transition_outcome(instance_id, graph_id, target).await?; + if !matches!(outcome, GraphStatusTransitionOutcome::AlreadyCurrent) { + return Ok(outcome); + } + + let Some(sub_status) = sub_status else { + return Ok(GraphStatusTransitionOutcome::AlreadyCurrent); + }; + + let current_time = get_current_timestamp_secs(); + let result = sqlx::query( + "UPDATE graph SET sub_status = ?, updated_at = ? \ + WHERE graph_id = ? AND instance_id = ? AND status = ?", + ) + .bind(sub_status) + .bind(current_time) + .bind(graph_id) + .bind(instance_id) + .bind(target.to_string()) + .execute(self.conn()) + .await?; + if result.rows_affected() > 0 { + return Ok(GraphStatusTransitionOutcome::AlreadyCurrent); + } + + // The row may have changed between the outcome read and the optional + // sub-status update. Re-read so a concurrent transition is never + // misreported as an idempotent replay or a missing graph. + self.graph_status_transition_outcome(instance_id, graph_id, target).await + } + + async fn graph_status_transition_outcome( + &mut self, + instance_id: Uuid, + graph_id: Uuid, + target: GraphStatus, + ) -> anyhow::Result { + let Some(current_graph) = self.find_graph(&graph_id).await? else { + return Ok(GraphStatusTransitionOutcome::NotFound); + }; + if current_graph.instance_id != instance_id { + return Ok(GraphStatusTransitionOutcome::NotFound); + } + let current = GraphStatus::from_str(¤t_graph.status).map_err(|_| { + anyhow::anyhow!( + "graph {graph_id} has invalid persisted status {}", + current_graph.status + ) + })?; + Ok(if current == target { + GraphStatusTransitionOutcome::AlreadyCurrent + } else { + GraphStatusTransitionOutcome::Rejected { current } + }) } pub async fn find_graph(&mut self, graph_id: &Uuid) -> anyhow::Result> { @@ -1342,6 +1646,7 @@ impl<'a> StorageProcessor<'a> { status, sub_status, operator_pubkey, + definition_hash, cur_prekickoff_txid, next_prekickoff, force_skip_kickoff_txid, @@ -1529,47 +1834,6 @@ impl<'a> StorageProcessor<'a> { Ok(()) } - pub async fn update_graphs_status_with_instance_id( - &mut self, - instance_id: Uuid, - ignore_graph_id: Option, - status: &str, - ) -> anyhow::Result<()> { - let current_time = get_current_timestamp_secs(); - if let Some(ignore_graph_id) = ignore_graph_id { - sqlx::query!( - "UPDATE graph - SET status = ?, - status_updated_at = ?, - updated_at = ? - WHERE instance_id = ? - AND graph_id != ?", - status, - current_time, - current_time, - instance_id, - ignore_graph_id - ) - .execute(self.conn()) - .await?; - } else { - sqlx::query!( - "UPDATE graph - SET status = ?, - status_updated_at = ?, - updated_at = ? - WHERE instance_id = ?", - status, - current_time, - current_time, - instance_id, - ) - .execute(self.conn()) - .await?; - } - Ok(()) - } - pub async fn find_graph_neighbor_ids( &mut self, graph_id: Uuid, @@ -1796,14 +2060,17 @@ impl<'a> StorageProcessor<'a> { state: String, ) -> anyhow::Result { let current_time = get_current_timestamp_secs(); - let res = sqlx::query!( - "Update message Set state = ?, updated_at = ? WHERE message_id = ? AND message_version = ?", - state, - current_time, - message_id, - message_version - - ).execute(self.conn()).await?; + let res = sqlx::query( + "UPDATE message \ + SET state = ?, updated_at = ? \ + WHERE message_id = ? AND message_version = ? AND state != 'Cancelled'", + ) + .bind(state) + .bind(current_time) + .bind(message_id) + .bind(message_version) + .execute(self.conn()) + .await?; Ok(res.rows_affected() > 0) } @@ -1818,23 +2085,31 @@ impl<'a> StorageProcessor<'a> { let current_time = get_current_timestamp_secs(); let res = match msg_type { Some(msg_type) => { - sqlx::query!( - "Update message Set state = ?, updated_at = ? WHERE business_id = ? AND msg_type = ? AND state = ?", - new_state, - current_time, - business_id, - msg_type, - old_state - ).execute(self.conn()).await? + sqlx::query( + "UPDATE message \ + SET state = ?, updated_at = ? \ + WHERE business_id = ? AND msg_type = ? AND state = ? AND state != 'Cancelled'", + ) + .bind(new_state) + .bind(current_time) + .bind(business_id) + .bind(msg_type) + .bind(old_state) + .execute(self.conn()) + .await? } None => { - sqlx::query!( - "Update message Set state = ?, updated_at = ? WHERE business_id = ? AND state = ?", - new_state, - current_time, - business_id, - old_state - ).execute(self.conn()).await? + sqlx::query( + "UPDATE message \ + SET state = ?, updated_at = ? \ + WHERE business_id = ? AND state = ? AND state != 'Cancelled'", + ) + .bind(new_state) + .bind(current_time) + .bind(business_id) + .bind(old_state) + .execute(self.conn()) + .await? } }; Ok(res.rows_affected() > 0) @@ -1884,10 +2159,9 @@ impl<'a> StorageProcessor<'a> { business_id: &Uuid, msg_type: &str, ) -> anyhow::Result> { - let res = sqlx::query_as!( - Message, + let row = sqlx::query( "SELECT message_id, - business_id AS \"business_id:Uuid\", + business_id, from_peer, actor, msg_type, @@ -1895,24 +2169,24 @@ impl<'a> StorageProcessor<'a> { message_version, state, weight, - lock_time_until + lock_time_until, + created_at FROM message WHERE business_id = ? AND msg_type = ?", - business_id, - msg_type, ) + .bind(business_id) + .bind(msg_type) .fetch_optional(self.conn()) .await?; - Ok(res) + Ok(row.map(|row| message_from_row(&row)).transpose()?) } pub async fn find_messages_by_id( &mut self, message_id: &str, ) -> anyhow::Result> { - let res = sqlx::query_as!( - Message, + let row = sqlx::query( "SELECT message_id, - business_id AS \"business_id:Uuid\", + business_id, from_peer, actor, msg_type, @@ -1920,14 +2194,15 @@ impl<'a> StorageProcessor<'a> { message_version, state, weight, - lock_time_until + lock_time_until, + created_at FROM message WHERE message_id = ?", - message_id, ) + .bind(message_id) .fetch_optional(self.conn()) .await?; - Ok(res) + Ok(row.map(|row| message_from_row(&row)).transpose()?) } pub async fn filter_messages( @@ -1939,10 +2214,9 @@ impl<'a> StorageProcessor<'a> { limit: i64, offset: i64, ) -> anyhow::Result> { - let res = sqlx::query_as!( - Message, + let rows = sqlx::query( "SELECT message_id, - business_id AS \"business_id:Uuid\", + business_id, from_peer, actor, msg_type, @@ -1950,7 +2224,8 @@ impl<'a> StorageProcessor<'a> { message_version, state, weight, - lock_time_until + lock_time_until, + created_at FROM message WHERE state = ? AND weight >= ? @@ -1958,21 +2233,51 @@ impl<'a> StorageProcessor<'a> { AND updated_at >= ? ORDER BY created_at ASC LIMIT ? OFFSET ?", - state, - weight, - lock_time_until, - expired, - limit, - offset ) + .bind(state) + .bind(weight) + .bind(lock_time_until) + .bind(expired) + .bind(limit) + .bind(offset) .fetch_all(self.conn()) .await?; - Ok(res) + rows.into_iter() + .map(|row| message_from_row(&row)) + .collect::, _>>() + .map_err(Into::into) + } + + pub async fn get_message_queue_stats( + &mut self, + actor: &str, + now: i64, + ) -> anyhow::Result { + let row = sqlx::query( + r#"SELECT + COALESCE(SUM(CASE WHEN state = 'Pending' AND lock_time_until <= ? THEN 1 ELSE 0 END), 0) AS pending_ready, + COALESCE(SUM(CASE WHEN state = 'Pending' AND lock_time_until > ? THEN 1 ELSE 0 END), 0) AS pending_locked, + COALESCE(SUM(CASE WHEN state = 'Failed' THEN 1 ELSE 0 END), 0) AS failed, + MIN(CASE WHEN state = 'Pending' THEN created_at END) AS oldest_pending_at + FROM message + WHERE actor = ?"#, + ) + .bind(now) + .bind(now) + .bind(actor) + .fetch_one(self.conn()) + .await?; + Ok(MessageQueueStats { + pending_ready: row.try_get("pending_ready")?, + pending_locked: row.try_get("pending_locked")?, + failed: row.try_get("failed")?, + oldest_pending_at: row.try_get("oldest_pending_at")?, + }) } pub async fn upsert_message(&mut self, msg: Message) -> anyhow::Result { let current_time = get_current_timestamp_secs(); - let res = sqlx::query!( + let res = sqlx::query( r#"INSERT INTO message (message_id, business_id, from_peer, actor, msg_type, content, state, message_version, lock_time_until, weight, updated_at, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(message_id) DO UPDATE SET business_id = excluded.business_id, @@ -1984,26 +2289,65 @@ impl<'a> StorageProcessor<'a> { message_version = message.message_version + 1, lock_time_until = excluded.lock_time_until, weight = excluded.weight, - updated_at = excluded.updated_at"#, - msg.message_id, - msg.business_id, - msg.from_peer, - msg.actor, - msg.msg_type, - msg.content, - msg.state, - msg.message_version, - msg.lock_time_until, - msg.weight, - current_time, - current_time - - ) - .execute(self.conn()) + updated_at = excluded.updated_at + WHERE message.state != 'Cancelled'"#, + ) + .bind(msg.message_id) + .bind(msg.business_id) + .bind(msg.from_peer) + .bind(msg.actor) + .bind(msg.msg_type) + .bind(msg.content) + .bind(msg.state) + .bind(msg.message_version) + .bind(msg.lock_time_until) + .bind(msg.weight) + .bind(current_time) + .bind(current_time) + .execute(self.conn()) .await?; Ok(res.rows_affected() > 0) } + /// Record that a chain-derived graph message has been durably enqueued. + /// + /// Queue rows are intentionally pruned after their retention period, but + /// replaying an old SyncGraph must not recreate already-completed protocol + /// actions. The caller inserts this marker and its queue row in one + /// transaction. + pub async fn insert_graph_compensation_marker( + &mut self, + graph_id: Uuid, + message_id: &str, + ) -> anyhow::Result { + let result = sqlx::query( + "INSERT INTO graph_compensation_marker (graph_id, message_id, created_at) \ + VALUES (?, ?, ?) ON CONFLICT(graph_id, message_id) DO NOTHING", + ) + .bind(graph_id) + .bind(message_id) + .bind(get_current_timestamp_secs()) + .execute(self.conn()) + .await?; + Ok(result.rows_affected() > 0) + } + + pub async fn has_graph_compensation_marker( + &mut self, + graph_id: Uuid, + message_id: &str, + ) -> anyhow::Result { + let exists: i64 = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM graph_compensation_marker \ + WHERE graph_id = ? AND message_id = ?)", + ) + .bind(graph_id) + .bind(message_id) + .fetch_one(self.conn()) + .await?; + Ok(exists != 0) + } + pub async fn upsert_pegin_instance_process_data( &mut self, pegin_instance_process_data: &PeginInstanceProcessData, @@ -2128,18 +2472,48 @@ impl<'a> StorageProcessor<'a> { pub async fn upsert_graph_raw_data( &mut self, + instance_id: Uuid, graph_raw_data: GraphRawData, + definition_hash: &str, ) -> anyhow::Result { - let result = sqlx::query!( - r#" - INSERT OR REPLACE INTO graph_raw_data (graph_id, raw_data, created_at, updated_at) - VALUES (?, ?, ?, ?) - "#, - graph_raw_data.graph_id, - graph_raw_data.raw_data, - graph_raw_data.created_at, - graph_raw_data.updated_at + if definition_hash.is_empty() { + anyhow::bail!( + "graph {} raw definition is missing its canonical definition hash", + graph_raw_data.graph_id + ); + } + let Some(graph) = self.find_graph(&graph_raw_data.graph_id).await? else { + anyhow::bail!( + "cannot store raw definition for missing graph {}", + graph_raw_data.graph_id + ); + }; + if graph.instance_id != instance_id { + anyhow::bail!( + "raw definition instance does not match graph {}: stored={}, incoming={instance_id}", + graph_raw_data.graph_id, + graph.instance_id + ); + } + if graph.definition_hash != definition_hash { + anyhow::bail!( + "raw definition hash does not match graph {}: stored={}, incoming={definition_hash}", + graph_raw_data.graph_id, + graph.definition_hash + ); + } + + let result = sqlx::query( + "INSERT INTO graph_raw_data (graph_id, raw_data, created_at, updated_at) + VALUES (?, ?, ?, ?) + ON CONFLICT(graph_id) DO UPDATE SET + raw_data = excluded.raw_data, + updated_at = excluded.updated_at", ) + .bind(graph_raw_data.graph_id) + .bind(graph_raw_data.raw_data) + .bind(graph_raw_data.created_at) + .bind(graph_raw_data.updated_at) .execute(self.conn()) .await?; @@ -2166,37 +2540,6 @@ impl<'a> StorageProcessor<'a> { Ok(row) } - pub async fn update_graph_raw_data( - &mut self, - graph_id: &Uuid, - raw_data: &str, - ) -> anyhow::Result { - let timestamp = get_current_timestamp_millis(); - - let result = sqlx::query!( - r#" - UPDATE graph_raw_data - SET raw_data = ?, updated_at = ? - WHERE graph_id = ? - "#, - raw_data, - timestamp, - graph_id - ) - .execute(self.conn()) - .await?; - - Ok(result.rows_affected()) - } - - pub async fn delete_graph_raw_data(&mut self, graph_id: &str) -> anyhow::Result { - let result = sqlx::query!(r#"DELETE FROM graph_raw_data WHERE graph_id = ?"#, graph_id) - .execute(self.conn()) - .await?; - - Ok(result.rows_affected()) - } - pub async fn find_watch_contract( &mut self, addr: &str, @@ -3147,7 +3490,7 @@ pub async fn create_local_db(db_path: &str) -> LocalDB { } #[cfg(test)] -mod sequencer_set_tests { +mod tests { use super::*; async fn setup_db() -> LocalDB { diff --git a/crates/store/src/schema.rs b/crates/store/src/schema.rs index f87f671b8..baf08ef75 100644 --- a/crates/store/src/schema.rs +++ b/crates/store/src/schema.rs @@ -199,8 +199,8 @@ pub enum InstanceBridgeInStatus { // committee won't answer if userRequest is invalid(e.g. insufficient fee) CommitteesAnswered, // enough committee responsed & window expired UserBroadcastPeginPrepare, // user pegin prepare - Presigned, // all committee signed PeginConfirm - PresignedFailed, // includes operator and Committee presigns + Presigned, // enough graphs completed committee pre-signing + PresignedFailed, // required graph pre-signing did not finish in time RelayerL1Broadcasted, // PeginConfirm broadcast by relayer RelayerL2Minted, // success RelayerL2MintedFailed, @@ -300,6 +300,26 @@ pub enum GraphStatus { Disproving, } +/// The evidence that authorizes a graph status transition. +/// +/// Graph definitions, finalized Goat events, and chain reconciliation have +/// different authority. Keeping the source explicit prevents a generic +/// database update from accidentally becoming a state-transition bypass. +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum GraphStatusSource { + Definition, + GoatEvent, + ChainReconcile, +} + +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum GraphStatusTransitionOutcome { + Applied, + AlreadyCurrent, + Rejected { current: GraphStatus }, + NotFound, +} + impl GraphStatus { pub fn get_closed_status() -> Vec { vec![ @@ -329,47 +349,97 @@ impl GraphStatus { pub fn is_obsoleted(&self) -> bool { self.eq(&GraphStatus::Obsoleted) } - pub fn get_previous_status(&self) -> Option { - match self { - GraphStatus::OperatorPresigned => None, - GraphStatus::CommitteePresigned => Some(GraphStatus::OperatorPresigned), - GraphStatus::OperatorDataPushed => Some(GraphStatus::CommitteePresigned), - GraphStatus::PreKickoff => Some(GraphStatus::OperatorDataPushed), - GraphStatus::Skipped => Some(GraphStatus::OperatorDataPushed), - GraphStatus::Obsoleted => Some(GraphStatus::OperatorDataPushed), - GraphStatus::OperatorKickOff => Some(GraphStatus::PreKickoff), - GraphStatus::OperatorTake1 => Some(GraphStatus::OperatorKickOff), - GraphStatus::Challenge => Some(GraphStatus::OperatorKickOff), - GraphStatus::Disprove => Some(GraphStatus::Challenge), - GraphStatus::OperatorTake2 => Some(GraphStatus::Challenge), - // frontend use only - GraphStatus::Created => None, - GraphStatus::Presigned => Some(GraphStatus::Created), - GraphStatus::L2Recorded => Some(GraphStatus::Presigned), - GraphStatus::OperatorKickOffing => Some(GraphStatus::L2Recorded), - GraphStatus::Challenging => Some(GraphStatus::OperatorKickOffing), - GraphStatus::Disproving => Some(GraphStatus::Challenging), - } - } - pub fn is_before(&self, other: &GraphStatus) -> bool { - let mut current = *other; - while let Some(prev) = current.get_previous_status() { - if &prev == self { - return true; - } - current = prev; - } - false - } - pub fn is_after(&self, other: &GraphStatus) -> bool { - let mut current = *self; - while let Some(prev) = current.get_previous_status() { - if &prev == other { - return true; - } - current = prev; + + /// Whether this is a backend protocol state rather than a frontend-only + /// display alias. + pub fn is_protocol_status(self) -> bool { + matches!( + self, + Self::OperatorPresigned + | Self::CommitteePresigned + | Self::OperatorDataPushed + | Self::PreKickoff + | Self::OperatorKickOff + | Self::Challenge + | Self::Disprove + | Self::Obsoleted + | Self::Skipped + | Self::OperatorTake1 + | Self::OperatorTake2 + ) + } + + /// States from which `self` may be reached with the supplied evidence. + /// + /// These sets intentionally contain transitive predecessors. A chain scan + /// can observe several confirmed transactions in one pass, so requiring a + /// locally persisted intermediate state would make recovery after restart + /// impossible. Closed states are deliberately absent from all sets. + pub fn allowed_transition_from(self, source: GraphStatusSource) -> &'static [GraphStatus] { + use GraphStatus::*; + + const EARLY: &[GraphStatus] = &[OperatorPresigned, CommitteePresigned]; + const EARLY_OR_OBSOLETED: &[GraphStatus] = + &[OperatorPresigned, CommitteePresigned, Obsoleted]; + const PRE_KICKOFF: &[GraphStatus] = + &[OperatorPresigned, CommitteePresigned, OperatorDataPushed, Obsoleted]; + const KICKOFF: &[GraphStatus] = + &[OperatorPresigned, CommitteePresigned, OperatorDataPushed, PreKickoff, Obsoleted]; + const CHALLENGE: &[GraphStatus] = &[ + OperatorPresigned, + CommitteePresigned, + OperatorDataPushed, + PreKickoff, + Obsoleted, + OperatorKickOff, + ]; + const TAKE1: &[GraphStatus] = &[ + OperatorPresigned, + CommitteePresigned, + OperatorDataPushed, + PreKickoff, + Obsoleted, + OperatorKickOff, + ]; + const TAKE2_OR_DISPROVE: &[GraphStatus] = &[ + OperatorPresigned, + CommitteePresigned, + OperatorDataPushed, + PreKickoff, + Obsoleted, + OperatorKickOff, + Challenge, + ]; + const SKIPPED: &[GraphStatus] = + &[OperatorPresigned, CommitteePresigned, OperatorDataPushed, PreKickoff, Obsoleted]; + + match source { + GraphStatusSource::Definition => match self { + CommitteePresigned => &[OperatorPresigned], + _ => &[], + }, + GraphStatusSource::GoatEvent => match self { + OperatorDataPushed => EARLY, + OperatorTake1 => TAKE1, + OperatorTake2 | Disprove => TAKE2_OR_DISPROVE, + _ => &[], + }, + GraphStatusSource::ChainReconcile => match self { + CommitteePresigned => &[OperatorPresigned], + // Obsoleted is a provisional branch when GraphData was not + // yet visible. A later full chain scan may correct it, but + // ordinary Goat event replay may not. + OperatorDataPushed => EARLY_OR_OBSOLETED, + PreKickoff | Obsoleted => PRE_KICKOFF, + OperatorKickOff => KICKOFF, + Challenge => CHALLENGE, + OperatorTake1 => TAKE1, + OperatorTake2 | Disprove => TAKE2_OR_DISPROVE, + Skipped => SKIPPED, + OperatorPresigned | Created | Presigned | L2Recorded | OperatorKickOffing + | Challenging | Disproving => &[], + }, } - false } } @@ -386,6 +456,9 @@ pub struct Graph { pub status: String, // GraphStatus pub sub_status: String, // GraphStatus pub operator_pubkey: String, + /// Canonical hash of the immutable graph parameters. This binds the + /// runtime projection to exactly one transaction-graph definition. + pub definition_hash: String, pub next_prekickoff: Option, pub cur_prekickoff_txid: Option, pub force_skip_kickoff_txid: Option, @@ -448,6 +521,7 @@ pub struct Message { pub message_version: i64, pub weight: i64, pub lock_time_until: i64, + pub created_at: i64, } #[derive(Clone, FromRow, Debug, Serialize, Deserialize, Default)] diff --git a/node/Cargo.toml b/node/Cargo.toml index f805d25a1..38ad959ad 100644 --- a/node/Cargo.toml +++ b/node/Cargo.toml @@ -15,6 +15,10 @@ path = "src/bin/send_pegin_request.rs" name = "challenge" path = "src/bin/send_challenge.rs" +[[bin]] +name = "verifier-challenge" +path = "src/bin/send_verifier_challenge.rs" + [[bin]] name = "sequencer-set-publish" path = "src/bin/sequencer-set-publish.rs" @@ -93,7 +97,7 @@ prometheus-client = { workspace = true } esplora-client = { workspace = true } tracing = { workspace = true } -tracing-subscriber = { workspace = true, features = ["env-filter"] } +tracing-subscriber = { workspace = true, features = ["env-filter", "json"] } thiserror = "2.0" bitcoin = { workspace = true, features = ["serde"] } diff --git a/node/README.md b/node/README.md index 8bf705ead..4c897dc36 100644 --- a/node/README.md +++ b/node/README.md @@ -412,20 +412,27 @@ stateDiagram-v2 PreKickoff --> OperatorKickOff: Kickoff tx broadcast PreKickoff --> Skipped: Guardian intervention or force skip + Obsoleted --> OperatorKickOff: Kickoff tx observed on-chain after all + Obsoleted --> Skipped: Guardian intervention or force skip + OperatorKickOff --> OperatorTake1: Timelock expired without challenge OperatorKickOff --> Challenge: WatchtowerChallengeInit confirmed + OperatorKickOff --> Disprove: Guardian disprove detected before any Challenge opened - Challenge --> OperatorTake2: All challenge phases completed normally - Challenge --> Disprove: WT-Ack/BlockHash-Commit/Assert Timeout or Fraud scripts detected + Challenge --> OperatorTake2: Take2 tx confirmed and no disprove detected + Challenge --> Disprove: Guardian disprove, watchtower-flow disprove, verifier disprove, or unrecognized connector-D spend OperatorTake1 --> [*]: Happy Path complete OperatorTake2 --> [*]: Challenge Path complete Skipped --> [*]: Graph skipped - Obsoleted --> [*]: Graph obsoleted Disprove --> [*]: Dispute resolved ``` -**Description**: Graph goes through 9 main states. The normal flow starts from `OperatorPresigned`, proceeds through signature collection, data publishing, and Kickoff, finally completing via Take1 (Happy Path) or Take2 (Challenge Path). Abnormal cases enter `Obsoleted`, `Skipped`, or `Disprove` terminal states. +**Description**: Graph goes through 9 main states. The normal flow starts from `OperatorPresigned`, proceeds through signature collection, data publishing, and Kickoff, finally completing via Take1 (Happy Path) or Take2 (Challenge Path). `Skipped` and `Disprove` are terminal. + +**`Obsoleted` is not terminal.** It is a provisional status: the local node marks a graph `Obsoleted` when it observes the PreKickoff tx on-chain before graph data was posted (or the pegin becomes non-withdrawable), but if it later observes the actual kickoff tx confirm, the graph resumes into `OperatorKickOff` exactly as if it had come from `PreKickoff` — see `scan_graph_chain_state`, `node/src/utils.rs:1418-1434`, which checks `matches!(current_status, GraphStatus::PreKickoff | GraphStatus::Obsoleted)` for both the kickoff and force-skip cases. + +`OperatorKickOff --> Disprove` is a direct edge, distinct from the `Challenge --> Disprove` edge below: a guardian can detect and prove operator misbehavior (`detect_guardian_disprove`, `node/src/utils.rs:1237-1262`) before a watchtower ever opens a `Challenge` at all (`node/src/utils.rs:1448-1461`). ### Fig-04-2-Graph-Status-Transitions @@ -438,53 +445,44 @@ stateDiagram-v2 | `OperatorDataPushed` | `Obsoleted` | Pegin not withdrawable and no withdrawal request | | `PreKickoff` | `OperatorKickOff` | Kickoff tx broadcast successfully | | `PreKickoff` | `Skipped` | Guardian intervention or force skip | -| `OperatorKickOff` | `OperatorTake1` | Timelock expired without challenge | -| `OperatorKickOff` | `Challenge` | WatchtowerChallengeInit tx confirmed | -| `Challenge` | `OperatorTake2` | All sub-phases completed normally | -| `Challenge` | `Disprove` | Timeout or challenge failure detected | +| `Obsoleted` | `OperatorKickOff` | Kickoff tx broadcast successfully (Obsoleted is provisional, not terminal) | +| `Obsoleted` | `Skipped` | Guardian intervention or force skip | +| `OperatorKickOff` | `OperatorTake1` | Connector-A spent by the take1 tx | +| `OperatorKickOff` | `Challenge` | Connector-A spent by anything other than the take1 tx | +| `OperatorKickOff` | `Disprove` | Guardian disprove detected (bypasses `Challenge`) | +| `Challenge` | `OperatorTake2` | Take2 tx observed on-chain and no disprove detector fired | +| `Challenge` | `Disprove` | Any of: guardian disprove, watchtower-flow disprove (operator-challenge-nack or operator-commit-timeout tx), a verifier's assert answered by its matching disprove tx, or an unrecognized connector-D spend | -### Fig-04-3-Challenge-SubStatus-ER +Source: `scan_graph_chain_state`, `node/src/utils.rs:1328-1681` (the sole writer of these edges; verified by full read, 2026-07-17). -```mermaid -erDiagram - ChallengeSubStatus { - WatchtowerChallengeStatus watchtower_challenge_status - CommitBlockHashStatus commit_blockhash_status - AssertCommitStatus assert_commit_status - DisproveTxType disprove_type - int disprove_index - } +### Fig-04-3-Challenge-SubStatus - WatchtowerChallengeStatus { - enum None - enum OperatorInit - enum WatchtowerChallenge - enum WatchtowerChallengeTimeout - enum OperatorACKTimeout - enum WatchtowerChallengeNormalFinished - enum WatchtowerChallengeDisproveFinished - } +`ChallengeSubStatus` is **not** three parallel single-value status enums as earlier revisions of this document claimed — that shape does not exist in the code. The actual struct (`node/src/scheduled_tasks/graph_maintenance_tasks.rs:52-58`) is: - CommitBlockHashStatus { - enum None - enum WatchtowerChallengeProcessed - enum OperatorCommit - enum OperatorCommitTimeout - } +```rust +pub struct ChallengeSubStatus { + pub watchtower_challenge_status: Vec, // per-watchtower: true once that watchtower's challenge connector is spent + pub verifier_challenge_status: Vec, // per-verifier progress + pub disprove_type: Option, + pub disprove_index: i32, +} - AssertCommitStatus { - enum None - enum OperatorInit - enum OperatorCommit - enum OperatorCommitTimeout - } +pub enum VerifierChallengeStatus { None, VerifierAsserted, ProverAnswered, Disproved } - ChallengeSubStatus ||--|| WatchtowerChallengeStatus : contains - ChallengeSubStatus ||--|| CommitBlockHashStatus : contains - ChallengeSubStatus ||--|| AssertCommitStatus : contains +pub enum DisproveTxType { Disprove, QuickChallenge, ChallengeIncompleteKickoff, PubinDisprove, OperatorChallengeNack, OperatorCommitTimeout } ``` -**Description**: `ChallengeSubStatus` aggregates three parallel status trackers. Only when `watchtower_challenge_status == NormalFinished`, `commit_blockhash_status == OperatorCommit`, and `assert_commit_status == OperatorCommit` can the system transition to `OperatorTake2` state. +`watchtower_challenge_status` and `verifier_challenge_status` are **observational bookkeeping only** — they are recorded for the frontend/API (`node/src/rpc_service/bitvm.rs:672-693`, collapsed there into a simpler `SimpleChallengeSubStatus{None, WatchtowerChallenge, Assert}` view) but are never read by anything that decides `GraphStatus`. In particular, `ChallengeSubStatus::is_watchtower_challenge_success` (`graph_maintenance_tasks.rs:98-105`) has no call sites anywhere in the repo. The real `Challenge --> OperatorTake2` gate is purely `tx_on_chain(take2_txid)` after none of the four disprove detectors fired (`node/src/utils.rs:1661-1670`); the operator's actual authorization to broadcast take2 is enforced by Bitcoin-script timelocks (`detect_take2`, `graph_maintenance_tasks.rs:1089-1233`), not by any application-level quorum over watchtower or verifier counts. + +### Known gap: no ordering guarantee between chain-scan and L2-event writers + +`GraphStatus` is written from two independently-scheduled places with no inherent coordination between them: +- `scan_graph_chain_state` above (Bitcoin-poll derived), and +- the GoatChain L2-event watcher, `node/src/scheduled_tasks/event_watch_task.rs:392-502`, which writes `OperatorDataPushed`/`OperatorTake1`/`OperatorTake2`/`Disprove` directly via `StorageProcessor::update_graph` with no precondition on the graph's current status. + +Both currently go through a raw `UPDATE graph SET status = ?` (`crates/store/src/localdb.rs`) with no guard preventing a closed/terminal status (`GraphStatus::is_closed()`: `OperatorTake1`, `OperatorTake2`, `Skipped`, `Disprove`) from being overwritten by a later, differently-ordered write from the other subsystem — e.g. a `Disprove` status can be silently reverted by a stale `PostGraphDataEvent` replay. **This is an open bug, not yet fixed in code.** + +`node/tla/GraphLifecycle.tla` / `GraphLifecycle.cfg` has the machine-checked counterexample (`OperatorTake1 -> OperatorTake2` in 2 steps). A verified fix design exists and is formally proven correct — `GraphLifecycleFixed.cfg` (atomic guard: fold the check into the `UPDATE`'s `WHERE status IN (...)` clause rather than a separate read-then-decide-then-write) passes all safety and liveness properties, and `GraphLifecycleFineGrainedFixed.tla` additionally proves that atomicity is necessary (a naive read-then-write version of the same guard, modeled in `GraphLifecycleFineGrained.tla`, still fails). Applying that design to `node/src/utils.rs`/`crates/store/src/localdb.rs` is tracked as follow-up work, not part of this audit pass. --- diff --git a/node/src/action.rs b/node/src/action.rs index 4aae08fc1..bf0c13ee0 100644 --- a/node/src/action.rs +++ b/node/src/action.rs @@ -8,7 +8,7 @@ use crate::middleware::AllBehaviours; use crate::rpc_service::current_time_secs; use crate::utils::*; use alloy::primitives::Address as EvmAddress; -use anyhow::{Context, Result, anyhow}; +use anyhow::{Context, Result, anyhow, bail}; use bitcoin::{PublicKey, Txid}; use bitvm_lib::actors::Actor; use bitvm_lib::babe_adapter::{BabeBundleBuilder, CACSetupPackage}; @@ -23,9 +23,9 @@ use musig2::{PartialSignature, PubNonce}; use secp256k1::schnorr::Signature as SchnorrSignature; use serde::{Deserialize, Serialize}; use std::sync::Arc; +use std::time::Instant; use store::MessageState; use store::localdb::LocalDB; -use tracing::warn; use uuid::Uuid; #[derive(Serialize, Deserialize, Clone)] @@ -80,6 +80,55 @@ pub enum GOATMessageContent { Tick, } +impl GOATMessageContent { + /// Stable message name for logs/metrics. Keep this independent of `Debug`, whose + /// output can include protocol payloads (and, for proofs, be very large). + pub fn event_type(&self) -> &'static str { + match self { + Self::PeginRequest(_) => "PeginRequest", + Self::CreateGraph(_) => "CreateGraph", + Self::ConfirmInstance(_) => "ConfirmInstance", + Self::InitGraph(_) => "InitGraph", + Self::GenCircuits(_) => "GenCircuits", + Self::CutCircuits(_) => "CutCircuits", + Self::SolderingProofReady(_) => "SolderingProofReady", + Self::VerifierGraphParamsEndorsement(_) => "VerifierGraphParamsEndorsement", + Self::NonceGeneration(_) => "NonceGeneration", + Self::CommitteePresign(_) => "CommitteePresign", + Self::EndorseGraph(_) => "EndorseGraph", + Self::GraphFinalize(_) => "GraphFinalize", + Self::PeginConfirmNonce(_) => "PeginConfirmNonce", + Self::PeginConfirmPartialSig(_) => "PeginConfirmPartialSig", + Self::PostReady(_) => "PostReady", + Self::KickoffReady(_) => "KickoffReady", + Self::KickoffSent(_) => "KickoffSent", + Self::PreKickoffSent(_) => "PreKickoffSent", + Self::ChallengeSent(_) => "ChallengeSent", + Self::WatchtowerChallengeInitSent(_) => "WatchtowerChallengeInitSent", + Self::WatchtowerChallengeSent(_) => "WatchtowerChallengeSent", + Self::WatchtowerChallengeTimeout(_) => "WatchtowerChallengeTimeout", + Self::NackReady(_) => "NackReady", + Self::OperatorCommitPubinReady(_) => "OperatorCommitPubinReady", + Self::OperatorCommitPubinTimeout(_) => "OperatorCommitPubinTimeout", + Self::AssertReady(_) => "AssertReady", + Self::AssertSent(_) => "AssertSent", + Self::ChallengeAssertSent(_) => "ChallengeAssertSent", + Self::WronglyChallengeTimeout(_) => "WronglyChallengeTimeout", + Self::DisproveSent(_) => "DisproveSent", + Self::Take1Ready(_) => "Take1Ready", + Self::Take1Sent(_) => "Take1Sent", + Self::Take2Ready(_) => "Take2Ready", + Self::Take2Sent(_) => "Take2Sent", + Self::RequestNodeInfo(_) => "RequestNodeInfo", + Self::ResponseNodeInfo(_) => "ResponseNodeInfo", + Self::SyncGraphRequest(_) => "SyncGraphRequest", + Self::SyncGraph(_) => "SyncGraph", + Self::InstanceDiscarded(_) => "InstanceDiscarded", + Self::Tick => "Tick", + } + } +} + /// Pegin #[derive(Serialize, Deserialize, Clone)] @@ -389,20 +438,37 @@ pub async fn handle_self_p2p_msg( message: &[u8], ) -> Result<()> { if id != GOATMessage::default_message_id() { - tracing::warn!("handle_self_p2p_msg received unexpected message id: {:?}", id); + tracing::warn!( + event = "local_message_queue", + outcome = "unexpected_message_id", + message_id = ?id, + "ignoring local queue trigger with an unexpected message id" + ); return Ok(()); } let message = GOATMessage::deserialize_message(message).await?; tracing::info!( - "Got self p2p message: {} with id: {} from peer: {:?}", - &message.actor.to_string(), - id, - from_peer_id + event = "local_message_queue", + outcome = "trigger_received", + role = %message.actor, + message_type = message.content.event_type(), + message_id = ?id, + from_peer_id = %from_peer_id, + "received local queue trigger" ); let messages = pop_batch_local_unhandle_msg(local_db, actor.clone(), current_time_secs(), 0, 50).await?; + tracing::info!( + event = "local_message_queue", + outcome = "batch_loaded", + role = %actor, + batch_size = messages.len(), + "loaded pending local messages" + ); for message in messages { + let queue_wait_secs = current_time_secs().saturating_sub(message.created_at); + let started_at = Instant::now(); match recv_and_dispatch( swarm, local_db, @@ -419,19 +485,53 @@ pub async fn handle_self_p2p_msg( { Ok(_) => { let mut storage_processor = local_db.acquire().await?; - storage_processor + let state_updated = storage_processor .update_messages_state( &message.message_id, message.message_version, MessageState::Processed.to_string(), ) .await?; + if state_updated { + tracing::info!( + event = "local_message_queue", + outcome = "processed", + role = %actor, + business_id = %message.business_id, + queued_message_id = %message.message_id, + message_type = %message.msg_type, + queue_wait_secs, + elapsed_ms = started_at.elapsed().as_millis() as u64, + "processed local message" + ); + } else { + tracing::warn!( + event = "local_message_queue", + outcome = "state_update_conflict", + role = %actor, + business_id = %message.business_id, + queued_message_id = %message.message_id, + message_type = %message.msg_type, + queue_wait_secs, + elapsed_ms = started_at.elapsed().as_millis() as u64, + "local message handler completed but its processed state was not persisted" + ); + } } Err(err) => { let lock_time = 600; - warn!( - "fail to process message:{}: {} will lock seconds {lock_time}", - message.message_id, err + tracing::warn!( + event = "local_message_queue", + outcome = "deferred", + role = %actor, + business_id = %message.business_id, + queued_message_id = %message.message_id, + message_type = %message.msg_type, + retry_after_secs = lock_time, + queue_wait_secs, + elapsed_ms = started_at.elapsed().as_millis() as u64, + error = %err, + "failed to process local message; deferred for retry" ); let mut storage_processor = local_db.acquire().await?; storage_processor @@ -470,6 +570,10 @@ pub async fn recv_and_dispatch( // Determine whether the message comes from this node itself to optionally skip validations let is_self_peer = get_local_node_info().peer_id == from_peer_id.to_string(); let message = GOATMessage::deserialize_message(message).await?; + let message_type = message.content.event_type(); + let role = actor.to_string(); + let from_peer_id_string = from_peer_id.to_string(); + let started_at = Instant::now(); let mut handler_ctx = HandlerContext { swarm, local_db, @@ -482,10 +586,34 @@ pub async fn recv_and_dispatch( id, is_self_peer, }; - handle_dispatch(&mut handler_ctx, message.content()).await + let result = handle_dispatch(&mut handler_ctx, message.content()).await; + match &result { + Ok(()) => tracing::info!( + event = "message_dispatch_result", + outcome = "handled", + role, + message_type, + from_peer_id = %from_peer_id_string, + is_self_peer, + elapsed_ms = started_at.elapsed().as_millis() as u64, + "message dispatch completed" + ), + Err(err) => tracing::warn!( + event = "message_dispatch_result", + outcome = "failed", + role, + message_type, + from_peer_id = %from_peer_id_string, + is_self_peer, + elapsed_ms = started_at.elapsed().as_millis() as u64, + error = %err, + "message dispatch failed" + ), + } + result } -pub async fn try_finalize_graph( +pub(crate) async fn try_finalize_graph( swarm: &mut Swarm, local_db: &LocalDB, goat_client: &GOATClient, @@ -493,7 +621,7 @@ pub async fn try_finalize_graph( graph_id: Uuid, graph: Option<&SimplifiedBitvmGcGraph>, broadcast_graph_finalize: bool, -) -> Result<()> { +) -> Result> { let endorsements = get_committee_endorsements_for_graph(local_db, instance_id, graph_id).await?; let params_endorsements = @@ -516,6 +644,15 @@ pub async fn try_finalize_graph( BitvmGcGraph::from_simplified(&g)? } }; + if graph.parameters.instance_parameters.instance_id != instance_id + || graph.parameters.graph_id != graph_id + { + bail!( + "refuse to finalize graph {instance_id}:{graph_id} with mismatched graph parameters {}:{}", + graph.parameters.instance_parameters.instance_id, + graph.parameters.graph_id + ); + } let pub_nonces = order_committee_values(&committee_pubkeys, pub_nonoces, "graph committee pub nonces")?; let agg_nonces = nonces_aggregation(&pub_nonces)?; @@ -527,7 +664,9 @@ pub async fn try_finalize_graph( let committee_sig_for_graph = signature_aggregation(&partial_sigs, &agg_nonces, &graph)?; push_committee_pre_signatures(&mut graph, &committee_sig_for_graph)?; let simplified_graph = graph.to_simplified()?; - store_graph(local_db, &simplified_graph).await?; + let store_outcome = store_finalized_graph_if_needed(local_db, &simplified_graph).await?; + mark_graph_as_endorsed(local_db, instance_id, graph_id).await?; + try_transition_instance_to_presigned(local_db, instance_id).await?; if broadcast_graph_finalize { let message_content = GOATMessageContent::GraphFinalize(GraphFinalize { instance_id, @@ -539,21 +678,44 @@ pub async fn try_finalize_graph( }); send_to_peer(swarm, GOATMessage::new(Actor::All, message_content)).await?; } + return Ok(Some((graph, store_outcome))); } - Ok(()) + Ok(None) } pub async fn send_to_peer( swarm: &mut Swarm, message: GOATMessage, ) -> Result { - let actor = message.actor.to_string(); - let topic = crate::middleware::get_topic_name(&actor); + let target_actor = message.actor.to_string(); + let message_type = message.content.event_type(); + let topic = crate::middleware::get_topic_name(&target_actor); let gossipsub_topic = gossipsub::IdentTopic::new(topic); - Ok(swarm - .behaviour_mut() - .gossipsub - .publish(gossipsub_topic, message.serialize_message().await?)?) + let serialized = message.serialize_message().await?; + match swarm.behaviour_mut().gossipsub.publish(gossipsub_topic, serialized) { + Ok(message_id) => { + tracing::info!( + event = "p2p_message_publish", + outcome = "published", + target_actor, + message_type, + message_id = ?message_id, + "published protocol message" + ); + Ok(message_id) + } + Err(err) => { + tracing::warn!( + event = "p2p_message_publish", + outcome = "failed", + target_actor, + message_type, + error = %err, + "failed to publish protocol message" + ); + Err(err.into()) + } + } } pub async fn push_local_unhandled_messages( @@ -592,21 +754,49 @@ pub(crate) async fn get_graph_or_defer( Some(g) => Ok(Some(g)), None => { // Ask for sync and push to local queue with a short retry delay - if let Err(e) = + let sync_request_outcome = if let Err(error) = try_send_sync_graph_request(swarm, goat_client, instance_id, graph_id).await { - tracing::warn!("Failed to send SyncGraphRequest for {instance_id}:{graph_id}: {e}"); - } + tracing::warn!( + event = "graph_resolution", + outcome = "sync_request_failed", + instance_id = %instance_id, + graph_id = %graph_id, + message_type = message.content.event_type(), + error_class = "p2p", + error = %error, + "failed to request graph synchronization" + ); + "failed" + } else { + "submitted" + }; let delay_secs: usize = 60; // 1 min default retry - if let Err(e) = + if let Err(error) = push_local_unhandled_messages(local_db, graph_id, message, delay_secs).await { - tracing::warn!( - "Failed to enqueue deferred message for {instance_id}:{graph_id}: {e}" + tracing::error!( + event = "graph_resolution", + outcome = "defer_failed", + instance_id = %instance_id, + graph_id = %graph_id, + message_type = message.content.event_type(), + retry_after_secs = delay_secs, + error_class = "database", + error = %error, + "failed to enqueue message while graph is missing" ); + return Err(error).context("failed to defer message while graph is missing"); } tracing::info!( - "Graph missing locally for {instance_id}:{graph_id}; requested sync and deferred" + event = "graph_resolution", + outcome = "deferred_missing_graph", + instance_id = %instance_id, + graph_id = %graph_id, + message_type = message.content.event_type(), + sync_request_outcome, + retry_after_secs = delay_secs, + "graph missing locally; requested sync and deferred message" ); Ok(None) } diff --git a/node/src/bin/mock_rpc.rs b/node/src/bin/mock_rpc.rs index c9e086442..e004d3613 100644 --- a/node/src/bin/mock_rpc.rs +++ b/node/src/bin/mock_rpc.rs @@ -21,9 +21,11 @@ use clap::Parser; use client::Utxo; use prometheus_client::registry::Registry; use secp256k1::Secp256k1; +use store::localdb::{GraphRuntimeUpdate, StorageProcessor}; use store::{ BridgeOutGlobalStats, GoatTxProcessingStatus, GoatTxRecord, GoatTxType, Graph, GraphStatus, - Instance, InstanceBridgeInStatus, InstanceBridgeOutStatus, Node, UInt64Array3, create_local_db, + GraphStatusSource, Instance, InstanceBridgeInStatus, InstanceBridgeOutStatus, Node, + UInt64Array3, create_local_db, }; use tokio::signal; use tokio_util::sync::CancellationToken; @@ -211,6 +213,7 @@ fn seeded_graph( status: status.to_string(), sub_status, operator_pubkey: operator_pubkey.to_string(), + definition_hash: format!("mock-{graph_id}"), init_withdraw_tx_hash, bridge_out_start_at: now - 300, status_updated_at: now - 120, @@ -221,6 +224,52 @@ fn seeded_graph( } } +async fn seed_graph_runtime(tx: &mut StorageProcessor<'_>, graph: &Graph) -> Result<()> { + let target_status = GraphStatus::from_str(&graph.status) + .map_err(|_| anyhow::anyhow!("invalid mock graph status {}", graph.status))?; + let sub_status = graph.sub_status.clone(); + let challenge_txid = graph.challenge_txid.clone(); + let init_withdraw_tx_hash = graph.init_withdraw_tx_hash.clone(); + let bridge_out_start_at = graph.bridge_out_start_at; + let proceed_withdraw_height = graph.proceed_withdraw_height; + + let mut definition = graph.clone(); + definition.status = GraphStatus::OperatorPresigned.to_string(); + definition.sub_status.clear(); + definition.challenge_txid = None; + definition.init_withdraw_tx_hash = None; + definition.bridge_out_start_at = 0; + definition.proceed_withdraw_height = 0; + tx.upsert_graph_definition(&definition).await?; + + if target_status != GraphStatus::OperatorPresigned { + tx.transition_graph_status( + graph.instance_id, + graph.graph_id, + target_status, + GraphStatusSource::ChainReconcile, + (!sub_status.is_empty()).then_some(sub_status), + ) + .await?; + } + + let mut runtime = GraphRuntimeUpdate::new(graph.instance_id, graph.graph_id); + if let Some(challenge_txid) = challenge_txid { + runtime = runtime.with_challenge_txid(challenge_txid); + } + if let Some(init_withdraw_tx_hash) = init_withdraw_tx_hash { + runtime = runtime.with_init_withdraw_tx_hash(init_withdraw_tx_hash); + } + if bridge_out_start_at != 0 { + runtime = runtime.with_bridge_out_start_at(bridge_out_start_at); + } + if proceed_withdraw_height != 0 { + runtime = runtime.with_proceed_withdraw_height(proceed_withdraw_height); + } + tx.update_graph_runtime(&runtime).await?; + Ok(()) +} + async fn seed_mock_data( local_db: &store::localdb::LocalDB, current_peer_id: &str, @@ -294,7 +343,7 @@ async fn seed_mock_data( tx.upsert_instance(&instance).await?; } for graph in [ready_graph, challenge_graph] { - tx.upsert_graph(&graph).await?; + seed_graph_runtime(&mut tx, &graph).await?; } tx.upsert_goat_tx_record(&GoatTxRecord { instance_id: bridge_out_instance_id, diff --git a/node/src/bin/send_verifier_challenge.rs b/node/src/bin/send_verifier_challenge.rs new file mode 100644 index 000000000..5ba8dbb5e --- /dev/null +++ b/node/src/bin/send_verifier_challenge.rs @@ -0,0 +1,85 @@ +//! verifier-challenge: force a verifier ChallengeAssert transaction via node API. +//! +//! Purpose: +//! - Call the verifier node's test endpoint to broadcast its ChallengeAssert +//! transaction after an operator assert. +//! +//! Env: +//! - BITVM_SECRET: the node's secret key, used to sign the auth headers +//! +//! Args: +//! - --rpc-url: verifier node API base URL (default: http://localhost:8080) +//! - --graph-id: target graph UUID +//! +//! Example: +//! - cargo run -p bitvm-noded --bin verifier-challenge -- \ +//! --rpc-url http://localhost:8910 \ +//! --graph-id + +use anyhow::{Context, Result}; +use bitvm_noded::env::get_bitvm_key; +use bitvm_noded::rpc_service::auth::{ + AUTH_SIGNATURE_HEADER, AUTH_TIMESTAMP_HEADER, sign_request_auth, +}; +use clap::Parser; +use serde::Deserialize; + +#[derive(Debug, Parser)] +#[command( + name = "verifier-challenge", + version, + about = "Force a verifier ChallengeAssert transaction for a graph (via node API)", + long_about = "Force a verifier ChallengeAssert transaction for a graph via the verifier node's REST API.\n\nThe verifier node must be running and reachable at the given --rpc-url." +)] +struct Args { + /// Graph UUID to challenge + #[arg(long)] + graph_id: uuid::Uuid, + + /// Verifier node API base URL + #[arg(long, default_value = "http://localhost:8080")] + rpc_url: String, +} + +#[derive(Debug, Deserialize)] +struct SendVerifierChallengeResponse { + challenge_assert_txid: String, + verifier_index: usize, +} + +#[tokio::main] +async fn main() -> Result<()> { + dotenv::dotenv().ok(); + let args = Args::parse(); + let url = format!( + "{}/v1/graphs/{}/send-verifier-challenge", + args.rpc_url.trim_end_matches('/'), + args.graph_id + ); + + let keypair = get_bitvm_key().context("failed to load BITVM_SECRET")?; + let (timestamp, signature) = sign_request_auth(&keypair); + + let client = reqwest::Client::new(); + let resp = client + .post(&url) + .header(AUTH_TIMESTAMP_HEADER, ×tamp) + .header(AUTH_SIGNATURE_HEADER, &signature) + .send() + .await + .with_context(|| format!("failed to reach verifier node API at {url}"))?; + + if !resp.status().is_success() { + let status = resp.status(); + let body = resp.text().await.unwrap_or_default(); + anyhow::bail!("API returned {status}: {body}"); + } + + let body: SendVerifierChallengeResponse = + resp.json().await.context("failed to parse API response")?; + println!( + "Verifier ChallengeAssert tx broadcasted: {} (verifier_index={})", + body.challenge_assert_txid, body.verifier_index + ); + Ok(()) +} diff --git a/node/src/handle.rs b/node/src/handle.rs index 480b43a9b..3ed9c50be 100644 --- a/node/src/handle.rs +++ b/node/src/handle.rs @@ -235,13 +235,33 @@ pub async fn dispatch(ctx: &mut HandlerContext<'_>, content: &GOATMessageContent .await } ( - GOATMessageContent::CreateGraph(CreateGraph { instance_id, graph_id, graph, .. }), + GOATMessageContent::CreateGraph(CreateGraph { + instance_id, + graph_id, + graph_nonce, + graph, + }), Actor::Verifier, - ) => handle_create_graph_verifier(ctx, *instance_id, *graph_id, graph).await, + ) => handle_create_graph_verifier(ctx, *instance_id, *graph_id, *graph_nonce, graph).await, ( - GOATMessageContent::CreateGraph(CreateGraph { instance_id, graph_id, graph, .. }), + GOATMessageContent::CreateGraph(CreateGraph { + instance_id, + graph_id, + graph_nonce, + graph, + }), Actor::Committee, - ) => handle_create_graph_committee(ctx, *instance_id, *graph_id, graph, content).await, + ) => { + handle_create_graph_committee( + ctx, + *instance_id, + *graph_id, + *graph_nonce, + graph, + content, + ) + .await + } ( GOATMessageContent::VerifierGraphParamsEndorsement(VerifierGraphParamsEndorsement { instance_id, @@ -374,10 +394,10 @@ pub async fn dispatch(ctx: &mut HandlerContext<'_>, content: &GOATMessageContent GOATMessageContent::GraphFinalize(GraphFinalize { instance_id, graph_id, + graph_nonce, graph, endorse_sigs, params_endorse_sigs, - .. }), Actor::Committee, ) => { @@ -385,6 +405,7 @@ pub async fn dispatch(ctx: &mut HandlerContext<'_>, content: &GOATMessageContent ctx, *instance_id, *graph_id, + *graph_nonce, graph, endorse_sigs, params_endorse_sigs, @@ -395,10 +416,10 @@ pub async fn dispatch(ctx: &mut HandlerContext<'_>, content: &GOATMessageContent GOATMessageContent::GraphFinalize(GraphFinalize { instance_id, graph_id, + graph_nonce, graph, endorse_sigs, params_endorse_sigs, - .. }), _, ) => { @@ -406,6 +427,7 @@ pub async fn dispatch(ctx: &mut HandlerContext<'_>, content: &GOATMessageContent ctx, *instance_id, *graph_id, + *graph_nonce, graph, endorse_sigs, params_endorse_sigs, @@ -659,6 +681,32 @@ fn make_message(ctx: &HandlerContext<'_>, content: &GOATMessageContent) -> GOATM GOATMessage::new(ctx.actor.clone(), content.clone()) } +/// A graph-bearing message has two identity representations: its envelope and +/// the signed graph parameters. Never use one to read/write local state while +/// using the other to reconstruct or scan the graph. +fn message_identity_matches( + message_kind: &str, + message_instance_id: Uuid, + message_graph_id: Uuid, + message_graph_nonce: Option, + graph_instance_id: Uuid, + graph_graph_id: Uuid, + graph_nonce: u64, +) -> bool { + let nonce_matches = message_graph_nonce.is_none_or(|nonce| nonce == graph_nonce); + if message_instance_id == graph_instance_id + && message_graph_id == graph_graph_id + && nonce_matches + { + return true; + } + + tracing::warn!( + "Ignore {message_kind}: message identity {message_instance_id}:{message_graph_id}:{message_graph_nonce:?} does not match graph parameters {graph_instance_id}:{graph_graph_id}:{graph_nonce}" + ); + false +} + /// Freezes the first accepted Verifiers and assigns deterministic graph slots. fn freeze_operator_candidates(state: &mut OperatorBabeSetupState) -> Result<()> { if state.frozen_verifier_pubkeys.is_some() { @@ -1001,58 +1049,62 @@ async fn refresh_and_compensate( ctx: &HandlerContext<'_>, instance_id: Uuid, graph_id: Uuid, - graph: Option<&BitvmGcGraph>, - scan_from_status: Option, - compensate_from_status: GraphStatus, -) -> Result<(GraphStatus, Option)> { - let (graph_status, sub_status, scan) = refresh_graph( - ctx.local_db, - ctx.btc_client, - ctx.goat_client, - instance_id, - graph_id, - graph, - scan_from_status, - None, - ) - .await?; + graph: &BitvmGcGraph, + compensation_anchor_status: GraphStatus, +) -> Result<(GraphStatus, Option, bool)> { + let refresh = + refresh_graph(ctx.local_db, ctx.btc_client, ctx.goat_client, instance_id, graph_id, graph) + .await?; + let graph_status = refresh.status; + let sub_status = refresh.sub_status; tracing::info!("Graph {graph_id} latest status: {graph_status}"); - compensate_graph_events( - ctx.local_db, - ctx.btc_client, - instance_id, - graph_id, - graph, - scan.as_ref(), - scan_from_status, - compensate_from_status, - graph_status, - ) - .await?; - Ok((graph_status, sub_status)) + if refresh.status_transition_accepted { + compensate_graph_events( + ctx.local_db, + ctx.btc_client, + instance_id, + graph_id, + graph, + refresh.scan.as_ref(), + compensation_anchor_status, + graph_status, + ) + .await?; + } + Ok((graph_status, sub_status, refresh.status_transition_accepted)) +} + +async fn refresh_newly_finalized_graph( + ctx: &HandlerContext<'_>, + instance_id: Uuid, + graph_id: Uuid, + graph: &BitvmGcGraph, +) -> Result<()> { + // Reconcile every verified GraphFinalize, including a duplicate. A prior + // attempt may have stored the finalized definition but failed before its + // first chain scan completed. + refresh_and_compensate(ctx, instance_id, graph_id, graph, GraphStatus::CommitteePresigned) + .await?; + Ok(()) } -async fn get_graph_and_status( +async fn get_graph_for_refresh( ctx: &HandlerContext<'_>, instance_id: Uuid, graph_id: Uuid, -) -> Result<(BitvmGcGraph, GraphStatus)> { +) -> Result { let graph = get_graph(ctx.local_db, instance_id, graph_id) .await? .ok_or_else(|| anyhow!("Graph not found for {instance_id}:{graph_id}"))?; - let graph = BitvmGcGraph::from_simplified(&graph)?; - let graph_start_status = get_graph_status(ctx.local_db, instance_id, graph_id) - .await? - .ok_or_else(|| anyhow!("Graph status not found for {instance_id}:{graph_id}"))?; - Ok((graph, graph_start_status)) + BitvmGcGraph::from_simplified(&graph) } -async fn get_graph_and_status_or_defer( +async fn get_graph_for_refresh_or_defer( ctx: &mut HandlerContext<'_>, instance_id: Uuid, graph_id: Uuid, message: &GOATMessage, -) -> Result> { +) -> Result> { let graph = match get_graph_or_defer( ctx.swarm, ctx.local_db, @@ -1066,11 +1118,7 @@ async fn get_graph_and_status_or_defer( Some(g) => g, None => return Ok(None), }; - let graph = BitvmGcGraph::from_simplified(&graph)?; - let graph_start_status = get_graph_status(ctx.local_db, instance_id, graph_id) - .await? - .ok_or_else(|| anyhow!("Graph status not found for {instance_id}:{graph_id}"))?; - Ok(Some((graph, graph_start_status))) + Ok(Some(BitvmGcGraph::from_simplified(&graph)?)) } async fn refresh_graph_status( @@ -1080,24 +1128,23 @@ async fn refresh_graph_status( message: Option<&GOATMessage>, compensate_from_status: GraphStatus, ) -> Result)>> { - let (graph, graph_start_status) = match message { + let graph = match message { Some(message) => { - match get_graph_and_status_or_defer(ctx, instance_id, graph_id, message).await? { + match get_graph_for_refresh_or_defer(ctx, instance_id, graph_id, message).await? { Some(v) => v, None => return Ok(None), } } - None => get_graph_and_status(ctx, instance_id, graph_id).await?, + None => get_graph_for_refresh(ctx, instance_id, graph_id).await?, }; - let (graph_status, sub_status) = refresh_and_compensate( - ctx, - instance_id, - graph_id, - Some(&graph), - Some(graph_start_status), - compensate_from_status, - ) - .await?; + let (graph_status, sub_status, status_transition_accepted) = + refresh_and_compensate(ctx, instance_id, graph_id, &graph, compensate_from_status).await?; + if !status_transition_accepted { + tracing::warn!( + "Ignore graph action for {instance_id}:{graph_id}: chain scan status was rejected or graph is missing" + ); + return Ok(None); + } Ok(Some((graph, graph_status, sub_status))) } @@ -1182,6 +1229,69 @@ async fn handle_pegin_request_default( Ok(()) } +async fn defer_confirm_instance_until_previous_graph_presigned( + ctx: &mut HandlerContext<'_>, + instance_id: Uuid, + successor_nonce: u64, + operator_pubkey: &PublicKey, +) -> Result { + if successor_nonce == 0 { + return Ok(false); + } + + let previous_nonce = successor_nonce - 1; + let retry_message = GOATMessage::new( + Actor::Operator, + GOATMessageContent::ConfirmInstance(ConfirmInstance { instance_id }), + ); + let Some((previous_instance_id, previous_graph_id)) = + get_graph_id_by_nonce(ctx.local_db, previous_nonce, operator_pubkey).await? + else { + push_local_unhandled_messages(ctx.local_db, instance_id, &retry_message, 60).await?; + tracing::warn!( + "Defer ConfirmInstance for {instance_id}: previous graph with nonce {previous_nonce} is not available locally" + ); + return Ok(true); + }; + let Some(previous_graph) = + get_graph(ctx.local_db, previous_instance_id, previous_graph_id).await? + else { + if let Err(error) = try_send_sync_graph_request( + ctx.swarm, + ctx.goat_client, + previous_instance_id, + previous_graph_id, + ) + .await + { + tracing::warn!( + "Failed to send SyncGraphRequest for previous graph {previous_instance_id}:{previous_graph_id}: {error}" + ); + } + push_local_unhandled_messages(ctx.local_db, instance_id, &retry_message, 60).await?; + tracing::info!( + "Defer ConfirmInstance for {instance_id}: waiting for previous graph raw data {previous_instance_id}:{previous_graph_id}" + ); + return Ok(true); + }; + if previous_graph.committee_pre_signed() { + return Ok(false); + } + + push_local_unhandled_messages(ctx.local_db, instance_id, &retry_message, 60).await?; + let message_content = GOATMessageContent::CreateGraph(CreateGraph { + instance_id: previous_instance_id, + graph_id: previous_graph_id, + graph_nonce: previous_graph.parameters.graph_nonce, + graph: previous_graph, + }); + send_to_peer(ctx.swarm, GOATMessage::new(Actor::All, message_content)).await?; + tracing::info!( + "Defer ConfirmInstance for {instance_id}: re-broadcast previous CreateGraph {previous_instance_id}:{previous_graph_id} until it is committee pre-signed" + ); + Ok(true) +} + #[tracing::instrument(level = "info", skip_all, fields(instance_id = %instance_id))] async fn handle_confirm_instance_operator( ctx: &mut HandlerContext<'_>, @@ -1198,6 +1308,16 @@ async fn handle_confirm_instance_operator( ) .await? { + if defer_confirm_instance_until_previous_graph_presigned( + ctx, + instance_id, + graph.parameters.graph_nonce, + &local_operator_pubkey, + ) + .await? + { + return Ok(()); + } let graph_id = graph.parameters.graph_id; tracing::info!("Graph already created for {instance_id}, graph_id: {}", graph_id); let message_content = GOATMessageContent::CreateGraph(CreateGraph { @@ -1222,6 +1342,18 @@ async fn handle_confirm_instance_operator( .map(|pending| pending.graph_id) }; if let Some(graph_id) = pending_graph_id { + if let Some((next_graph_nonce, _)) = + get_current_prekickoff_tx(ctx.local_db, &local_operator_pubkey).await? + && defer_confirm_instance_until_previous_graph_presigned( + ctx, + instance_id, + next_graph_nonce, + &local_operator_pubkey, + ) + .await? + { + return Ok(()); + } tracing::info!("Resume pending graph setup for {instance_id}, graph_id: {graph_id}"); let message_content = GOATMessageContent::InitGraph(InitGraph { instance_id, graph_id }); send_to_peer(ctx.swarm, GOATMessage::new(Actor::Verifier, message_content)).await?; @@ -1246,6 +1378,19 @@ async fn handle_confirm_instance_operator( ); bail!("Invalid ConfirmInstance: pegin deposit tx {pegin_deposit_txid} not found on chain"); } + + if let Some((next_graph_nonce, _)) = + get_current_prekickoff_tx(ctx.local_db, &local_operator_pubkey).await? + && defer_confirm_instance_until_previous_graph_presigned( + ctx, + instance_id, + next_graph_nonce, + &local_operator_pubkey, + ) + .await? + { + return Ok(()); + } // after PeginPrepare is confirmed, broadcast InitGraph and let Verifiers generate GC. // 2. save the instance data to local db @@ -1682,7 +1827,7 @@ fn decode_soldering_proof_payload( verifier_index, actual_len = payload.len(), total_len, - payload_hash = %soldering_payload_hash_hex(&payload_hash), + payload_hash = %soldering_payload_hash_hex(payload_hash), "SolderingProof payload length mismatch" ); bail!( @@ -1696,7 +1841,7 @@ fn decode_soldering_proof_payload( instance_id = %instance_id, graph_id = %graph_id, verifier_index, - expected_hash = %soldering_payload_hash_hex(&payload_hash), + expected_hash = %soldering_payload_hash_hex(payload_hash), actual_hash = %soldering_payload_hash_hex(&actual_hash), "SolderingProof payload hash mismatch" ); @@ -1843,7 +1988,7 @@ async fn handle_compact_soldering_proof_operator( operator_pre_sign(operator_master_key.master_keypair(), &mut graph)?; let graph = graph.to_simplified()?; - store_graph(ctx.local_db, &graph).await?; + store_operator_presigned_graph(ctx.local_db, &graph).await?; let mut storage = ctx.local_db.acquire().await?; storage.delete_pending_graph_init(&instance_id, &local_operator_pubkey.to_string()).await?; @@ -1888,8 +2033,21 @@ async fn handle_create_graph_verifier( ctx: &mut HandlerContext<'_>, instance_id: Uuid, graph_id: Uuid, + graph_nonce: u64, graph: &SimplifiedBitvmGcGraph, ) -> Result<()> { + if !message_identity_matches( + "CreateGraph", + instance_id, + graph_id, + Some(graph_nonce), + graph.parameters.instance_parameters.instance_id, + graph.parameters.graph_id, + graph.parameters.graph_nonce, + ) { + return Ok(()); + } + let verifier_master_key = VerifierMasterKey::new(get_bitvm_key()?); let local_verifier_pubkey: PublicKey = verifier_master_key.master_keypair().public_key().into(); let Some(verifier_index) = @@ -2122,9 +2280,22 @@ async fn handle_create_graph_committee( ctx: &mut HandlerContext<'_>, instance_id: Uuid, graph_id: Uuid, + graph_nonce: u64, graph: &SimplifiedBitvmGcGraph, content: &GOATMessageContent, ) -> Result<()> { + if !message_identity_matches( + "CreateGraph", + instance_id, + graph_id, + Some(graph_nonce), + graph.parameters.instance_parameters.instance_id, + graph.parameters.graph_id, + graph.parameters.graph_nonce, + ) { + return Ok(()); + } + // received from Operator if graph.parameters.graph_nonce > 0 { let previous_nonce = graph.parameters.graph_nonce - 1; @@ -2132,8 +2303,9 @@ async fn handle_create_graph_committee( .await? { Some((previous_instance_id, previous_graph_id)) => { - if get_graph(ctx.local_db, previous_instance_id, previous_graph_id).await?.is_none() - { + let Some(previous_graph) = + get_graph(ctx.local_db, previous_instance_id, previous_graph_id).await? + else { if let Err(e) = try_send_sync_graph_request( ctx.swarm, ctx.goat_client, @@ -2152,6 +2324,14 @@ async fn handle_create_graph_committee( "Defer CreateGraph for {instance_id}:{graph_id}: waiting for previous graph raw data {previous_instance_id}:{previous_graph_id}" ); return Ok(()); + }; + if !previous_graph.committee_pre_signed() { + let message = make_message(ctx, content); + push_local_unhandled_messages(ctx.local_db, graph_id, &message, 60).await?; + tracing::info!( + "Defer CreateGraph for {instance_id}:{graph_id}: waiting for previous graph {previous_instance_id}:{previous_graph_id} to be committee pre-signed" + ); + return Ok(()); } } None => { @@ -2176,7 +2356,7 @@ async fn handle_create_graph_committee( bail!(e) }; // 2. save the graph data to local db - store_graph(ctx.local_db, graph).await?; + store_operator_presigned_graph(ctx.local_db, graph).await?; // 3. start committee setup once verifier params endorsements are complete try_start_graph_committee_setup(ctx, instance_id, graph_id, graph).await } @@ -2249,7 +2429,7 @@ async fn handle_verifier_graph_params_endorsement_committee( graph_id, *verifier_pubkey, verifier_index, - signature.clone(), + *signature, ) .await?; try_start_graph_committee_setup(ctx, instance_id, graph_id, &graph).await @@ -2478,7 +2658,7 @@ async fn handle_nonce_generation_operator( // 3. if received enough endorsement signatures, mark the graph as endorsed, send the graph to local db, broadcast GraphFinalize // Operator may receive EndorseGraph, CommitteePresign or NonceGeneration messages in any order // So we need to check if we have collected enough endorsements, pub_nonces and partial_sigs every time we receive them - try_finalize_graph( + if let Some((finalized_graph, _)) = try_finalize_graph( ctx.swarm, ctx.local_db, ctx.goat_client, @@ -2487,7 +2667,10 @@ async fn handle_nonce_generation_operator( Some(&graph), true, ) - .await?; + .await? + { + refresh_newly_finalized_graph(ctx, instance_id, graph_id, &finalized_graph).await?; + } Ok(()) } @@ -2700,8 +2883,19 @@ async fn handle_committee_presign_operator( // 3. if received enough endorsement signatures, mark the graph as endorsed, send the graph to local database, broadcast GraphFinalize // Operator may receive EndorseGraph, CommitteePresign or NonceGeneration messages in any order // So we need to check if we have collected enough endorsements, pub_nonces and partial_sigs every time we receive them - try_finalize_graph(ctx.swarm, ctx.local_db, ctx.goat_client, instance_id, graph_id, None, true) - .await?; + if let Some((finalized_graph, _)) = try_finalize_graph( + ctx.swarm, + ctx.local_db, + ctx.goat_client, + instance_id, + graph_id, + None, + true, + ) + .await? + { + refresh_newly_finalized_graph(ctx, instance_id, graph_id, &finalized_graph).await?; + } Ok(()) } @@ -2789,7 +2983,7 @@ async fn handle_endorse_graph_operator( // 3. if received enough endorsement signatures, mark the graph as endorsed, send the graph to local database, broadcast GraphFinalize // Operator may receive EndorseGraph, CommitteePresign or NonceGeneration messages in any order // So we need to check if we have collected enough endorsements, pub_nonces and partial_sigs every time we receive them - try_finalize_graph( + if let Some((finalized_graph, _)) = try_finalize_graph( ctx.swarm, ctx.local_db, ctx.goat_client, @@ -2798,7 +2992,10 @@ async fn handle_endorse_graph_operator( Some(&graph), true, ) - .await?; + .await? + { + refresh_newly_finalized_graph(ctx, instance_id, graph_id, &finalized_graph).await?; + } Ok(()) } @@ -2807,10 +3004,23 @@ async fn handle_graph_finalize_committee( ctx: &mut HandlerContext<'_>, instance_id: Uuid, graph_id: Uuid, + graph_nonce: u64, graph: &SimplifiedBitvmGcGraph, endorse_sigs: &[(PublicKey, alloy::primitives::Address, Vec)], params_endorse_sigs: &[(PublicKey, alloy::primitives::Address, Vec)], ) -> Result<()> { + if !message_identity_matches( + "GraphFinalize", + instance_id, + graph_id, + Some(graph_nonce), + graph.parameters.instance_parameters.instance_id, + graph.parameters.graph_id, + graph.parameters.graph_nonce, + ) { + return Ok(()); + } + // received from Operator // 1. check graph data if let Err(e) = todo_funcs::validate_finalized_graph( @@ -2833,8 +3043,8 @@ async fn handle_graph_finalize_committee( } bail!(e) } - // 2. save the graph data to local db - store_graph(ctx.local_db, graph).await?; + // 2. Store a finalized graph only when it upgrades the local graph. + let _ = store_finalized_graph_if_needed(ctx.local_db, graph).await?; store_committee_endorsements_for_graph( ctx.local_db, instance_id, @@ -2843,12 +3053,13 @@ async fn handle_graph_finalize_committee( params_endorse_sigs.to_owned(), ) .await?; - // After storing, mark the graph as endorsed + // After storing, mark the graph as finalized for the instance threshold. mark_graph_as_endorsed(ctx.local_db, instance_id, graph_id).await?; + try_transition_instance_to_presigned(ctx.local_db, instance_id).await?; + let finalized_graph = BitvmGcGraph::from_simplified(graph)?; + refresh_newly_finalized_graph(ctx, instance_id, graph_id, &finalized_graph).await?; // 3. if endorsed graph count >= threshold, generate & broadcast PeginConfirmNonce - if get_endorsed_graph_count(ctx.local_db, instance_id).await? - >= todo_funcs::min_required_operator() - { + if has_required_presigned_graphs(ctx.local_db, instance_id).await? { let committee_master_key = CommitteeMasterKey::new(get_bitvm_key()?); let instance_keypair = load_committee_instance_keypair(&committee_master_key, instance_id)?; let local_committee_pubkey = instance_keypair.public_key().into(); @@ -2924,10 +3135,23 @@ async fn handle_graph_finalize_default( ctx: &mut HandlerContext<'_>, instance_id: Uuid, graph_id: Uuid, + graph_nonce: u64, graph: &SimplifiedBitvmGcGraph, endorse_sigs: &[(PublicKey, alloy::primitives::Address, Vec)], params_endorse_sigs: &[(PublicKey, alloy::primitives::Address, Vec)], ) -> Result<()> { + if !message_identity_matches( + "GraphFinalize", + instance_id, + graph_id, + Some(graph_nonce), + graph.parameters.instance_parameters.instance_id, + graph.parameters.graph_id, + graph.parameters.graph_nonce, + ) { + return Ok(()); + } + // received from Operator // 1. check graph data if let Err(e) = todo_funcs::validate_finalized_graph( @@ -2950,8 +3174,20 @@ async fn handle_graph_finalize_default( } bail!(e) } - // 2. save the graph data to local db - store_graph(ctx.local_db, graph).await?; + // 2. Store a finalized graph only when it upgrades the local graph. + let _ = store_finalized_graph_if_needed(ctx.local_db, graph).await?; + store_committee_endorsements_for_graph( + ctx.local_db, + instance_id, + graph_id, + endorse_sigs.to_owned(), + params_endorse_sigs.to_owned(), + ) + .await?; + mark_graph_as_endorsed(ctx.local_db, instance_id, graph_id).await?; + try_transition_instance_to_presigned(ctx.local_db, instance_id).await?; + let finalized_graph = BitvmGcGraph::from_simplified(graph)?; + refresh_newly_finalized_graph(ctx, instance_id, graph_id, &finalized_graph).await?; Ok(()) } @@ -3421,35 +3657,33 @@ async fn handle_kickoff_ready_operator( None => return Ok(()), }; let mut current_graph = BitvmGcGraph::from_simplified(¤t_graph)?; - let current_graph_start_status = - get_graph_status(ctx.local_db, current_instance_id, current_graph_id) - .await? - .ok_or_else(|| { - anyhow!("Graph status not found for {current_instance_id}:{current_graph_id}") - })?; - let (current_graph_status, _current_graph_sub_status, current_graph_scan) = refresh_graph( + let current_graph_refresh = refresh_graph( ctx.local_db, ctx.btc_client, ctx.goat_client, current_instance_id, current_graph_id, - Some(¤t_graph), - Some(current_graph_start_status), - None, - ) - .await?; - compensate_graph_events( - ctx.local_db, - ctx.btc_client, - current_instance_id, - current_graph_id, - Some(¤t_graph), - current_graph_scan.as_ref(), - Some(current_graph_start_status), - current_graph_start_status, - current_graph_status, + ¤t_graph, ) .await?; + let current_graph_status = current_graph_refresh.status; + if current_graph_refresh.status_transition_accepted { + compensate_graph_events( + ctx.local_db, + ctx.btc_client, + current_instance_id, + current_graph_id, + ¤t_graph, + current_graph_refresh.scan.as_ref(), + // This is a recovery scan, not a transition from the local + // projection. Start from the protocol baseline so a previous + // crash between status persistence and message enqueue can be + // repaired by idempotent message upserts. + GraphStatus::OperatorPresigned, + current_graph_status, + ) + .await?; + } if current_graph_status.is_closed() { continue; } else if current_graph_status.is_pegout_started() { @@ -3732,18 +3966,24 @@ async fn handle_previous_graph_after_prekickoff( None => return Ok(()), }; let prev_graph = BitvmGcGraph::from_simplified(&prev_graph)?; - let prev_graph_start_status = get_graph_status(ctx.local_db, prev_instance_id, prev_graph_id) - .await? - .ok_or_else(|| anyhow!("Graph status not found for {prev_instance_id}:{prev_graph_id}"))?; - let (prev_graph_status, _prev_graph_sub_status) = refresh_and_compensate( - ctx, - prev_instance_id, - prev_graph_id, - Some(&prev_graph), - Some(prev_graph_start_status), - prev_graph_start_status, - ) - .await?; + let (prev_graph_status, _prev_graph_sub_status, status_transition_accepted) = + refresh_and_compensate( + ctx, + prev_instance_id, + prev_graph_id, + &prev_graph, + // Do not use the persisted status as a compensation anchor: it + // may have been committed just before a crash. Idempotent message + // upserts make a baseline recovery scan safe to retry. + GraphStatus::OperatorPresigned, + ) + .await?; + if !status_transition_accepted { + tracing::warn!( + "Ignore prekickoff follow-up for {instance_id}:{graph_id}: previous graph status scan was rejected" + ); + return Ok(()); + } if !tx_on_chain(ctx.btc_client, &prev_graph.kickoff.tx().compute_txid()).await? { verifier_force_skip_kickoff(ctx.btc_client, &prev_graph).await?; } else if !prev_graph_status.is_closed() { @@ -4559,93 +4799,83 @@ async fn handle_assert_sent_verifier( let outpoint = connector_e_input.outpoint; if let Some(commit_pubin_txid) = outpoint_spent_txid(ctx.btc_client, &outpoint.txid, outpoint.vout as u64).await? + && let Some(commit_pubin_tx) = ctx.btc_client.get_tx(&commit_pubin_txid).await? + && let Some(commit_pubin_txin) = commit_pubin_tx.input.first() { - if let Some(commit_pubin_tx) = ctx.btc_client.get_tx(&commit_pubin_txid).await? { - if let Some(commit_pubin_txin) = commit_pubin_tx.input.first() { - let assert_txin = assert_tx - .input - .first() - .ok_or_else(|| anyhow!("operator assert transaction has no input"))?; - let wci_txid = graph.watchtower_challenge_init.tx().compute_txid(); - let watchtower_timeout_txids = graph - .watchtower_challenge_timeouts - .iter() - .map(|tx| tx.tx().compute_txid()) - .collect::>(); - let ack_txins = match collect_ack_txins( - ctx.btc_client, - &wci_txid, - &watchtower_timeout_txids, + let assert_txin = assert_tx + .input + .first() + .ok_or_else(|| anyhow!("operator assert transaction has no input"))?; + let wci_txid = graph.watchtower_challenge_init.tx().compute_txid(); + let watchtower_timeout_txids = graph + .watchtower_challenge_timeouts + .iter() + .map(|tx| tx.tx().compute_txid()) + .collect::>(); + let ack_txins = match collect_ack_txins( + ctx.btc_client, + &wci_txid, + &watchtower_timeout_txids, + ) + .await + { + Ok(txins) => txins, + Err(e) => { + let delay_secs = todo_funcs::avg_block_time_secs(ctx.btc_client.network()); + let message = make_message(ctx, content); + push_local_unhandled_messages( + ctx.local_db, + graph_id, + &message, + delay_secs as usize, ) - .await - { - Ok(txins) => txins, - Err(e) => { - let delay_secs = - todo_funcs::avg_block_time_secs(ctx.btc_client.network()); - let message = make_message(ctx, content); - push_local_unhandled_messages( - ctx.local_db, - graph_id, - &message, - delay_secs as usize, - ) - .await?; - tracing::info!( - "Retry AssertSent later for {instance_id}:{graph_id}: ACK inputs are not ready: {e}" - ); - return Ok(()); - } + .await?; + tracing::info!( + "Retry AssertSent later for {instance_id}:{graph_id}: ACK inputs are not ready: {e}" + ); + return Ok(()); + } + }; + match validate_pubin_disprove(&graph, commit_pubin_txin, assert_txin, &ack_txins) { + Ok(Some((witness_data, _))) => { + // build pubin disprove Tx + let pubin_disprove_tx_total_input_amount = graph + .operator_assert + .connector_d_input() + .map_err(|e| anyhow!("failed to get connector-d input: {e}"))? + .amount; + let pubin_disprove_txin = build_pubin_disprove_txin(&graph, witness_data)?; + let pubin_disprove_tx = bitcoin::Transaction { + version: bitcoin::transaction::Version(2), + lock_time: bitcoin::absolute::LockTime::ZERO, + input: vec![pubin_disprove_txin], + output: vec![goat::scripts::p2a_output()], }; - match validate_pubin_disprove( - &graph, - commit_pubin_txin, - assert_txin, - &ack_txins, - ) { - Ok(Some((witness_data, _))) => { - // build pubin disprove Tx - let pubin_disprove_tx_total_input_amount = graph - .operator_assert - .connector_d_input() - .map_err(|e| anyhow!("failed to get connector-d input: {e}"))? - .amount; - let pubin_disprove_txin = - build_pubin_disprove_txin(&graph, witness_data)?; - let pubin_disprove_tx = bitcoin::Transaction { - version: bitcoin::transaction::Version(2), - lock_time: bitcoin::absolute::LockTime::ZERO, - input: vec![pubin_disprove_txin], - output: vec![goat::scripts::p2a_output()], - }; - broadcast_tx_with_cpfp( - ctx.btc_client, - pubin_disprove_tx, - pubin_disprove_tx_total_input_amount, - ) - .await?; - return Ok(()); - } - Ok(None) => tracing::debug!( - "PubinDisprove invalid for {instance_id}:{graph_id}: operator pubin consistent, proceeding to ChallengeAssert" - ), - Err(e) => { - let delay_secs = - todo_funcs::avg_block_time_secs(ctx.btc_client.network()); - let message = make_message(ctx, content); - push_local_unhandled_messages( - ctx.local_db, - graph_id, - &message, - delay_secs as usize, - ) - .await?; - tracing::warn!( - "Retry AssertSent later for {instance_id}:{graph_id}: PubinDisprove check failed: {e}" - ); - return Ok(()); - } - } + broadcast_tx_with_cpfp( + ctx.btc_client, + pubin_disprove_tx, + pubin_disprove_tx_total_input_amount, + ) + .await?; + return Ok(()); + } + Ok(None) => tracing::debug!( + "PubinDisprove invalid for {instance_id}:{graph_id}: operator pubin consistent, proceeding to ChallengeAssert" + ), + Err(e) => { + let delay_secs = todo_funcs::avg_block_time_secs(ctx.btc_client.network()); + let message = make_message(ctx, content); + push_local_unhandled_messages( + ctx.local_db, + graph_id, + &message, + delay_secs as usize, + ) + .await?; + tracing::warn!( + "Retry AssertSent later for {instance_id}:{graph_id}: PubinDisprove check failed: {e}" + ); + return Ok(()); } } } @@ -4667,25 +4897,82 @@ async fn handle_assert_sent_verifier( return Ok(()); } - let Some(saved_verifier_state) = load_babe_setup_state(ctx.local_db, instance_id, graph_id)? - .and_then(|state| state.verifier) + let strict = false; + broadcast_verifier_challenge_assert_tx( + ctx.local_db, + ctx.btc_client, + &graph, + instance_id, + graph_id, + &assert_tx, + strict, + ) + .await?; + + Ok(()) +} + +pub async fn broadcast_verifier_challenge_assert_tx( + local_db: &LocalDB, + btc_client: &BTCClient, + graph: &BitvmGcGraph, + instance_id: Uuid, + graph_id: Uuid, + assert_tx: &bitcoin::Transaction, + strict: bool, +) -> Result> { + let verifier_master_key = VerifierMasterKey::new(get_bitvm_key()?); + let verifier_pubkey = verifier_master_key.master_keypair().public_key().into(); + let Some(verifier_index) = + find_verifier_index_by_pubkey(&graph.parameters.gc_data, &verifier_pubkey)? + else { + if strict { + bail!("local verifier has no graph slot for {instance_id}:{graph_id}"); + } + tracing::debug!( + "Ignore AssertSent for {instance_id}:{graph_id}: local verifier has no graph slot" + ); + return Ok(None); + }; + validate_verifier_slot(graph, verifier_index)?; + + let assert_txin = assert_tx + .input + .first() + .ok_or_else(|| anyhow!("operator assert transaction has no input"))?; + let assert_witness = extract_operator_assert_witness_for_challenge(graph, assert_txin) + .map_err(|e| anyhow!("failed to extract operator assert witness: {e}"))?; + let vk = crate::vk::get_vk().await.context("load Groth16 verifying key for operator assert")?; + let static_input = derive_operator_static_input()?; + + let Some(saved_verifier_state) = + load_babe_setup_state(local_db, instance_id, graph_id)?.and_then(|state| state.verifier) else { + if strict { + bail!("missing BABE verifier setup state for {instance_id}:{graph_id}"); + } tracing::warn!( "Ignore AssertSent for {instance_id}:{graph_id}: missing BABE verifier setup state" ); - return Ok(()); + return Ok(None); }; let Some(soldering_proof_ready) = saved_verifier_state.soldering_proof_ready.as_ref() else { + if strict { + bail!("missing soldering proof reference for {instance_id}:{graph_id}"); + } tracing::warn!( "Ignore AssertSent for {instance_id}:{graph_id}: missing soldering proof reference" ); - return Ok(()); + return Ok(None); }; if soldering_proof_ready.verifier_index != verifier_index { + if strict { + bail!("local setup slot does not match graph owner slot for {instance_id}:{graph_id}"); + } tracing::warn!( "Ignore AssertSent for {instance_id}:{graph_id}: local setup slot does not match graph owner slot" ); - return Ok(()); + return Ok(None); } let challenge_witness = build_real_challenge_assert_witness( &saved_verifier_state.private_state, @@ -4716,17 +5003,20 @@ async fn handle_assert_sent_verifier( .cloned() .ok_or_else(|| anyhow!("operator assert transaction has no input"))?; let challenge_assert_tx = - build_verifier_assert_tx(&graph, operator_assert_txin, verifier_index, labels)?; + build_verifier_assert_tx(graph, operator_assert_txin, verifier_index, labels)?; + let challenge_assert_txid = challenge_assert_tx.compute_txid(); + if btc_client.get_tx(&challenge_assert_txid).await?.is_some() { + tracing::info!( + "Verifier ChallengeAssert already exists for {instance_id}:{graph_id}: verifier_index={verifier_index}, txid={challenge_assert_txid}" + ); + return Ok(Some((challenge_assert_txid, verifier_index))); + } let challenge_assert_tx_total_input_amount = graph.verifier_asserts[verifier_index].prev_outs().iter().map(|o| o.value).sum::(); - broadcast_tx_with_cpfp( - ctx.btc_client, - challenge_assert_tx, - challenge_assert_tx_total_input_amount, - ) - .await?; + broadcast_tx_with_cpfp(btc_client, challenge_assert_tx, challenge_assert_tx_total_input_amount) + .await?; - Ok(()) + Ok(Some((challenge_assert_txid, verifier_index))) } // compute msg after ChallengeAssert is broadcast and broadcast WronglyChallenge transaction. @@ -4849,23 +5139,37 @@ async fn handle_challenge_assert_sent_operator( vk, dyn_pubin, )?; - let (wrongly_challenged_input, _amount) = operator_sign_wrongly_challenged( + let (wrongly_challenged_input, input_amount) = operator_sign_wrongly_challenged( &graph, verifier_index, &wrongly_challenged_witness.final_msg, )?; - let wrongly_challenged_tx = bitcoin::Transaction { - version: bitcoin::transaction::Version(2), - lock_time: bitcoin::absolute::LockTime::ZERO, - input: vec![wrongly_challenged_input], - output: vec![goat::scripts::p2a_output()], - }; - broadcast_tx(ctx.btc_client, &wrongly_challenged_tx).await?; + let operator_keypair = OperatorMasterKey::new(get_bitvm_key()?).master_keypair(); + build_sign_and_broadcast_tx( + ctx.btc_client, + operator_keypair, + vec![wrongly_challenged_input], + input_amount, + vec![ + goat::scripts::p2a_output(), + // This makes the non-witness transaction size safely exceed the + // relay minimum even before the locally funded fee input is added. + bitcoin::TxOut { + value: Amount::ZERO, + script_pubkey: goat::scripts::generate_opreturn_script( + WRONGLY_CHALLENGED_OP_RETURN_DATA.to_vec(), + ), + }, + ], + ) + .await?; Ok(()) } +const WRONGLY_CHALLENGED_OP_RETURN_DATA: &[u8] = b"wrongly-challenged"; + // broadcast NoWithdraw after the ChallengeAssert timelock expires. #[tracing::instrument(level = "info", skip_all, fields(instance_id = %instance_id, graph_id = %graph_id))] async fn handle_wrongly_challenge_timeout_verifier( @@ -5461,13 +5765,6 @@ async fn handle_sync_graph( graph: &SimplifiedBitvmGcGraph, ) -> Result<()> { // sent by relayer nodes in response to SyncGraphRequest - if graph_exists(ctx.local_db, instance_id, graph_id).await? { - tracing::warn!( - "Ignore SyncGraph for {instance_id}:{graph_id}: graph already exists locally" - ); - return Ok(()); - } - if !ctx .goat_client .committee_mana_is_validate_peer_id(&ctx.from_peer_id.to_bytes()) @@ -5486,12 +5783,15 @@ async fn handle_sync_graph( return Ok(()); } - if graph.parameters.instance_parameters.instance_id != instance_id - || graph.parameters.graph_id != graph_id - { - tracing::warn!( - "Ignore SyncGraph for {instance_id}:{graph_id}: message identifiers do not match graph parameters" - ); + if !message_identity_matches( + "SyncGraph", + instance_id, + graph_id, + None, + graph.parameters.instance_parameters.instance_id, + graph.parameters.graph_id, + graph.parameters.graph_nonce, + ) { return Ok(()); } @@ -5535,16 +5835,9 @@ async fn handle_sync_graph( return Ok(()); } let simplified_graph = graph.to_simplified()?; - store_graph(ctx.local_db, &simplified_graph).await?; - refresh_and_compensate( - ctx, - instance_id, - graph_id, - Some(&graph), - None, - GraphStatus::OperatorPresigned, - ) - .await?; + let _ = store_finalized_graph_if_needed(ctx.local_db, &simplified_graph).await?; + refresh_and_compensate(ctx, instance_id, graph_id, &graph, GraphStatus::OperatorPresigned) + .await?; Ok(()) } @@ -5792,8 +6085,8 @@ mod tests { for label in labels { witness.push(label.as_slice()); } - witness.push(&[0xde, 0xad]); // placeholder script - witness.push(&[0xbe, 0xef]); // placeholder control block + witness.push([0xde, 0xad]); // placeholder script + witness.push([0xbe, 0xef]); // placeholder control block bitcoin::Transaction { version: bitcoin::transaction::Version(2), lock_time: bitcoin::absolute::LockTime::ZERO, @@ -5856,4 +6149,29 @@ mod tests { }; assert!(recover_challenge_assert_witness(&empty_tx, 0).is_err()); } + + #[test] + fn wrongly_challenge_outputs_exceed_minimum_non_witness_size() { + // Bitcoin Core rejects standard transactions smaller than 65 non-witness bytes. + let tx = bitcoin::Transaction { + version: bitcoin::transaction::Version(2), + lock_time: bitcoin::absolute::LockTime::ZERO, + input: vec![bitcoin::TxIn::default()], + output: vec![ + goat::scripts::p2a_output(), + bitcoin::TxOut { + value: Amount::ZERO, + script_pubkey: goat::scripts::generate_opreturn_script( + WRONGLY_CHALLENGED_OP_RETURN_DATA.to_vec(), + ), + }, + ], + }; + + assert!( + tx.base_size() >= 65, + "wrongly-challenge transaction is {} non-witness bytes", + tx.base_size() + ); + } } diff --git a/node/src/main.rs b/node/src/main.rs index 4a87fb48b..1166376e1 100644 --- a/node/src/main.rs +++ b/node/src/main.rs @@ -13,6 +13,8 @@ use libp2p::PeerId; use libp2p_metrics::Registry; use std::error::Error; use std::sync::{Arc, Mutex}; +use std::time::Instant; +use tracing::Instrument; use tracing_subscriber::EnvFilter; use bitvm_noded::utils::{ @@ -121,7 +123,13 @@ async fn main() -> Result<(), Box> { } return Ok(()); } - let _ = tracing_subscriber::fmt().with_env_filter(EnvFilter::from_default_env()).try_init(); + let _ = tracing_subscriber::fmt() + .json() + .flatten_event(true) + .with_current_span(true) + .with_span_list(true) + .with_env_filter(EnvFilter::from_default_env()) + .try_init(); validate_soldering_proof_payload_store_config(&actor)?; let is_publisher = actor == Actor::Publisher || actor == Actor::All; @@ -138,6 +146,7 @@ async fn main() -> Result<(), Box> { // Create cancellation token for graceful shutdown let cancellation_token = CancellationToken::new(); let mut task_handles: Vec>> = vec![]; + let mut task_names: Vec<&'static str> = vec![]; // init bitvmswarm let bitvm_network_manager = BitvmNetworkManager::new( BitvmSwarmConfig { @@ -157,6 +166,16 @@ async fn main() -> Result<(), Box> { &mut metric_registry, )?; let peer_id_string = bitvm_network_manager.get_peer_id_string(); + let bitcoin_network = get_network().to_string(); + let node_span = tracing::info_span!( + "node", + service = "bitvm-noded", + role = %actor, + peer_id = %peer_id_string, + bitcoin_network = %bitcoin_network, + version = env!("CARGO_PKG_VERSION"), + ); + let _node_span_guard = node_span.enter(); let local_db = store::create_local_db(&opt.db_path).await; let handler = BitvmNodeProcessor { local_db: local_db.clone(), @@ -167,6 +186,17 @@ async fn main() -> Result<(), Box> { .then(|| Arc::new(BabeBundleBuilder::new())), }; + tracing::info!( + event = "service_started", + service = "bitvm-noded", + role = %actor, + peer_id = %peer_id_string, + bitcoin_network = ?get_network(), + version = env!("CARGO_PKG_VERSION"), + rpc_addr = %opt.rpc_addr, + "node initialization completed" + ); + let actor_clone1 = actor.clone(); let actor_clone2 = actor.clone(); let actor_clone3 = actor.clone(); @@ -188,48 +218,76 @@ async fn main() -> Result<(), Box> { // Spawn RPC service task with cancellation support let cancel_token_clone = cancellation_token.clone(); - task_handles.push(tokio::spawn(async move { - match rpc_service::serve( - opt_rpc_addr, - local_db_clone1, - actor_clone1, - peer_id_string_clone, - metric_registry_clone, - cancel_token_clone, - ) - .await - { - Ok(tag) => Ok(tag), - Err(e) => { - tracing::error!("RPC service error: {}", e); - Err("rpc_error".to_string()) + let rpc_task_span = node_span.clone(); + task_handles.push(tokio::spawn( + async move { + match rpc_service::serve( + opt_rpc_addr, + local_db_clone1, + actor_clone1, + peer_id_string_clone, + metric_registry_clone, + cancel_token_clone, + ) + .await + { + Ok(tag) => Ok(tag), + Err(error) => { + tracing::error!( + event = "core_task_wrapper_error", + service = "bitvm-noded", + task = "rpc_service", + outcome = "failed", + error_class = "rpc", + error = %error, + "RPC service task exited with an error" + ); + Err("rpc_error".to_string()) + } } } - })); + .instrument(rpc_task_span), + )); + task_names.push("rpc_service"); // if actor == Actor::Committee || actor == Actor::Operator { let cancel_token_clone = cancellation_token.clone(); - task_handles.push(tokio::spawn(async move { - let goat_init_config = goat_config_from_env().await; - let goat_client = Arc::new(GOATClient::new(goat_init_config.clone(), get_goat_network())); - let btc_client = Arc::new(BTCClient::new(get_network(), get_btc_url_from_env().as_deref())); - match run_watch_event_task( - actor_clone2, - local_db_clone2, - btc_client, - goat_client, - 5, - cancel_token_clone, - goat_init_config, - ) - .await - { - Ok(tag) => Ok(tag), - Err(e) => { - tracing::error!("Watch event task error: {}", e); - Err("watch_error".to_string()) + let event_watcher_task_span = node_span.clone(); + task_handles.push(tokio::spawn( + async move { + let goat_init_config = goat_config_from_env().await; + let goat_client = + Arc::new(GOATClient::new(goat_init_config.clone(), get_goat_network())); + let btc_client = + Arc::new(BTCClient::new(get_network(), get_btc_url_from_env().as_deref())); + match run_watch_event_task( + actor_clone2, + local_db_clone2, + btc_client, + goat_client, + 5, + cancel_token_clone, + goat_init_config, + ) + .await + { + Ok(tag) => Ok(tag), + Err(error) => { + tracing::error!( + event = "core_task_wrapper_error", + service = "bitvm-noded", + task = "event_watcher", + outcome = "failed", + error_class = "watcher", + error = %error, + "event watcher task exited with an error" + ); + Err("watch_error".to_string()) + } } } - })); + .instrument(event_watcher_task_span), + )); + task_names.push("event_watcher"); if is_publisher { let start_cosmos_block = sequencer_set_monitor_start_cosmos_block.unwrap(); @@ -249,85 +307,194 @@ async fn main() -> Result<(), Box> { .await { Ok(tag) => Ok(tag), - Err(e) => { - tracing::error!("Sequencer set monitor task error: {}", e); + Err(error) => { + tracing::error!("Sequencer set monitor task error: {}", error); Err("sequencer_set_monitor_error".to_string()) } } })); + task_names.push("sequencer_set_monitor"); } // } let cancel_token_clone = cancellation_token.clone(); - task_handles.push(tokio::spawn(async move { - let goat_client = - Arc::new(GOATClient::new(goat_config_from_env().await, get_goat_network())); - let btc_client = Arc::new(BTCClient::new(get_network(), get_btc_url_from_env().as_deref())); - match run_maintenance_tasks( - actor_clone3, - local_db_clone3, - btc_client, - goat_client, - 10, - cancel_token_clone, - ) - .await - { - Ok(tag) => Ok(tag), - Err(e) => { - tracing::error!("Maintenance task error: {}", e); - Err("maintenance_error".to_string()) + let maintenance_task_span = node_span.clone(); + task_handles.push(tokio::spawn( + async move { + let goat_client = + Arc::new(GOATClient::new(goat_config_from_env().await, get_goat_network())); + let btc_client = + Arc::new(BTCClient::new(get_network(), get_btc_url_from_env().as_deref())); + match run_maintenance_tasks( + actor_clone3, + local_db_clone3, + btc_client, + goat_client, + 10, + cancel_token_clone, + ) + .await + { + Ok(tag) => Ok(tag), + Err(error) => { + tracing::error!( + event = "core_task_wrapper_error", + service = "bitvm-noded", + task = "maintenance", + outcome = "failed", + error_class = "maintenance", + error = %error, + "maintenance task exited with an error" + ); + Err("maintenance_error".to_string()) + } } } - })); + .instrument(maintenance_task_span), + )); + task_names.push("maintenance"); let swarm_actor = actor.clone(); let cancel_token_clone = cancellation_token.clone(); - task_handles.push(tokio::spawn(async move { - let result = tokio::task::spawn_blocking(move || { - let rt = tokio::runtime::Handle::current(); - rt.block_on(async { - start_handle_swarm_msg_task( - swarm_actor, - bitvm_network_manager, - handler, - cancel_token_clone, - ) - .await + let p2p_task_span = node_span.clone(); + let p2p_blocking_span = p2p_task_span.clone(); + task_handles.push(tokio::spawn( + async move { + let result = tokio::task::spawn_blocking(move || { + let _span_guard = p2p_blocking_span.enter(); + let rt = tokio::runtime::Handle::current(); + rt.block_on(async { + start_handle_swarm_msg_task( + swarm_actor, + bitvm_network_manager, + handler, + cancel_token_clone, + ) + .await + }) }) - }) - .await; - match result { - Ok(tag) => Ok(tag), - Err(e) => { - tracing::error!("Swarm task spawn error: {}", e); - Err("swarm_spawn_error".to_string()) + .await; + match result { + Ok(Ok(tag)) => Ok(tag), + Ok(Err(error)) => { + tracing::error!( + event = "core_task_wrapper_error", + service = "bitvm-noded", + task = "p2p_swarm", + outcome = "failed", + error_class = "p2p", + error = %error, + "p2p swarm task exited with an error" + ); + Err("swarm_error".to_string()) + } + Err(error) => { + tracing::error!( + event = "core_task_wrapper_error", + service = "bitvm-noded", + task = "p2p_swarm", + outcome = "failed", + error_class = "join", + error = %error, + "p2p swarm blocking task failed to join" + ); + Err("swarm_spawn_error".to_string()) + } + } + } + .instrument(p2p_task_span), + )); + task_names.push("p2p_swarm"); + + let heartbeat_actor = actor.clone(); + let heartbeat_peer_id = peer_id_string.clone(); + let cancel_token_clone = cancellation_token.clone(); + let heartbeat_task_span = node_span.clone(); + task_handles.push(tokio::spawn( + async move { + let started_at = Instant::now(); + loop { + tokio::select! { + _ = tokio::time::sleep(tokio::time::Duration::from_secs(60)) => { + tracing::info!( + event = "service_heartbeat", + service = "bitvm-noded", + role = %heartbeat_actor, + peer_id = %heartbeat_peer_id, + uptime_secs = started_at.elapsed().as_secs(), + "node service heartbeat" + ); + } + _ = cancel_token_clone.cancelled() => { + return Ok("heartbeat stopped after cancellation".to_string()); + } + } } } - })); + .instrument(heartbeat_task_span), + )); + task_names.push("heartbeat"); // Wait for shutdown signal or any task completion let task_count = task_handles.len(); + tracing::info!( + event = "service_ready", + service = "bitvm-noded", + role = %actor, + peer_id = %peer_id_string, + task_count, + "all node background tasks have been started" + ); tokio::select! { (result, index, remaining_handles) = future::select_all(task_handles) => { + let task_name = task_names[index]; // Log the specific failure let failure_reason = match &result { Ok(Ok(tag)) => { - tracing::warn!("Task {} completed unexpectedly: {}", index, tag); + tracing::warn!( + event = "core_task_result", + service = "bitvm-noded", + task = task_name, + outcome = "unexpected_completion", + detail = %tag, + "node background task completed unexpectedly" + ); "unexpected completion" } Ok(Err(error)) => { - tracing::error!("Task {} failed with business error: {}", index, error); + tracing::error!( + event = "core_task_result", + service = "bitvm-noded", + task = task_name, + outcome = "business_error", + error = %error, + "node background task failed" + ); "business error" } Err(join_error) => { - tracing::error!("Task {} failed with join error: {}", index, join_error); + tracing::error!( + event = "core_task_result", + service = "bitvm-noded", + task = task_name, + outcome = "join_error", + error = %join_error, + "node background task failed to join" + ); "join error" } }; - tracing::info!("Triggering shutdown due to {} in task {}/{}", failure_reason, index + 1, task_count); + tracing::info!( + event = "service_shutdown", + service = "bitvm-noded", + trigger = "core_task_result", + task = task_name, + reason = failure_reason, + task_count, + "triggering node shutdown after background task result" + ); // Initiate graceful shutdown cancellation_token.cancel(); @@ -338,7 +505,12 @@ async fn main() -> Result<(), Box> { // Force abort any tasks that didn't respond to cancellation remaining_handles.into_iter().for_each(|handle| handle.abort()); - tracing::info!("All tasks stopped"); + tracing::info!( + event = "service_shutdown", + service = "bitvm-noded", + outcome = "tasks_stopped", + "all node background tasks stopped" + ); // Handle panic propagation if let Err(join_error) = result && join_error.is_panic() { @@ -347,12 +519,23 @@ async fn main() -> Result<(), Box> { } } _ = shutdown_signal() => { - tracing::info!("Received shutdown signal, initiating graceful shutdown..."); + tracing::info!( + event = "service_shutdown", + service = "bitvm-noded", + outcome = "started", + trigger = "signal", + "received shutdown signal; initiating graceful shutdown" + ); cancellation_token.cancel(); // Give tasks some time to shutdown gracefully tokio::time::sleep(tokio::time::Duration::from_secs(2)).await; - tracing::info!("Graceful shutdown completed"); + tracing::info!( + event = "service_shutdown", + service = "bitvm-noded", + outcome = "completed", + "node graceful shutdown completed" + ); } } @@ -378,10 +561,18 @@ async fn shutdown_signal() { tokio::select! { _ = ctrl_c => { - tracing::info!("Received Ctrl+C signal, starting graceful shutdown..."); + tracing::info!( + event = "service_shutdown_signal", + signal = "SIGINT", + "received Ctrl+C signal" + ); }, _ = terminate => { - tracing::info!("Received SIGTERM signal, starting graceful shutdown..."); + tracing::info!( + event = "service_shutdown_signal", + signal = "SIGTERM", + "received SIGTERM signal" + ); }, } } @@ -391,9 +582,6 @@ pub async fn start_handle_swarm_msg_task( mut swarm: BitvmNetworkManager, handler: BitvmNodeProcessor, cancellation_token: CancellationToken, -) -> String { - swarm.run(actor, handler, cancellation_token).await.unwrap_or_else(|e| { - tracing::error!("Swarm run error: {}", e); - "swarm_error".to_string() - }) +) -> anyhow::Result { + swarm.run(actor, handler, cancellation_token).await } diff --git a/node/src/metrics_service.rs b/node/src/metrics_service.rs index e0a2570d8..3daf4ea4c 100644 --- a/node/src/metrics_service.rs +++ b/node/src/metrics_service.rs @@ -120,6 +120,7 @@ pub async fn metrics_middleware( request.uri().path().to_owned() }; let method = request.method().to_string(); + state.metrics_state.http_requests_in_flight.inc(); let response = next.run(request).await; state.metrics_state.http_requests_in_flight.dec(); let status = response.status().as_u16(); diff --git a/node/src/rpc_service/bitvm.rs b/node/src/rpc_service/bitvm.rs index 15f0fc722..851b2d45d 100644 --- a/node/src/rpc_service/bitvm.rs +++ b/node/src/rpc_service/bitvm.rs @@ -100,6 +100,12 @@ pub struct SendChallengeResponse { pub challenge_txid: String, } +#[derive(Debug, Deserialize, Serialize)] +pub struct SendVerifierChallengeResponse { + pub challenge_assert_txid: String, + pub verifier_index: usize, +} + #[derive(Debug, Deserialize, Serialize)] pub struct PegoutRequest { pub graph_id: Option, diff --git a/node/src/rpc_service/handler/bitvm2_handler.rs b/node/src/rpc_service/handler/bitvm2_handler.rs index ad4b9d90c..834964d79 100644 --- a/node/src/rpc_service/handler/bitvm2_handler.rs +++ b/node/src/rpc_service/handler/bitvm2_handler.rs @@ -3,6 +3,7 @@ use crate::env::{ get_goat_address_from_env, get_goat_gateway_contract_from_env, get_network, get_node_goat_address, get_node_pubkey, }; +use crate::handle::broadcast_verifier_challenge_assert_tx; use crate::rpc_service::auth::verify_request_auth; use crate::rpc_service::bitvm::*; use crate::rpc_service::node::ALIVE_TIME_JUDGE_THRESHOLD; @@ -12,23 +13,23 @@ use crate::rpc_service::response::{ use crate::rpc_service::validation::InputValidator; use crate::rpc_service::{AppState, current_time_secs}; use crate::utils::{ - find_instances_by_escrow_hash, gen_instance_parameters_local, get_bridge_out_global_stats, - parse_graph_raw_data, send_challenge_tx, + bridge_out_instance_id_from_escrow_hash, find_instances_by_escrow_hash, + gen_instance_parameters_local, get_bridge_out_global_stats, load_validated_graph_definition, + send_challenge_tx, }; use alloy::primitives::{Address, U256}; use axum::Json; use axum::extract::{Path, Query, State}; use bitcoin::consensus::encode::serialize_hex; -use bitvm_lib::types::{BitvmGcGraph, SimplifiedBitvmGcGraph}; +use bitvm_lib::types::BitvmGcGraph; use client::goat_chain::{PeginStatus, WithdrawStatus}; use goat::transactions::pre_signed::PreSignedTransaction; use http::{HeaderMap, StatusCode}; -use sha2::{Digest, Sha256}; use std::default::Default; use std::str::FromStr; use std::sync::Arc; use std::time::{SystemTime, UNIX_EPOCH}; -use store::localdb::{GraphQuery, InstanceQuery, StorageProcessor}; +use store::localdb::{GraphQuery, InstanceQuery, InstanceUpdate, StorageProcessor}; use store::{ GoatTxType, Graph, GraphStatus, Instance, InstanceBridgeInStatus, InstanceBridgeOutStatus, }; @@ -36,19 +37,6 @@ use tokio::time::{Duration, sleep}; use tracing::warn; use uuid::Uuid; -const BRIDGE_OUT_INSTANCE_ID_PREFIX: [u8; 4] = *b"BOID"; - -fn bridge_out_instance_id_from_escrow_hash(escrow_hash: &str) -> Uuid { - let mut hasher = Sha256::new(); - hasher.update(b"bridge-out:"); - hasher.update(escrow_hash.as_bytes()); - let digest = hasher.finalize(); - let mut bytes = [0u8; 16]; - bytes.copy_from_slice(&digest[..16]); - bytes[..4].copy_from_slice(&BRIDGE_OUT_INSTANCE_ID_PREFIX); - Uuid::from_bytes(bytes) -} - fn bridge_out_retry_jitter_ms(attempt: u32) -> u64 { let now = SystemTime::now() .duration_since(UNIX_EPOCH) @@ -306,51 +294,31 @@ pub async fn bridge_out_init_tag( let current_time = current_time_secs(); for attempt in 0..=MAX_BRIDGE_OUT_INIT_RETRIES { - if let Some(mut instance) = - find_instances_by_escrow_hash(&mut storage_process, &escrow_hash) - .await - .api_error("PUT_BRIDGE_OUT_INIT_TAG_ERROR")? - { - if instance.status == InstanceBridgeOutStatus::Initialize.to_string() { - instance.to_addr = payload.to_addr.clone(); - instance.network = get_network().to_string(); - storage_process - .upsert_instance(&instance) - .await - .api_error("PUT_BRIDGE_OUT_INIT_TAG_ERROR")?; - } - return ok_response(BridgeOutInitTagResponse {}); - } - - let candidate_instance_id = if attempt == 0 { - bridge_out_instance_id_from_escrow_hash(&escrow_hash) - } else { - Uuid::new_v4() - }; - - if let Some(existing) = storage_process - .find_instance(&candidate_instance_id) + if let Some(instance) = find_instances_by_escrow_hash(&mut storage_process, &escrow_hash) .await .api_error("PUT_BRIDGE_OUT_INIT_TAG_ERROR")? { - let escrow_hash_matches = existing - .escrow_hash - .as_ref() - .map(|hash| hash.eq_ignore_ascii_case(&escrow_hash)) - .unwrap_or(false); - if !existing.is_bridge_in && escrow_hash_matches { - continue; - } - if attempt < MAX_BRIDGE_OUT_INIT_RETRIES { - sleep(Duration::from_millis(bridge_out_retry_jitter_ms(attempt))).await; - continue; + let updated = storage_process + .update_instance( + &InstanceUpdate::new_with_instance_id(instance.instance_id) + .with_to_addr(payload.to_addr.clone()) + .with_only_if_status_in(vec![ + InstanceBridgeOutStatus::Initialize.to_string(), + ]) + .with_only_if_is_bridge_in(false), + ) + .await + .api_error("PUT_BRIDGE_OUT_INIT_TAG_ERROR")?; + if !updated { + warn!( + "bridge_out_init_tag ignored for resolved instance {} with status {}", + instance.instance_id, instance.status + ); } - return error_response( - "BRIDGE_OUT_INSTANCE_ID_CONFLICT".to_string(), - "failed to allocate bridge-out instance_id for escrow_hash".to_string(), - ); + return ok_response(BridgeOutInitTagResponse {}); } + let candidate_instance_id = bridge_out_instance_id_from_escrow_hash(&escrow_hash); let mut instance = Instance { instance_id: candidate_instance_id, from_addr: from_addr.clone(), @@ -365,12 +333,58 @@ pub async fn bridge_out_init_tag( }; instance.to_addr = payload.to_addr.clone(); - match storage_process.upsert_instance(&instance).await { - Ok(_) => return ok_response(BridgeOutInitTagResponse {}), + match storage_process.insert_instance_if_absent(&instance).await { + Ok(true) => return ok_response(BridgeOutInitTagResponse {}), + Ok(false) => { + let Some(existing) = storage_process + .find_instance(&candidate_instance_id) + .await + .api_error("PUT_BRIDGE_OUT_INIT_TAG_ERROR")? + else { + if attempt < MAX_BRIDGE_OUT_INIT_RETRIES { + sleep(Duration::from_millis(bridge_out_retry_jitter_ms(attempt))).await; + continue; + } + return error_response( + "BRIDGE_OUT_INSTANCE_ID_CONFLICT".to_string(), + "bridge-out instance disappeared while being created".to_string(), + ); + }; + let escrow_hash_matches = existing + .escrow_hash + .as_ref() + .map(|hash| hash.eq_ignore_ascii_case(&escrow_hash)) + .unwrap_or(false); + if existing.is_bridge_in || !escrow_hash_matches { + return error_response( + "BRIDGE_OUT_INSTANCE_ID_CONFLICT".to_string(), + "failed to allocate bridge-out instance_id for escrow_hash".to_string(), + ); + } + + let updated = storage_process + .update_instance( + &InstanceUpdate::new_with_instance_id(existing.instance_id) + .with_to_addr(payload.to_addr.clone()) + .with_only_if_status_in(vec![ + InstanceBridgeOutStatus::Initialize.to_string(), + ]) + .with_only_if_is_bridge_in(false), + ) + .await + .api_error("PUT_BRIDGE_OUT_INIT_TAG_ERROR")?; + if !updated { + warn!( + "bridge_out_init_tag ignored for concurrently resolved instance {} with status {}", + existing.instance_id, existing.status + ); + } + return ok_response(BridgeOutInitTagResponse {}); + } Err(err) => { if attempt < MAX_BRIDGE_OUT_INIT_RETRIES { warn!( - "bridge_out_init_tag upsert failed at attempt {}, retrying: {}", + "bridge_out_init_tag insert failed at attempt {}, retrying: {}", attempt, err ); sleep(Duration::from_millis(bridge_out_retry_jitter_ms(attempt))).await; @@ -1202,22 +1216,27 @@ pub async fn get_graph_tx( let mut storage_process = app_state.local_db.acquire().await.api_error("GET_GRAPH_TX_ERROR")?; - let graph_raw_data = storage_process - .find_graph_raw_data(&graph_id_uuid) - .await - .api_error("GET_GRAPH_TX_ERROR")?; let graph = storage_process.find_graph(&graph_id_uuid).await.api_error("GET_GRAPH_TX_ERROR")?; - if let (Some(graph_raw_data), Some(graph)) = (graph_raw_data, graph) { + if let Some(graph) = graph { let (progresses, fail_reason) = get_graph_btc_tx_process_data(&mut storage_process, tx_name, &graph) .await .api_error("GET_GRAPH_TX_ERROR")?; - let simplified_bitvm_graph: SimplifiedBitvmGcGraph = - parse_graph_raw_data(graph_raw_data.raw_data.clone(), graph_id_uuid) + let simplified_bitvm_graph = + load_validated_graph_definition(&mut storage_process, graph.instance_id, graph_id_uuid) .await - .api_error("GET_GRAPH_TXN_ERROR")?; + .api_error("GET_GRAPH_TXN_ERROR")? + .ok_or_else(|| { + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(ErrorResponse { + error: "GET_GRAPH_TXN_ERROR".to_string(), + message: format!("graph:{graph_id} raw data is not record in db"), + }), + ) + })?; let bitvm_graph: BitvmGcGraph = BitvmGcGraph::from_simplified(&simplified_bitvm_graph) .api_error("GET_GRAPH_TX_ERROR")?; @@ -1368,26 +1387,25 @@ pub async fn get_graph_txn( graph }; - let graph_raw_data = storage_processor - .find_graph_raw_data(&graph.graph_id) - .await - .api_error("GET_GRAPH_TXN_ERROR")? - .ok_or_else(|| { - ( - StatusCode::INTERNAL_SERVER_ERROR, - Json(ErrorResponse { - error: "GET_GRAPH_TXN_ERROR".to_string(), - message: format!( - "graph:{graph_id} with cursor:{} raw data is not record in db", - params.cursor - ), - }), - ) - })?; - let simplified_bitvm_graph: SimplifiedBitvmGcGraph = - parse_graph_raw_data(graph_raw_data.raw_data.clone(), graph.graph_id) - .await - .api_error("GET_GRAPH_TXN_ERROR")?; + let simplified_bitvm_graph = load_validated_graph_definition( + &mut storage_processor, + graph.instance_id, + graph.graph_id, + ) + .await + .api_error("GET_GRAPH_TXN_ERROR")? + .ok_or_else(|| { + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(ErrorResponse { + error: "GET_GRAPH_TXN_ERROR".to_string(), + message: format!( + "graph:{graph_id} with cursor:{} raw data is not record in db", + params.cursor + ), + }), + ) + })?; let bitvm_graph: BitvmGcGraph = BitvmGcGraph::from_simplified(&simplified_bitvm_graph) .api_error("GET_GRAPH_TXN_ERROR")?; @@ -1616,8 +1634,8 @@ pub async fn send_challenge( let mut storage_process = app_state.local_db.acquire().await.api_error("SEND_CHALLENGE_ERROR")?; - let graph_raw_data = storage_process - .find_graph_raw_data(&graph_id_uuid) + let graph = storage_process + .find_graph(&graph_id_uuid) .await .api_error("SEND_CHALLENGE_ERROR")? .ok_or_else(|| { @@ -1625,15 +1643,24 @@ pub async fn send_challenge( StatusCode::INTERNAL_SERVER_ERROR, Json(ErrorResponse { error: "SEND_CHALLENGE_ERROR".to_string(), - message: format!("graph:{graph_id} raw data not found in db"), + message: format!("graph:{graph_id} not found in db"), }), ) })?; - let simplified_bitvm_graph: SimplifiedBitvmGcGraph = - parse_graph_raw_data(graph_raw_data.raw_data, graph_id_uuid) + let simplified_bitvm_graph = + load_validated_graph_definition(&mut storage_process, graph.instance_id, graph_id_uuid) .await - .api_error("SEND_CHALLENGE_ERROR")?; + .api_error("SEND_CHALLENGE_ERROR")? + .ok_or_else(|| { + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(ErrorResponse { + error: "SEND_CHALLENGE_ERROR".to_string(), + message: format!("graph:{graph_id} raw data not found in db"), + }), + ) + })?; let bitvm_graph: BitvmGcGraph = BitvmGcGraph::from_simplified(&simplified_bitvm_graph).api_error("SEND_CHALLENGE_ERROR")?; @@ -1645,6 +1672,99 @@ pub async fn send_challenge( ok_response(SendChallengeResponse { challenge_txid: txid.to_string() }) } +/// Force the local verifier to broadcast its ChallengeAssert transaction for a graph. +/// +/// This is a test endpoint. Unlike the normal AssertSent verifier flow, it does +/// not skip ChallengeAssert when the operator assert proof is valid. +#[axum::debug_handler] +pub async fn send_verifier_challenge( + headers: HeaderMap, + Path(graph_id): Path, + State(app_state): State>, +) -> ApiResult { + verify_request_auth(&headers)?; + let graph_id_uuid = InputValidator::validate_uuid(&graph_id, "graph_id")?; + + let mut storage_process = + app_state.local_db.acquire().await.api_error("SEND_VERIFIER_CHALLENGE_ERROR")?; + + let graph = storage_process + .find_graph(&graph_id_uuid) + .await + .api_error("SEND_VERIFIER_CHALLENGE_ERROR")? + .ok_or_else(|| { + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(ErrorResponse { + error: "SEND_VERIFIER_CHALLENGE_ERROR".to_string(), + message: format!("graph:{graph_id} not found in db"), + }), + ) + })?; + + let simplified_bitvm_graph = + load_validated_graph_definition(&mut storage_process, graph.instance_id, graph_id_uuid) + .await + .api_error("SEND_VERIFIER_CHALLENGE_ERROR")? + .ok_or_else(|| { + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(ErrorResponse { + error: "SEND_VERIFIER_CHALLENGE_ERROR".to_string(), + message: format!("graph:{graph_id} raw data not found in db"), + }), + ) + })?; + + let bitvm_graph: BitvmGcGraph = BitvmGcGraph::from_simplified(&simplified_bitvm_graph) + .api_error("SEND_VERIFIER_CHALLENGE_ERROR")?; + + let assert_txid = bitvm_graph.operator_assert.tx().compute_txid(); + let assert_tx = app_state + .btc_client + .get_tx(&assert_txid) + .await + .api_error("SEND_VERIFIER_CHALLENGE_ERROR")? + .ok_or_else(|| { + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(ErrorResponse { + error: "SEND_VERIFIER_CHALLENGE_ERROR".to_string(), + message: format!("operator assert tx {assert_txid} not found on Bitcoin"), + }), + ) + })?; + + let strict = true; + let (challenge_assert_txid, verifier_index) = broadcast_verifier_challenge_assert_tx( + &app_state.local_db, + &app_state.btc_client, + &bitvm_graph, + simplified_bitvm_graph.parameters.instance_parameters.instance_id, + graph_id_uuid, + &assert_tx, + strict, + ) + .await + .api_error("SEND_VERIFIER_CHALLENGE_ERROR")? + .ok_or_else(|| { + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(ErrorResponse { + error: "SEND_VERIFIER_CHALLENGE_ERROR".to_string(), + message: format!( + "local verifier cannot broadcast ChallengeAssert for graph {graph_id}" + ), + }), + ) + })?; + + ok_response(SendVerifierChallengeResponse { + challenge_assert_txid: challenge_assert_txid.to_string(), + verifier_index, + }) +} + const BTC_DECIMALS: u8 = 8; fn sats_to_token_amount(amount_sats: u64, token_decimals: u8) -> U256 { diff --git a/node/src/rpc_service/mod.rs b/node/src/rpc_service/mod.rs index 8e8d5208a..8da2ccc05 100644 --- a/node/src/rpc_service/mod.rs +++ b/node/src/rpc_service/mod.rs @@ -16,11 +16,11 @@ use crate::rpc_service::handler::{ get_graph_neighbor_ids, get_graph_tx, get_graph_txn, get_graphs, get_instance, get_instance_escrow_data, get_instances, get_instances_overview, get_node, get_nodes, get_nodes_overview, get_operator_proof_desc, get_ready_to_kickoff_graph, - get_unsigned_pegin_txn, instance_settings, pegout, send_challenge, + get_unsigned_pegin_txn, instance_settings, pegout, send_challenge, send_verifier_challenge, }; +use anyhow::Context; use axum::body::Body; use axum::extract::Request; -use axum::middleware::Next; use axum::response::Response; use axum::routing::put; use axum::{ @@ -31,8 +31,6 @@ use bitvm_lib::actors::Actor; use client::btc_chain::BTCClient; use client::goat_chain::GOATClient; use client::http_client::async_client::HttpAsyncClient; -use http::{HeaderMap, StatusCode}; -use http_body_util::BodyExt; use prometheus_client::registry::Registry; use std::sync::{Arc, Mutex}; use std::time::{Duration, UNIX_EPOCH}; @@ -41,8 +39,7 @@ use tokio::net::TcpListener; use tokio_util::sync::CancellationToken; use tower_http::classify::ServerErrorsFailureClass; use tower_http::cors::CorsLayer; -use tower_http::trace::{DefaultMakeSpan, TraceLayer}; -use tracing::Level; +use tower_http::trace::TraceLayer; #[inline(always)] pub fn current_time_secs() -> i64 { @@ -144,6 +141,7 @@ pub async fn serve_with_app_state( app_state: Arc, cancellation_token: CancellationToken, ) -> anyhow::Result { + let node_span = tracing::Span::current(); let server = Router::new() .route(routes::ROOT, get(root)) .route(routes::v1::NODES_BASE, get(get_nodes)) @@ -164,45 +162,66 @@ pub async fn serve_with_app_state( .route(routes::v1::GRAPHS_TX_BY_ID, get(get_graph_tx)) .route(routes::v1::GRAPHS_NEIGHBOR_IDS, get(get_graph_neighbor_ids)) .route(routes::v1::GRAPHS_SEND_CHALLENGE, post(send_challenge)) + .route(routes::v1::GRAPHS_SEND_VERIFIER_CHALLENGE, post(send_verifier_challenge)) .route(routes::v1::PEGOUT, post(pegout)) .route(routes::v1::PROOFS_CHAIN_PROOFS_DESC, get(get_chain_proof_desc)) .route(routes::v1::PROOFS_OPERATOR_PROOF_DESC, get(get_operator_proof_desc)) .route(routes::METRICS, get(metrics_handler)) - .layer(middleware::from_fn(print_req_and_resp_detail)) .layer(create_secure_cors_layer()) .layer( TraceLayer::new_for_http() - .make_span_with(DefaultMakeSpan::new().level(Level::INFO)) + .make_span_with(move |request: &Request| { + tracing::info_span!( + parent: &node_span, + "http_request", + method = %request.method(), + path = request.uri().path(), + version = ?request.version(), + ) + }) .on_request(|request: &Request, _span: &tracing::Span| { tracing::info!( - "API Request: {} {}, {:?}, Headers: {:?}, Content-Type: {:?}", - request.method(), - request.uri(), - request.version(), - request.headers(), - request.headers().get("content-type") + event = "http_request", + method = %request.method(), + path = request.uri().path(), + content_type = ?request.headers().get("content-type"), + "RPC request received" ); }) .on_response( |response: &Response, latency: Duration, _span: &tracing::Span| { tracing::info!( - "API Response: - Status: {} - Latency: {:?}", - response.status(), - latency + event = "http_response", + status = %response.status(), + elapsed_ms = latency.as_millis() as u64, + "RPC response sent" ); }, ) .on_failure( - |error: ServerErrorsFailureClass, _latency: Duration, _span: &tracing::Span| { - tracing::error!("API Error: {:?}", error); + |error: ServerErrorsFailureClass, latency: Duration, _span: &tracing::Span| { + tracing::error!( + event = "http_request_failure", + error_class = ?error, + elapsed_ms = latency.as_millis() as u64, + "RPC request failed" + ); }, ), ) .layer(middleware::from_fn_with_state(app_state.clone(), metrics_middleware)) .with_state(app_state); - let listener = TcpListener::bind(addr).await.unwrap(); - tracing::info!("RPC listening on {}", listener.local_addr().unwrap()); + let listener = TcpListener::bind(&addr) + .await + .with_context(|| format!("failed to bind RPC listener to {addr}"))?; + let listening_addr = + listener.local_addr().context("failed to determine RPC listener address")?; + tracing::info!( + event = "rpc_listening", + address = %listening_addr, + "RPC listener started" + ); tokio::select! { result = axum::serve(listener, server) => { @@ -233,39 +252,6 @@ pub async fn serve( serve_with_app_state(addr, app_state, cancellation_token).await } -/// This method introduces performance overhead and is temporarily used for debugging with the frontend. -/// It will be removed afterwards. -async fn print_req_and_resp_detail( - _headers: HeaderMap, - req: Request, - next: Next, -) -> Result { - // TODO remove after the service stabilizes. - let mut print_str = format!( - "API Request: method:{}, uri:{}, content_type:{:?}, body:", - req.method(), - req.uri(), - req.headers().get("content-type") - ); - let (parts, body) = req.into_parts(); - let bytes = body.collect().await.unwrap().to_bytes(); - if !bytes.is_empty() { - print_str = format!("{print_str} {}", String::from_utf8_lossy(&bytes)); - } - tracing::debug!("{}", print_str); - let req = Request::from_parts(parts, axum::body::Body::from(bytes)); - let resp = next.run(req).await; - - let mut print_str = format!("API Response: status:{}, body:", resp.status(),); - let (parts, body) = resp.into_parts(); - let bytes = body.collect().await.unwrap().to_bytes(); - if !bytes.is_empty() { - print_str = format!("{print_str} {}", String::from_utf8_lossy(&bytes)); - } - tracing::debug!("{}", print_str); - Ok(Response::from_parts(parts, axum::body::Body::from(bytes))) -} - #[cfg(test)] mod tests { use crate::env::{ @@ -291,8 +277,11 @@ mod tests { use std::str::FromStr; use std::sync::{Arc, Mutex}; use std::time::Duration; - use store::localdb::LocalDB; - use store::{Graph, GraphStatus, Instance, InstanceBridgeInStatus, Node, create_local_db}; + use store::localdb::{GraphRuntimeUpdate, LocalDB, StorageProcessor}; + use store::{ + Graph, GraphStatus, GraphStatusSource, Instance, InstanceBridgeInStatus, Node, + create_local_db, + }; use tokio::time::sleep; use tokio_util::sync::CancellationToken; use tracing::{error, info}; @@ -398,6 +387,54 @@ mod tests { Ok(()) } + async fn seed_graph_runtime( + tx: &mut StorageProcessor<'_>, + graph: &Graph, + ) -> anyhow::Result<()> { + let target_status = GraphStatus::from_str(&graph.status)?; + let sub_status = graph.sub_status.clone(); + let challenge_txid = graph.challenge_txid.clone(); + let init_withdraw_tx_hash = graph.init_withdraw_tx_hash.clone(); + let bridge_out_start_at = graph.bridge_out_start_at; + let proceed_withdraw_height = graph.proceed_withdraw_height; + + let mut definition = graph.clone(); + definition.status = GraphStatus::OperatorPresigned.to_string(); + definition.sub_status.clear(); + definition.challenge_txid = None; + definition.init_withdraw_tx_hash = None; + definition.bridge_out_start_at = 0; + definition.proceed_withdraw_height = 0; + tx.upsert_graph_definition(&definition).await?; + + if target_status != GraphStatus::OperatorPresigned { + tx.transition_graph_status( + graph.instance_id, + graph.graph_id, + target_status, + GraphStatusSource::ChainReconcile, + (!sub_status.is_empty()).then_some(sub_status), + ) + .await?; + } + + let mut runtime = GraphRuntimeUpdate::new(graph.instance_id, graph.graph_id); + if let Some(challenge_txid) = challenge_txid { + runtime = runtime.with_challenge_txid(challenge_txid); + } + if let Some(init_withdraw_tx_hash) = init_withdraw_tx_hash { + runtime = runtime.with_init_withdraw_tx_hash(init_withdraw_tx_hash); + } + if bridge_out_start_at != 0 { + runtime = runtime.with_bridge_out_start_at(bridge_out_start_at); + } + if proceed_withdraw_height != 0 { + runtime = runtime.with_proceed_withdraw_height(proceed_withdraw_height); + } + tx.update_graph_runtime(&runtime).await?; + Ok(()) + } + async fn init_instance_graph_data( local_db: &LocalDB, instances: &[Instance], @@ -408,7 +445,7 @@ mod tests { tx.upsert_instance(instance).await?; } for graph in graphs { - tx.upsert_graph(graph).await?; + seed_graph_runtime(&mut tx, graph).await?; } tx.commit().await?; Ok(()) @@ -594,6 +631,7 @@ mod tests { status: graph_status.clone(), sub_status: "".to_string(), operator_pubkey: "".to_string(), + definition_hash: format!("fixture-{graph_id}"), next_prekickoff: None, cur_prekickoff_txid: None, force_skip_kickoff_txid: None, @@ -618,8 +656,9 @@ mod tests { created_at: current_time_secs(), updated_at: current_time_secs(), }); + let finalized_graph_id = Uuid::new_v4(); graphs.push(Graph { - graph_id: Uuid::new_v4(), + graph_id: finalized_graph_id, instance_id: bridge_in_instance_id, kickoff_index: 0, from_addr: graph_from.clone(), @@ -629,6 +668,7 @@ mod tests { status: GraphStatus::CommitteePresigned.to_string(), sub_status: "".to_string(), operator_pubkey: "".to_string(), + definition_hash: format!("fixture-{finalized_graph_id}"), next_prekickoff: None, cur_prekickoff_txid: None, force_skip_kickoff_txid: None, diff --git a/node/src/rpc_service/routes.rs b/node/src/rpc_service/routes.rs index 1b023c261..5b4b082be 100644 --- a/node/src/rpc_service/routes.rs +++ b/node/src/rpc_service/routes.rs @@ -21,6 +21,7 @@ pub(crate) mod v1 { pub const GRAPHS_NEIGHBOR_IDS: &str = "/v1/graphs/{:id}/neighbor-ids"; pub const GRAPHS_TX_BY_ID: &str = "/v1/graphs/{:id}/tx"; pub const GRAPHS_SEND_CHALLENGE: &str = "/v1/graphs/{:id}/send-challenge"; + pub const GRAPHS_SEND_VERIFIER_CHALLENGE: &str = "/v1/graphs/{:id}/send-verifier-challenge"; pub const PEGOUT: &str = "/v1/graphs/pegout"; // pub const PROOFS_BASE: &str = "/v1/proofs"; pub const PROOFS_CHAIN_PROOFS_DESC: &str = "/v1/proofs/chain_proofs_desc"; diff --git a/node/src/scheduled_tasks/event_watch_task.rs b/node/src/scheduled_tasks/event_watch_task.rs index ca8c344e6..3cab9ae64 100644 --- a/node/src/scheduled_tasks/event_watch_task.rs +++ b/node/src/scheduled_tasks/event_watch_task.rs @@ -11,8 +11,9 @@ use crate::scheduled_tasks::get_timestamp_from_contract_data; use crate::utils::evm_swap_utils::IEscrowManager::EscrowData; use crate::utils::evm_swap_utils::{extract_claim_data_from_tx, extract_escrow_data_from_tx}; use crate::utils::{ - GenerateInstanceParams, find_instances_by_escrow_hash, generate_instance, - get_bridge_out_global_stats, outpoint_available, reflect_goat_address, strip_hex_prefix_owned, + GenerateInstanceParams, bridge_out_instance_id_from_escrow_hash, find_instances_by_escrow_hash, + generate_instance, get_bridge_out_global_stats, outpoint_available, reflect_goat_address, + strip_hex_prefix_owned, }; use alloy::primitives::{Address as EvmAddress, U256}; use alloy::sol_types::{SolType, SolValue}; @@ -39,11 +40,11 @@ use std::ops::AddAssign; use std::str::FromStr; use std::sync::Arc; use std::time::Duration; -use store::localdb::{GraphUpdate, InstanceUpdate, LocalDB, NodeQuery, StorageProcessor}; +use store::localdb::{GraphRuntimeUpdate, InstanceUpdate, LocalDB, NodeQuery, StorageProcessor}; use store::{ - GoatTxProcessingStatus, GoatTxRecord, GoatTxType, GraphStatus, Instance, - InstanceBridgeInStatus, InstanceBridgeOutStatus, MessageState, WatchContract, - WatchContractStatus, + GoatTxProcessingStatus, GoatTxRecord, GoatTxType, GraphStatus, GraphStatusSource, + GraphStatusTransitionOutcome, Instance, InstanceBridgeInStatus, InstanceBridgeOutStatus, + MessageState, WatchContract, WatchContractStatus, }; use tokio::time::sleep; use tokio_util::sync::CancellationToken; @@ -292,13 +293,20 @@ async fn handle_user_withdraw_events<'a>( UserGraphWithdrawEvent::InitWithdraw(init_event) => { let instance_id = Uuid::from_str(&strip_hex_prefix_owned(&init_event.instance_id))?; let graph_id = Uuid::from_str(&strip_hex_prefix_owned(&init_event.graph_id))?; - storage_processor - .update_graph( - &GraphUpdate::new(graph_id) + if !graph_belongs_to_instance(storage_processor, instance_id, graph_id).await? { + continue; + } + if !storage_processor + .update_graph_runtime( + &GraphRuntimeUpdate::new(instance_id, graph_id) .with_bridge_out_start_at(current_time_secs()) .with_init_withdraw_tx_hash(init_event.transaction_hash.clone()), ) - .await?; + .await? + { + warn!("ignore InitWithdraw for missing graph {instance_id}:{graph_id}"); + continue; + } storage_processor .upsert_goat_tx_record(&GoatTxRecord { instance_id, @@ -317,13 +325,20 @@ async fn handle_user_withdraw_events<'a>( let instance_id = Uuid::from_str(&strip_hex_prefix_owned(&cancel_event.instance_id))?; let graph_id = Uuid::from_str(&strip_hex_prefix_owned(&cancel_event.graph_id))?; - storage_processor - .update_graph( - &GraphUpdate::new(graph_id) + if !graph_belongs_to_instance(storage_processor, instance_id, graph_id).await? { + continue; + } + if !storage_processor + .update_graph_runtime( + &GraphRuntimeUpdate::new(instance_id, graph_id) .with_bridge_out_start_at(0) .with_init_withdraw_tx_hash("".to_string()), ) - .await?; + .await? + { + warn!("ignore CancelWithdraw for missing graph {instance_id}:{graph_id}"); + continue; + } storage_processor .upsert_goat_tx_record(&GoatTxRecord { instance_id, @@ -356,25 +371,33 @@ async fn handle_proceed_withdraw_events<'a>( for event in proceed_withdraw_events { let graph_id = Uuid::from_str(&strip_hex_prefix_owned(&event.graph_id))?; let instance_id = Uuid::from_str(&strip_hex_prefix_owned(&event.instance_id))?; + if !graph_belongs_to_instance(storage_processor, instance_id, graph_id).await? { + continue; + } + let height = event.block_number.parse::()?; + if !storage_processor + .update_graph_runtime( + &GraphRuntimeUpdate::new(instance_id, graph_id) + .with_proceed_withdraw_height(height), + ) + .await? + { + warn!("ignore ProceedWithdraw for missing graph {instance_id}:{graph_id}"); + continue; + } storage_processor .upsert_goat_tx_record(&GoatTxRecord { instance_id, graph_id, tx_type: GoatTxType::ProceedWithdraw.to_string(), tx_hash: event.transaction_hash, - height: event.block_number.parse::()?, + height, is_local: false, processing_status: processing_status.clone(), extra: None, created_at: current_time_secs(), }) .await?; - storage_processor - .update_graph( - &GraphUpdate::new(graph_id) - .with_proceed_withdraw_height(event.block_number.parse::()?), - ) - .await?; // for history events storage_processor @@ -389,43 +412,104 @@ async fn handle_proceed_withdraw_events<'a>( Ok(()) } +async fn apply_gateway_graph_status<'a>( + storage_processor: &mut StorageProcessor<'a>, + instance_id: Uuid, + graph_id: Uuid, + target: GraphStatus, +) -> anyhow::Result { + match storage_processor + .transition_graph_status(instance_id, graph_id, target, GraphStatusSource::GoatEvent, None) + .await? + { + outcome @ (GraphStatusTransitionOutcome::Applied + | GraphStatusTransitionOutcome::AlreadyCurrent) => Ok(outcome), + GraphStatusTransitionOutcome::Rejected { current } => { + warn!( + "ignore stale gateway graph event: graph={graph_id}, target={target}, current={current}" + ); + Ok(GraphStatusTransitionOutcome::Rejected { current }) + } + GraphStatusTransitionOutcome::NotFound => { + warn!("ignore gateway graph event for missing graph {graph_id}: target={target}"); + Ok(GraphStatusTransitionOutcome::NotFound) + } + } +} + +async fn graph_belongs_to_instance<'a>( + storage_processor: &mut StorageProcessor<'a>, + instance_id: Uuid, + graph_id: Uuid, +) -> anyhow::Result { + match storage_processor.find_graph(&graph_id).await? { + Some(graph) if graph.instance_id == instance_id => Ok(true), + Some(graph) => { + warn!( + "ignore gateway event with mismatched graph instance: graph={graph_id}, event_instance={instance_id}, stored_instance={}", + graph.instance_id + ); + Ok(false) + } + None => { + warn!("ignore gateway event for missing graph {instance_id}:{graph_id}"); + Ok(false) + } + } +} + async fn handle_withdraw_paths_events<'a>( storage_processor: &mut StorageProcessor<'a>, withdraw_paths_events: Vec, ) -> anyhow::Result<()> { for event in withdraw_paths_events { - let reward_add = U256::from_str(&event.reward_amount_str()).unwrap_or_default(); - let (flag, goat_addr) = reflect_goat_address(Some(event.operator_addr())); - if !flag { - warn!( - "handle_withdraw_paths_events failed as cast operator address failed, detail: {}, {}", - event.tx_hash(), - event.operator_addr() - ); - continue; - } - add_node_reward(storage_processor, &goat_addr.unwrap(), reward_add).await?; let (graph_id, instance_id, tx_type, status) = match event.clone() { WithdrawPathsEvent::WithdrawHappyEvent(v) => ( v.graph_id.clone(), v.instance_id.clone(), GoatTxType::WithdrawHappyPath.to_string(), - GraphStatus::OperatorTake1.to_string(), + GraphStatus::OperatorTake1, ), WithdrawPathsEvent::WithdrawUnhappyEvent(v) => ( v.graph_id.clone(), v.instance_id.clone(), GoatTxType::WithdrawUnhappyPath.to_string(), - GraphStatus::OperatorTake2.to_string(), + GraphStatus::OperatorTake2, ), }; let graph_id = Uuid::from_str(&strip_hex_prefix_owned(&graph_id))?; let instance_id = Uuid::from_str(&strip_hex_prefix_owned(&instance_id))?; + if !graph_belongs_to_instance(storage_processor, instance_id, graph_id).await? { + continue; + } + + let reward_add = U256::from_str(&event.reward_amount_str()).unwrap_or_default(); + let (flag, goat_addr) = reflect_goat_address(Some(event.operator_addr())); + if !flag { + warn!( + "handle_withdraw_paths_events failed as cast operator address failed, detail: {}, {}", + event.tx_hash(), + event.operator_addr() + ); + continue; + } + let outcome = + apply_gateway_graph_status(storage_processor, instance_id, graph_id, status).await?; + if !matches!( + outcome, + GraphStatusTransitionOutcome::Applied | GraphStatusTransitionOutcome::AlreadyCurrent + ) { + continue; + } + let is_new_event = storage_processor + .find_graph_goat_tx_record(&instance_id, &graph_id, &tx_type) + .await? + .is_none(); storage_processor .upsert_goat_tx_record(&GoatTxRecord { instance_id, graph_id, - tx_type, + tx_type: tx_type.clone(), tx_hash: event.tx_hash(), height: event.get_block_number(), is_local: false, @@ -434,8 +518,9 @@ async fn handle_withdraw_paths_events<'a>( created_at: current_time_secs(), }) .await?; - storage_processor.update_graph(&GraphUpdate::new(graph_id).with_status(status)).await?; - // cancel unfinished p2p message + if is_new_event { + add_node_reward(storage_processor, &goat_addr.unwrap(), reward_add).await?; + } storage_processor .update_messages_state_by_business_id( &graph_id, @@ -454,6 +539,10 @@ async fn handle_withdraw_disproved_events<'a>( ) -> anyhow::Result<()> { for event in withdraw_disproved_events { let graph_id = Uuid::from_str(&strip_hex_prefix_owned(&event.graph_id))?; + let instance_id = Uuid::from_str(&strip_hex_prefix_owned(&event.instance_id))?; + if !graph_belongs_to_instance(storage_processor, instance_id, graph_id).await? { + continue; + } let (flag, verifier_addr) = reflect_goat_address(Some(event.challenger_addr.clone())); if !flag { warn!( @@ -471,24 +560,51 @@ async fn handle_withdraw_disproved_events<'a>( continue; } - add_node_reward( + let outcome = apply_gateway_graph_status( storage_processor, - &verifier_addr.unwrap(), - U256::from_str(&event.challenger_amount_sats).unwrap_or_default(), - ) - .await?; - add_node_reward( - storage_processor, - &disprover_addr.unwrap(), - U256::from_str(&event.disprover_amount_sats).unwrap_or_default(), + instance_id, + graph_id, + GraphStatus::Disprove, ) .await?; + if !matches!( + outcome, + GraphStatusTransitionOutcome::Applied | GraphStatusTransitionOutcome::AlreadyCurrent + ) { + continue; + } + let tx_type = GoatTxType::WithdrawDisproved.to_string(); + let is_new_event = storage_processor + .find_graph_goat_tx_record(&instance_id, &graph_id, &tx_type) + .await? + .is_none(); storage_processor - .update_graph( - &GraphUpdate::new(graph_id).with_status(GraphStatus::Disprove.to_string()), + .upsert_goat_tx_record(&GoatTxRecord { + instance_id, + graph_id, + tx_type, + tx_hash: event.transaction_hash.clone(), + height: event.block_number.parse::()?, + is_local: false, + processing_status: GoatTxProcessingStatus::Pending.to_string(), + extra: None, + created_at: current_time_secs(), + }) + .await?; + if is_new_event { + add_node_reward( + storage_processor, + &verifier_addr.unwrap(), + U256::from_str(&event.challenger_amount_sats).unwrap_or_default(), + ) + .await?; + add_node_reward( + storage_processor, + &disprover_addr.unwrap(), + U256::from_str(&event.disprover_amount_sats).unwrap_or_default(), ) .await?; - // cancel unfinished p2p message + } storage_processor .update_messages_state_by_business_id( &graph_id, @@ -631,69 +747,114 @@ async fn handle_swap_init_events<'a>( .await? { let create_time = event.block_timestamp.parse::()?; - let (instance_id, instance) = if let Some(instance) = + let event_height = event.block_number.parse::()?; + let (instance_id, initialized) = if let Some(instance) = find_instances_by_escrow_hash(storage_processor, &event.escrow_hash).await? { - if instance.status != InstanceBridgeOutStatus::Initialize.to_string() { - (instance.instance_id, None) - } else { - (instance.instance_id, Some(instance)) - } + let initialized = storage_processor + .update_instance( + &swap_init_instance_update( + instance.instance_id, + &escrow_data, + &event, + event_height, + ) + .with_only_if_status_in(vec![ + InstanceBridgeOutStatus::Initialize.to_string(), + ]) + .with_only_if_is_bridge_in(false) + .with_only_if_goat_tx_hash(String::new()), + ) + .await?; + (instance.instance_id, initialized) } else { - let instance_id = Uuid::new_v4(); - ( + let instance_id = bridge_out_instance_id_from_escrow_hash(&event.escrow_hash); + let instance = Instance { instance_id, - Some(Instance { - instance_id, - is_bridge_in: false, - network: get_network().to_string(), - from_addr: escrow_data.offerer.to_string(), - input_utxos: "[]".to_string(), - status: InstanceBridgeOutStatus::Initialize.to_string(), - escrow_hash: Some(event.escrow_hash.clone()), - status_updated_at: create_time, - created_at: create_time, - ..Default::default() - }), - ) - }; - if let Some(mut instance) = instance { - instance.bridge_out_amount = escrow_data.amount.to_string(); - instance.goat_tx_hash = event.transaction_hash.clone(); - instance.goat_tx_height = event.block_number.parse::()?; - instance.user_change_addr = escrow_data.claimer.to_string(); - instance.user_refund_addr = escrow_data.claimer.to_string(); - instance.bridge_out_lock_time = - get_timestamp_from_contract_data(&escrow_data.refundData.0); - instance.status_updated_at = create_time; - storage_processor.upsert_instance(&instance).await?; - if escrow_data.token == *gateway_peg_btc_address { - let mut bridge_out_global_stats = - get_bridge_out_global_stats(storage_processor).await?; - let mut initial_amount = - U256::from_str(&bridge_out_global_stats.initial_amount).unwrap_or_default(); - initial_amount.add_assign(&escrow_data.amount); - bridge_out_global_stats.initial_amount = initial_amount.to_string(); - bridge_out_global_stats.initial_txn += 1; + is_bridge_in: false, + network: get_network().to_string(), + from_addr: escrow_data.offerer.to_string(), + input_utxos: "[]".to_string(), + status: InstanceBridgeOutStatus::Initialize.to_string(), + escrow_hash: Some(event.escrow_hash.clone()), + bridge_out_amount: escrow_data.amount.to_string(), + goat_tx_hash: event.transaction_hash.clone(), + goat_tx_height: event_height, + user_change_addr: escrow_data.claimer.to_string(), + user_refund_addr: escrow_data.claimer.to_string(), + bridge_out_lock_time: get_timestamp_from_contract_data( + &escrow_data.refundData.0, + ), + status_updated_at: create_time, + created_at: create_time, + ..Default::default() + }; + let initialized = if storage_processor.insert_instance_if_absent(&instance).await? { + true + } else if let Some(existing) = storage_processor.find_instance(&instance_id).await? + { + let escrow_hash_matches = existing + .escrow_hash + .as_ref() + .map(|hash| hash.eq_ignore_ascii_case(&event.escrow_hash)) + .unwrap_or(false); + if existing.is_bridge_in || !escrow_hash_matches { + anyhow::bail!( + "bridge-out instance ID collision for escrow hash {}", + event.escrow_hash + ); + } storage_processor - .upsert_bridge_out_global_stats(&bridge_out_global_stats) - .await?; - info!( - "swap initialize stats included: tx_hash={}, escrow_hash={}, token={}, amount={}", - event.transaction_hash, - event.escrow_hash, - escrow_data.token, - escrow_data.amount, - ); + .update_instance( + &swap_init_instance_update( + existing.instance_id, + &escrow_data, + &event, + event_height, + ) + .with_only_if_status_in(vec![ + InstanceBridgeOutStatus::Initialize.to_string(), + ]) + .with_only_if_is_bridge_in(false) + .with_only_if_goat_tx_hash(String::new()), + ) + .await? } else { - info!( - "swap initialize stats skipped(non-pegBTC): tx_hash={}, escrow_hash={}, token={}, expect_token={}", - event.transaction_hash, - event.escrow_hash, - escrow_data.token, - gateway_peg_btc_address, + anyhow::bail!( + "bridge-out instance {} disappeared during creation", + instance_id ); - } + }; + (instance_id, initialized) + }; + if initialized && escrow_data.token == *gateway_peg_btc_address { + let mut bridge_out_global_stats = + get_bridge_out_global_stats(storage_processor).await?; + let mut initial_amount = + U256::from_str(&bridge_out_global_stats.initial_amount).unwrap_or_default(); + initial_amount.add_assign(&escrow_data.amount); + bridge_out_global_stats.initial_amount = initial_amount.to_string(); + bridge_out_global_stats.initial_txn += 1; + storage_processor.upsert_bridge_out_global_stats(&bridge_out_global_stats).await?; + info!( + "swap initialize stats included: tx_hash={}, escrow_hash={}, token={}, amount={}", + event.transaction_hash, + event.escrow_hash, + escrow_data.token, + escrow_data.amount, + ); + } else if initialized { + info!( + "swap initialize stats skipped(non-pegBTC): tx_hash={}, escrow_hash={}, token={}, expect_token={}", + event.transaction_hash, + event.escrow_hash, + escrow_data.token, + gateway_peg_btc_address, + ); + } else { + info!( + "swap initialize ignored for resolved or previously initialized instance {instance_id}" + ); } storage_processor .upsert_goat_tx_record(&GoatTxRecord { @@ -701,7 +862,7 @@ async fn handle_swap_init_events<'a>( graph_id: Uuid::nil(), tx_type: GoatTxType::SwapInitialize.to_string(), tx_hash: event.transaction_hash.clone(), - height: event.block_number.parse::()?, + height: event_height, is_local: false, processing_status: GoatTxProcessingStatus::Skipped.to_string(), extra: Some(hex::encode(escrow_data.abi_encode())), @@ -715,6 +876,21 @@ async fn handle_swap_init_events<'a>( Ok(()) } +fn swap_init_instance_update( + instance_id: Uuid, + escrow_data: &EscrowData, + event: &SwapInitializeEvent, + event_height: i64, +) -> InstanceUpdate { + InstanceUpdate::new_with_instance_id(instance_id) + .with_bridge_out_amount(escrow_data.amount.to_string()) + .with_goat_tx_hash(event.transaction_hash.clone()) + .with_goat_tx_height(event_height) + .with_user_change_addr(escrow_data.claimer.to_string()) + .with_user_refund_addr(escrow_data.claimer.to_string()) + .with_bridge_out_lock_time(get_timestamp_from_contract_data(&escrow_data.refundData.0)) +} + async fn handle_swap_claim_events<'a>( storage_processor: &mut StorageProcessor<'a>, goat_client: Arc, @@ -740,6 +916,25 @@ async fn handle_swap_claim_events<'a>( bitcoin::Script::from_bytes(&claim_data.output_script), get_network(), )?; + let transitioned = storage_processor + .update_instance( + &InstanceUpdate::new_with_instance_id(instance_id) + .with_status(InstanceBridgeOutStatus::Claim.to_string()) + .with_btc_txid(claim_data.txid.into()) + .with_to_addr(to_addr.to_string()) + .with_only_if_status_in(vec![ + InstanceBridgeOutStatus::Initialize.to_string(), + ]) + .with_only_if_is_bridge_in(false), + ) + .await?; + if !transitioned { + info!( + "swap claim ignored for resolved instance {instance_id} with status {}", + instance.status + ); + continue; + } storage_processor .upsert_goat_tx_record(&GoatTxRecord { instance_id, @@ -753,14 +948,6 @@ async fn handle_swap_claim_events<'a>( created_at: current_time_secs(), }) .await?; - storage_processor - .update_instance( - &InstanceUpdate::new_with_escrow_hash(event.escrow_hash.clone()) - .with_status(InstanceBridgeOutStatus::Claim.to_string()) - .with_btc_txid(claim_data.txid.into()) - .with_to_addr(to_addr.to_string()), - ) - .await?; let is_peg_btc_swap = is_gateway_peg_btc_swap_instance( storage_processor, @@ -807,12 +994,23 @@ async fn handle_swap_refund_events<'a>( if let Some(instance) = find_instances_by_escrow_hash(storage_processor, &event.escrow_hash).await? { - storage_processor + let transitioned = storage_processor .update_instance( - &InstanceUpdate::new_with_escrow_hash(event.escrow_hash.clone()) - .with_status(InstanceBridgeOutStatus::Refund.to_string()), + &InstanceUpdate::new_with_instance_id(instance.instance_id) + .with_status(InstanceBridgeOutStatus::Refund.to_string()) + .with_only_if_status_in(vec![ + InstanceBridgeOutStatus::Initialize.to_string(), + ]) + .with_only_if_is_bridge_in(false), ) .await?; + if !transitioned { + info!( + "swap refund ignored for resolved instance {} with status {}", + instance.instance_id, instance.status + ); + continue; + } let is_peg_btc_swap = is_gateway_peg_btc_swap_instance( storage_processor, &instance.instance_id, @@ -932,15 +1130,20 @@ async fn handle_post_graph_data_events<'a>( post_graph_data_events: Vec, ) -> anyhow::Result<()> { for event in post_graph_data_events { - if let Ok(graph_id) = Uuid::from_str(&strip_hex_prefix_owned(&event.graph_id)) { - storage_processor - .update_graph( - &GraphUpdate::new(graph_id) - .with_status(GraphStatus::OperatorDataPushed.to_string()), + match ( + Uuid::from_str(&strip_hex_prefix_owned(&event.instance_id)), + Uuid::from_str(&strip_hex_prefix_owned(&event.graph_id)), + ) { + (Ok(instance_id), Ok(graph_id)) => { + let _ = apply_gateway_graph_status( + storage_processor, + instance_id, + graph_id, + GraphStatus::OperatorDataPushed, ) .await?; - } else { - warn!("failed to parse instance id:{event:?}"); + } + _ => warn!("failed to parse graph event identifiers: {event:?}"), } } Ok(()) diff --git a/node/src/scheduled_tasks/graph_maintenance_tasks.rs b/node/src/scheduled_tasks/graph_maintenance_tasks.rs index 6489e2d4a..0d917e372 100644 --- a/node/src/scheduled_tasks/graph_maintenance_tasks.rs +++ b/node/src/scheduled_tasks/graph_maintenance_tasks.rs @@ -7,13 +7,15 @@ use crate::action::{ use crate::env::get_network; use crate::rpc_service::current_time_secs; use crate::scheduled_tasks::fetch_on_turn_graph_by_status; -use crate::utils::{SELF_SENDER, outpoint_spent_txid, parse_graph_raw_data, upsert_message}; +use crate::utils::{ + SELF_SENDER, load_validated_graph_definition, outpoint_spent_txid, upsert_message, +}; use bitcoin::Txid; use bitvm_lib::actors::Actor; use bitvm_lib::timelocks::{ - connector_f_timelock_blocks, default_timelock_config, disprove_timelock_blocks, - operator_ack_timelock_blocks, operator_commit_timelock_blocks, take1_timelock_blocks, - take2_timelock_blocks, validate_timelock_config, watchtower_challenge_timelock_blocks, + connector_f_timelock_blocks, disprove_timelock_blocks, operator_ack_timelock_blocks, + operator_commit_timelock_blocks, take1_timelock_blocks, take2_timelock_blocks, + validate_timelock_config, watchtower_challenge_timelock_blocks, }; use client::btc_chain::BTCClient; use client::goat_chain::DisproveTxType; @@ -77,16 +79,14 @@ async fn graph_timelock_config( local_db: &LocalDB, graph_id: Uuid, ) -> anyhow::Result { - let raw_data = { - let mut storage_processor = local_db.acquire().await?; - storage_processor.find_graph_raw_data(&graph_id).await? - }; - let Some(raw_data) = raw_data else { - warn!("graph {graph_id} raw data is missing, fallback to default timelock config"); - return Ok(default_timelock_config(get_network())); - }; - - let graph = parse_graph_raw_data(raw_data.raw_data, graph_id).await?; + let mut storage_processor = local_db.acquire().await?; + let graph_row = storage_processor.find_graph(&graph_id).await?.ok_or_else(|| { + anyhow::anyhow!("graph {graph_id} is missing while loading its timelock config") + })?; + let graph = + load_validated_graph_definition(&mut storage_processor, graph_row.instance_id, graph_id) + .await? + .ok_or_else(|| anyhow::anyhow!("graph {graph_id} has no validated raw definition"))?; validate_timelock_config( graph.parameters.instance_parameters.network, &graph.parameters.timelock_config, diff --git a/node/src/scheduled_tasks/instance_maintenance_tasks.rs b/node/src/scheduled_tasks/instance_maintenance_tasks.rs index 6d5d886cb..cd4e1b156 100644 --- a/node/src/scheduled_tasks/instance_maintenance_tasks.rs +++ b/node/src/scheduled_tasks/instance_maintenance_tasks.rs @@ -112,14 +112,12 @@ async fn update_instance<'a>( storage_processor: &mut StorageProcessor<'a>, params: &InstanceUpdate, ) -> anyhow::Result<()> { - if let Err(err) = storage_processor.update_instance(params).await { - warn!( - "update_instance_status with input: {:?} failed {}, will try later", - params, - err.to_string() - ); - } else { - info!("update instance with input: {:?}", params); + match storage_processor.update_instance(params).await { + Ok(true) => info!("update instance with input: {:?}", params), + Ok(false) => info!("skip stale instance update with input: {:?}", params), + Err(err) => { + warn!("update_instance_status with input: {:?} failed {}, will try later", params, err); + } } Ok(()) } @@ -227,44 +225,77 @@ pub async fn instance_window_expiration_monitor( .await?; let committee_quorum_size = goat_client.committee_mana_quorum_size().await?; - for mut instance in instances { - match goat_client.gateway_get_pegin_data(&instance.instance_id).await { - Ok(pegin_data) => { - for (committee_addr, pubkey) in - pegin_data.committee_addresses.iter().zip(pegin_data.committee_pubkeys) - { - instance.committees_answers.insert(committee_addr.to_string(), pubkey); - } - - if committee_quorum_size <= instance.committees_answers.len() as u64 { - instance.status = InstanceBridgeInStatus::CommitteesAnswered.to_string(); - if let Err(err) = update_pegin_txids(&mut instance) { - warn!( - "instance_window_expiration_monitor fail to update_pegin_txids for instance {}, err: {:?}", - instance.instance_id, err - ); - } - } else { - instance.status = - InstanceBridgeInStatus::NoEnoughCommitteesAnswered.to_string(); - } - let mut storage_processor = local_db.acquire().await?; - if let Err(err) = storage_processor.upsert_instance(&instance).await { - warn!( - "failed to upsert instance {}, err: {}", - instance.instance_id, - err.to_string() - ); - } - } + for snapshot in instances { + let pegin_data = match goat_client.gateway_get_pegin_data(&snapshot.instance_id).await { + Ok(pegin_data) => pegin_data, Err(err) => { warn!( "failed to get pegin data for instance {}, err: {}", - instance.instance_id, - err.to_string() + snapshot.instance_id, err + ); + continue; + } + }; + + // The RPC call above can take arbitrarily long. Re-read while holding + // SQLite's write lock, then make the decision and its update in the + // same short transaction so a late committee response cannot be + // overwritten by the stale page snapshot. + let mut storage_processor = local_db.start_immediate_transaction().await?; + let Some(mut instance) = storage_processor.find_instance(&snapshot.instance_id).await? + else { + storage_processor.commit().await?; + continue; + }; + if instance.status != InstanceBridgeInStatus::UserInited.to_string() { + info!( + "skip expired response-window reconciliation for instance {} in status {}", + instance.instance_id, instance.status + ); + storage_processor.commit().await?; + continue; + } + + for (committee_addr, pubkey) in + pegin_data.committee_addresses.iter().zip(pegin_data.committee_pubkeys) + { + instance.committees_answers.insert(committee_addr.to_string(), pubkey); + } + + let reached_quorum = committee_quorum_size <= instance.committees_answers.len() as u64; + let next_status = if reached_quorum { + if let Err(err) = update_pegin_txids(&mut instance) { + warn!( + "instance_window_expiration_monitor failed to update pegin txids for instance {}, err: {err:?}", + instance.instance_id ); } + InstanceBridgeInStatus::CommitteesAnswered + } else { + InstanceBridgeInStatus::NoEnoughCommitteesAnswered + }; + + let mut update = InstanceUpdate::new_with_instance_id(instance.instance_id) + .with_status(next_status.to_string()) + .with_committees_answers(instance.committees_answers.clone()) + .with_only_if_status_in(vec![InstanceBridgeInStatus::UserInited.to_string()]) + .with_only_if_is_bridge_in(true); + if reached_quorum { + if let Some(btc_txid) = instance.btc_txid.clone() { + update = update.with_btc_txid(btc_txid); + } + if let Some(pegin_confirm_txid) = instance.pegin_confirm_txid.clone() { + update = update.with_pegin_confirm_txid(pegin_confirm_txid); + } + if let Some(pegin_cancel_txid) = instance.pegin_cancel_txid.clone() { + update = update.with_pegin_cancel_txid(pegin_cancel_txid); + } + } + + if !storage_processor.update_instance(&update).await? { + warn!("skip stale response-window update for instance {}", instance.instance_id); } + storage_processor.commit().await?; } Ok(()) @@ -494,7 +525,9 @@ pub async fn instance_bridge_out_monitor(local_db: &LocalDB) -> anyhow::Result<( .await?; let mut storage_processor = local_db.acquire().await?; for instance in instances { - let mut instance_update = InstanceUpdate::new_with_instance_id(instance.instance_id); + let mut instance_update = InstanceUpdate::new_with_instance_id(instance.instance_id) + .with_only_if_status_in(vec![InstanceBridgeOutStatus::Initialize.to_string()]) + .with_only_if_is_bridge_in(false); let lock_time = if instance.bridge_out_lock_time == 0 { let lock_time = get_bridge_out_deadline(&mut storage_processor, &instance.instance_id).await?; diff --git a/node/src/scheduled_tasks/mod.rs b/node/src/scheduled_tasks/mod.rs index c94324966..6a91e0a1d 100644 --- a/node/src/scheduled_tasks/mod.rs +++ b/node/src/scheduled_tasks/mod.rs @@ -11,6 +11,7 @@ use crate::env::{ get_maintenance_run_timeout_secs, is_enable_babe_setup_state_cleanup, is_enable_update_spv_contract, is_relayer, }; +use crate::rpc_service::current_time_secs; use crate::scheduled_tasks::babe_setup_state_cleanup_task::babe_setup_state_cleanup_monitor; use crate::scheduled_tasks::graph_maintenance_tasks::{ detect_init_withdraw_call, detect_kickoff, detect_take1_or_challenge, process_graph_challenge, @@ -27,12 +28,13 @@ use client::btc_chain::BTCClient; use client::goat_chain::GOATClient; pub use event_watch_task::{is_processing_gateway_history_events, run_watch_event_task}; pub use sequencer_set_hash_monitor_task::run_sequencer_set_hash_monitor_task; +use std::future::Future; use std::sync::Arc; use std::time::{Duration, Instant}; use store::localdb::{LocalDB, StorageProcessor}; use store::{Graph, MessageType}; use tokio_util::sync::CancellationToken; -use tracing::{error, info, warn}; +use tracing::{debug, error, info, warn}; async fn fetch_on_turn_graph_by_status<'a>( storage_processor: &mut StorageProcessor<'a>, @@ -51,78 +53,124 @@ async fn fetch_on_turn_graph_by_status<'a>( } Ok(graphs) } + +async fn run_maintenance_subtask( + task: &'static str, + operation: impl Future>, +) { + let started_at = Instant::now(); + match operation.await { + Ok(_) => debug!( + event = "maintenance_subtask_result", + task, + outcome = "succeeded", + elapsed_ms = started_at.elapsed().as_millis() as u64, + "maintenance subtask completed" + ), + Err(error) => warn!( + event = "maintenance_subtask_result", + task, + outcome = "failed", + elapsed_ms = started_at.elapsed().as_millis() as u64, + error_class = "maintenance", + error = %error, + "maintenance subtask failed after execution" + ), + } +} + +enum MaintenanceRunOutcome { + Completed, + DeferredHistorySync, +} + async fn run( actor: Actor, local_db: &LocalDB, btc_client: Arc, goat_client: Arc, -) -> anyhow::Result<()> { +) -> anyhow::Result { let btc_client = btc_client.as_ref(); let goat_client = goat_client.as_ref(); if is_enable_babe_setup_state_cleanup() && matches!(&actor, Actor::Verifier | Actor::Operator | Actor::All) - && let Err(err) = babe_setup_state_cleanup_monitor(local_db).await { - warn!("babe_setup_state_cleanup_monitor, err {:?}", err) + run_maintenance_subtask( + "babe_setup_state_cleanup_monitor", + babe_setup_state_cleanup_monitor(local_db), + ) + .await; } - if (actor == Actor::Operator || is_relayer()) - && let Err(err) = node_available_pbtc_update_monitor(local_db, goat_client).await - { - warn!("node_available_pbtc_update_monitor, err {:?}", err) + if actor == Actor::Operator || is_relayer() { + run_maintenance_subtask( + "node_available_pbtc_update_monitor", + node_available_pbtc_update_monitor(local_db, goat_client), + ) + .await; } - if is_enable_update_spv_contract() - && let Err(err) = spv_header_hash_update(btc_client, goat_client).await - { - warn!("spv_header_hash_update, err {:?}", err) + if is_enable_update_spv_contract() { + run_maintenance_subtask( + "spv_header_hash_update", + spv_header_hash_update(btc_client, goat_client), + ) + .await; } if is_processing_gateway_history_events(local_db, goat_client).await? { - warn!("Still in history events processing"); - return Ok(()); - } - - if let Err(err) = instance_answers_monitor(local_db, btc_client, goat_client).await { - warn!("instance_answers_monitor, err {:?}", err) - } - if let Err(err) = instance_window_expiration_monitor(local_db, goat_client).await { - warn!("instance_window_expiration_monitor, err {:?}", err) - } - - if let Err(err) = instance_expiration_monitor(local_db, btc_client).await { - warn!("instance_expiration_monitor, err {:?}", err) - } - - if let Err(err) = instance_btc_tx_monitor(local_db, btc_client).await { - warn!("instance_btc_tx_monitor, err {:?}", err) - } - - if let Err(err) = instance_committee_key_cleanup_monitor(local_db, btc_client).await { - warn!("instance_committee_key_cleanup_monitor, err {:?}", err) - } - - if let Err(err) = instance_bridge_out_monitor(local_db).await { - warn!("instance_bridge_out_monitor, err {:?}", err) - } - - if let Err(err) = detect_init_withdraw_call(local_db).await { - warn!("detect_init_withdraw_call, err {:?}", err) - } - - if let Err(err) = detect_kickoff(local_db, btc_client).await { - warn!("detect_kickoff, err {:?}", err) + info!( + event = "maintenance_subtask_result", + task = "gateway_history_sync", + outcome = "deferred", + role = %actor, + reason = "history_sync_in_progress", + "maintenance protocol work deferred while gateway history sync is active" + ); + return Ok(MaintenanceRunOutcome::DeferredHistorySync); } - if let Err(err) = detect_take1_or_challenge(local_db, btc_client).await { - warn!("detect_take1_or_challenge, err {:?}", err) - } - - if let Err(err) = process_graph_challenge(local_db, btc_client).await { - warn!("process_grpah_challenge, err {:?}", err) - } - Ok(()) + run_maintenance_subtask( + "instance_answers_monitor", + instance_answers_monitor(local_db, btc_client, goat_client), + ) + .await; + run_maintenance_subtask( + "instance_window_expiration_monitor", + instance_window_expiration_monitor(local_db, goat_client), + ) + .await; + run_maintenance_subtask( + "instance_expiration_monitor", + instance_expiration_monitor(local_db, btc_client), + ) + .await; + run_maintenance_subtask( + "instance_btc_tx_monitor", + instance_btc_tx_monitor(local_db, btc_client), + ) + .await; + run_maintenance_subtask( + "instance_committee_key_cleanup_monitor", + instance_committee_key_cleanup_monitor(local_db, btc_client), + ) + .await; + run_maintenance_subtask("instance_bridge_out_monitor", instance_bridge_out_monitor(local_db)) + .await; + run_maintenance_subtask("detect_init_withdraw_call", detect_init_withdraw_call(local_db)).await; + run_maintenance_subtask("detect_kickoff", detect_kickoff(local_db, btc_client)).await; + run_maintenance_subtask( + "detect_take1_or_challenge", + detect_take1_or_challenge(local_db, btc_client), + ) + .await; + run_maintenance_subtask( + "process_graph_challenge", + process_graph_challenge(local_db, btc_client), + ) + .await; + Ok(MaintenanceRunOutcome::Completed) } pub async fn run_maintenance_tasks( @@ -140,26 +188,102 @@ pub async fn run_maintenance_tasks( _ = tokio::time::sleep(Duration::from_secs(interval)) => { tick += 1; let tick_start = Instant::now(); - info!(tick, interval_secs = interval, "maintenance task tick start"); + info!( + event = "maintenance_tick", + tick, + interval_secs = interval, + outcome = "started", + "maintenance task tick started" + ); // Execute the normal monitoring logic match tokio::time::timeout( maintenance_run_timeout, run(actor.clone(),&local_db,btc_client.clone(),goat_client.clone()), ).await { - Ok(Ok(_)) => {} - Ok(Err(err)) => {error!("run_scheduled_tasks, err {:?}", err)} + Ok(Ok(MaintenanceRunOutcome::Completed)) => { + info!( + event = "maintenance_tick_result", + tick, + outcome = "succeeded", + elapsed_ms = tick_start.elapsed().as_millis() as u64, + "maintenance task tick completed" + ); + } + Ok(Ok(MaintenanceRunOutcome::DeferredHistorySync)) => { + info!( + event = "maintenance_tick_result", + tick, + outcome = "deferred", + reason = "history_sync_in_progress", + elapsed_ms = tick_start.elapsed().as_millis() as u64, + "maintenance protocol work was deferred" + ); + } + Ok(Err(error)) => { + error!( + event = "maintenance_tick_result", + tick, + outcome = "failed", + elapsed_ms = tick_start.elapsed().as_millis() as u64, + error = %error, + "maintenance task returned an error" + ) + } Err(_) => { error!( + event = "maintenance_tick_result", tick, + outcome = "timed_out", timeout_secs = maintenance_run_timeout.as_secs(), - "maintenance run timeout" + elapsed_ms = tick_start.elapsed().as_millis() as u64, + "maintenance task tick timed out" ) } } - info!(tick, elapsed_ms = tick_start.elapsed().as_millis() as u64, "maintenance task tick end"); + if tick.is_multiple_of(6) { + let queue_started_at = Instant::now(); + match local_db.acquire().await { + Ok(mut storage) => match storage + .get_message_queue_stats(&actor.to_string(), current_time_secs()) + .await + { + Ok(stats) => info!( + event = "message_queue_snapshot", + role = %actor, + pending_ready = stats.pending_ready, + pending_locked = stats.pending_locked, + failed = stats.failed, + oldest_pending_at = ?stats.oldest_pending_at, + elapsed_ms = queue_started_at.elapsed().as_millis() as u64, + "local message queue snapshot" + ), + Err(error) => error!( + event = "db_operation_result", + operation = "get_message_queue_stats", + outcome = "failed", + elapsed_ms = queue_started_at.elapsed().as_millis() as u64, + error = %error, + "failed to collect local message queue snapshot" + ), + }, + Err(error) => error!( + event = "db_operation_result", + operation = "acquire_db_for_message_queue_snapshot", + outcome = "failed", + elapsed_ms = queue_started_at.elapsed().as_millis() as u64, + error = %error, + "failed to acquire local database for message queue snapshot" + ), + } + } } _ = cancellation_token.cancelled() => { - tracing::info!("maintenance task received shutdown signal"); + tracing::info!( + event = "maintenance_lifecycle", + outcome = "shutdown", + role = %actor, + "maintenance task received shutdown signal" + ); return Ok("maintenance_shutdown".to_string()); } } diff --git a/node/src/utils.rs b/node/src/utils.rs index da6f792f5..abb5a8dea 100644 --- a/node/src/utils.rs +++ b/node/src/utils.rs @@ -1,7 +1,6 @@ use crate::action::{ ChallengeSent, DisproveSent, GOATMessage, GOATMessageContent, KickoffSent, NodeInfo, - PreKickoffSent, SolderingProofReady, Take1Sent, Take2Sent, push_local_unhandled_messages, - send_to_peer, + PreKickoffSent, SolderingProofReady, Take1Sent, Take2Sent, send_to_peer, }; use crate::env::*; use crate::error::SpecialError; @@ -68,12 +67,8 @@ use std::net::SocketAddr; use std::panic::{AssertUnwindSafe, catch_unwind}; use std::path::{Path, PathBuf}; use std::str::FromStr; - -pub const SELF_SENDER: &str = "self"; -use std::time::{SystemTime, UNIX_EPOCH}; -use store::localdb::{ - GraphQuery, GraphUpdate, InstanceQuery, InstanceUpdate, LocalDB, StorageProcessor, -}; +use std::time::{Instant, SystemTime, UNIX_EPOCH}; +use store::localdb::{GraphQuery, GraphRuntimeUpdate, InstanceQuery, LocalDB, StorageProcessor}; use crate::env; use crate::rpc_service::routes::v1::{ @@ -98,9 +93,10 @@ use proof_builder::{ WatchtowerProofTimeoutUpdateResponse, }; use store::{ - BridgeOutGlobalStats, ByteArray32, Graph, GraphRawData, GraphStatus, Instance, - InstanceBridgeInStatus, Message, MessageState, MessageType, Node, PeginGraphProcessData, - PeginInstanceProcessData, SerializableTxid, UInt64Array3, + BridgeOutGlobalStats, ByteArray32, Graph, GraphRawData, GraphStatus, GraphStatusSource, + GraphStatusTransitionOutcome, Instance, InstanceBridgeInStatus, Message, MessageState, + MessageType, Node, PeginGraphProcessData, PeginInstanceProcessData, SerializableTxid, + UInt64Array3, }; use stun_client::{Attribute, Class, Client}; use tracing::{error, info, warn}; @@ -113,6 +109,29 @@ use zkm_verifier::{ load_ark_public_inputs_from_bytes, }; +pub const SELF_SENDER: &str = "self"; +const BRIDGE_OUT_INSTANCE_ID_PREFIX: [u8; 4] = *b"BOID"; + +/// Derive the shared bridge-out instance ID from its escrow hash. +/// +/// Both the RPC tag endpoint and the chain-event watcher must use this ID so +/// their concurrent create attempts collide on the primary key instead of +/// creating two rows for one escrow. +pub(crate) fn bridge_out_instance_id_from_escrow_hash(escrow_hash: &str) -> Uuid { + let normalized_escrow_hash = escrow_hash + .strip_prefix("0x") + .or_else(|| escrow_hash.strip_prefix("0X")) + .unwrap_or(escrow_hash); + let mut hasher = Sha256::new(); + hasher.update(b"bridge-out:"); + hasher.update(normalized_escrow_hash.to_ascii_lowercase().as_bytes()); + let digest = hasher.finalize(); + let mut bytes = [0u8; 16]; + bytes.copy_from_slice(&digest[..16]); + bytes[..4].copy_from_slice(&BRIDGE_OUT_INSTANCE_ID_PREFIX); + Uuid::from_bytes(bytes) +} + pub(crate) const BRIDGE_OUT_GLOBAL_STATS_ID: i64 = 1; pub type VerifyingKey = ark_groth16::VerifyingKey; @@ -357,16 +376,17 @@ pub mod todo_funcs { ) .await?; if let Some(previous_graph) = graphs.first() { - let Some(graph_raw_data) = - storage_processor.find_graph_raw_data(&previous_graph.graph_id).await? - else { - bail!(SpecialError::InvalidGraph(format!( + let simplified_graph = super::load_validated_graph_definition( + &mut storage_processor, + previous_graph.instance_id, + previous_graph.graph_id, + ) + .await? + .ok_or_else(|| { + SpecialError::InvalidGraph(format!( "previous graph raw data is missing for graph nonce {previous_nonce}" - ))); - }; - let simplified_graph = - super::parse_graph_raw_data(graph_raw_data.raw_data, previous_graph.graph_id) - .await?; + )) + })?; let expected_cur_prekickoff = BitvmGcGraph::from_simplified(&simplified_graph)?.next_prekickoff; let expected_txid = expected_cur_prekickoff.finalize().compute_txid(); @@ -1126,29 +1146,20 @@ pub(crate) async fn refresh_graph( goat_client: &GOATClient, instance_id: Uuid, graph_id: Uuid, - graph: Option<&BitvmGcGraph>, - scan_from_status: Option, - scan_from_sub_status: Option, -) -> Result<(GraphStatus, Option, Option)> { - let Some(graph) = graph else { - let status = scan_from_status.unwrap_or(GraphStatus::OperatorPresigned); - return Ok((status, scan_from_sub_status, None)); - }; - - let scan = scan_graph_chain_state( - btc_client, - goat_client, - graph, - scan_from_status, - scan_from_sub_status, - ) - .await?; - - if let Some(challenge_txid) = scan.challenge_txid { - update_graph_challenge_txid_if_needed(local_db, graph_id, challenge_txid).await?; + graph: &BitvmGcGraph, +) -> Result { + if graph.parameters.instance_parameters.instance_id != instance_id + || graph.parameters.graph_id != graph_id + { + bail!( + "refuse to refresh graph {instance_id}:{graph_id} with mismatched graph parameters {}:{}", + graph.parameters.instance_parameters.instance_id, + graph.parameters.graph_id + ); } - update_graph_status( + let scan = scan_graph_chain_state(btc_client, goat_client, graph).await?; + let outcome = update_graph_status( local_db, instance_id, graph_id, @@ -1157,7 +1168,54 @@ pub(crate) async fn refresh_graph( ) .await?; - Ok((scan.status, Some(scan.sub_status.clone()), Some(scan))) + match outcome { + GraphStatusTransitionOutcome::Applied => { + if let Some(challenge_txid) = scan.challenge_txid { + update_graph_challenge_txid_if_needed(local_db, graph_id, challenge_txid).await?; + } + Ok(GraphRefreshResult { + status: scan.status, + sub_status: Some(scan.sub_status.clone()), + scan: Some(scan), + status_transition_accepted: true, + }) + } + GraphStatusTransitionOutcome::AlreadyCurrent => { + if let Some(challenge_txid) = scan.challenge_txid { + update_graph_challenge_txid_if_needed(local_db, graph_id, challenge_txid).await?; + } + Ok(GraphRefreshResult { + status: scan.status, + sub_status: Some(scan.sub_status.clone()), + // Preserve the verified scan even for an idempotent status + // replay. The previous attempt may have committed the status + // before it could enqueue the corresponding local message. + scan: Some(scan), + status_transition_accepted: true, + }) + } + GraphStatusTransitionOutcome::Rejected { current } => { + warn!( + "ignore stale chain scan for graph {instance_id}:{graph_id}: candidate={}, current={current}", + scan.status + ); + Ok(GraphRefreshResult { + status: current, + sub_status: None, + scan: None, + status_transition_accepted: false, + }) + } + GraphStatusTransitionOutcome::NotFound => { + warn!("graph {instance_id}:{graph_id} disappeared while applying chain scan"); + Ok(GraphRefreshResult { + status: scan.status, + sub_status: None, + scan: None, + status_transition_accepted: false, + }) + } + } } #[derive(Clone, Debug)] @@ -1180,11 +1238,15 @@ pub(crate) struct GraphChainScan { disprove: Option, } -fn normalize_challenge_sub_status( - mut sub_status: ChallengeSubStatus, - watchtower_num: usize, - verifier_num: usize, -) -> ChallengeSubStatus { +pub(crate) struct GraphRefreshResult { + pub(crate) status: GraphStatus, + pub(crate) sub_status: Option, + pub(crate) scan: Option, + pub(crate) status_transition_accepted: bool, +} + +fn initial_challenge_sub_status(watchtower_num: usize, verifier_num: usize) -> ChallengeSubStatus { + let mut sub_status = ChallengeSubStatus::default(); sub_status.watchtower_challenge_status.resize(watchtower_num, false); sub_status.verifier_challenge_status.resize(verifier_num, VerifierChallengeStatus::None); sub_status @@ -1227,7 +1289,10 @@ async fn update_graph_challenge_txid_if_needed( }; if graph.challenge_txid.as_ref().map(|txid| txid.0) != Some(challenge_txid) { storage_processor - .update_graph(&GraphUpdate::new(graph_id).with_challenge_txid(challenge_txid.into())) + .update_graph_runtime( + &GraphRuntimeUpdate::new(graph.instance_id, graph_id) + .with_challenge_txid(challenge_txid.into()), + ) .await?; } Ok(()) @@ -1329,34 +1394,19 @@ async fn scan_graph_chain_state( btc_client: &BTCClient, goat_client: &GOATClient, graph: &BitvmGcGraph, - scan_from_status: Option, - scan_from_sub_status: Option, ) -> Result { let instance_id = graph.parameters.instance_parameters.instance_id; let graph_id = graph.parameters.graph_id; let watchtower_num = graph.parameters.watchtower_pubkeys.len(); let verifier_num = graph.verifier_asserts.len(); - let mut sub_status = normalize_challenge_sub_status( - scan_from_sub_status.unwrap_or_default(), - watchtower_num, - verifier_num, - ); - let mut current_status = match scan_from_status { - Some(s) => s, - None => { - if graph.committee_pre_signed() { - GraphStatus::CommitteePresigned - } else { - return Ok(GraphChainScan { - status: GraphStatus::OperatorPresigned, - sub_status, - challenge_txid: None, - watchtower_challenge_init_on_chain: false, - operator_assert_on_chain: false, - disprove: None, - }); - } - } + let mut sub_status = initial_challenge_sub_status(watchtower_num, verifier_num); + // Derive the candidate from the graph and both chains, rather than using + // the locally persisted status as the scan start. The stored status is a + // projection and can be stale after a restart or missed listener event. + let mut current_status = if graph.committee_pre_signed() { + GraphStatus::CommitteePresigned + } else { + GraphStatus::OperatorPresigned }; let prekickoff_txid = graph.cur_prekickoff.tx().compute_txid(); @@ -1364,12 +1414,12 @@ async fn scan_graph_chain_state( let take1_txid = graph.take1.tx().compute_txid(); let take2_txid = graph.take2.tx().compute_txid(); - // check if Graph has been posted on GoatChain - if current_status == GraphStatus::CommitteePresigned { - let graph_data_on_goat = goat_client.gateway_get_graph_data(&graph_id).await?; - if graph_data_on_goat.operator_pubkey != [0u8; 32] { - current_status = GraphStatus::OperatorDataPushed; - } + // GraphData is a Goat-chain fact and must be checked regardless of the + // current local projection. A fully synced node may otherwise remain at + // OperatorPresigned forever after a prior status regression. + let graph_data_on_goat = goat_client.gateway_get_graph_data(&graph_id).await?; + if graph_data_on_goat.operator_pubkey != [0u8; 32] { + current_status = GraphStatus::OperatorDataPushed; } // check if Graph has been obsoleted on GoatChain if current_status == GraphStatus::OperatorDataPushed { @@ -1681,7 +1731,7 @@ async fn scan_graph_chain_state( } #[allow(clippy::enum_variant_names)] -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, Eq, PartialEq)] enum GraphCompensateEventKind { PreKickoffSent, // OperatorDataPushed -> PreKickoff KickoffSent, // PreKickoff -> OperatorKickOff @@ -1699,12 +1749,59 @@ fn map_transition_to_event(from: GraphStatus, to: GraphStatus) -> Option Some(GraphCompensateEventKind::Take1Sent), (OperatorKickOff, Challenge) => Some(GraphCompensateEventKind::ChallengeSent), (Challenge, Disprove) => Some(GraphCompensateEventKind::DisproveSent), - (OperatorKickOff, Disprove) => Some(GraphCompensateEventKind::DisproveSent), (Challenge, OperatorTake2) => Some(GraphCompensateEventKind::Take2Sent), _ => None, } } +/// The canonical protocol ancestry used only for recovering missed local +/// messages. It is deliberately separate from transition authorization: a +/// chain scan can authorize additional recovery edges such as +/// `Obsoleted -> OperatorDataPushed`, but those edges do not imply a P2P +/// phase message should be synthesized. +fn compensation_previous_status(status: GraphStatus) -> Option { + use GraphStatus::*; + + match status { + CommitteePresigned => Some(OperatorPresigned), + OperatorDataPushed => Some(CommitteePresigned), + PreKickoff | Obsoleted | Skipped => Some(OperatorDataPushed), + OperatorKickOff => Some(PreKickoff), + OperatorTake1 | Challenge => Some(OperatorKickOff), + Disprove | OperatorTake2 => Some(Challenge), + OperatorPresigned | Created | Presigned | L2Recorded | OperatorKickOffing | Challenging + | Disproving => None, + } +} + +fn compensation_events_from( + compensation_anchor_status: GraphStatus, + final_status: GraphStatus, +) -> Option> { + if compensation_anchor_status == final_status { + return Some(vec![]); + } + + let mut reverse_path = vec![final_status]; + let mut cursor = final_status; + while cursor != compensation_anchor_status { + let previous = compensation_previous_status(cursor)?; + reverse_path.push(previous); + cursor = previous; + } + reverse_path.reverse(); + + Some( + reverse_path + .windows(2) + .filter_map(|window| match window { + [from, to] => map_transition_to_event(*from, *to), + _ => None, + }) + .collect(), + ) +} + async fn upsert_graph_compensate_message( local_db: &LocalDB, graph_id: Uuid, @@ -1712,7 +1809,14 @@ async fn upsert_graph_compensate_message( actor: Actor, message_content: GOATMessageContent, ) -> Result<()> { - let mut storage_processor = local_db.acquire().await?; + let message_type = get_goat_message_content_type(&message_content); + let message_id = generate_message_id(graph_id, message_type.to_string(), sub_type.clone()); + let mut storage_processor = local_db.start_transaction().await?; + if !storage_processor.insert_graph_compensation_marker(graph_id, &message_id).await? { + storage_processor.commit().await?; + return Ok(()); + } + upsert_message( &mut storage_processor, false, @@ -1724,7 +1828,8 @@ async fn upsert_graph_compensate_message( 0, 0, ) - .await + .await?; + storage_processor.commit().await } async fn push_graph_compensate_message( @@ -1733,8 +1838,10 @@ async fn push_graph_compensate_message( actor: Actor, message_content: GOATMessageContent, ) -> Result<()> { - let message = GOATMessage::new(actor, message_content); - push_local_unhandled_messages(local_db, graph_id, &message, 0).await + // Unlike an action retry, an inferred chain event must not reset an + // existing queued message. This makes compensation safe to retry when the + // status write committed before the message was persisted. + upsert_graph_compensate_message(local_db, graph_id, None, actor, message_content).await } #[allow(dead_code)] @@ -1756,10 +1863,9 @@ pub(crate) async fn compensate_graph_events( _btc_client: &BTCClient, instance_id: Uuid, graph_id: Uuid, - _graph: Option<&BitvmGcGraph>, + _graph: &BitvmGcGraph, scan: Option<&GraphChainScan>, - scan_from_status: Option, - compensate_from_status: GraphStatus, + compensation_anchor_status: GraphStatus, final_status: GraphStatus, ) -> Result<()> { let Some(scan) = scan else { @@ -1769,41 +1875,17 @@ pub(crate) async fn compensate_graph_events( return Ok(()); }; - let scan_start = scan_from_status.unwrap_or(compensate_from_status); - let effective_from = if scan_start.is_after(&compensate_from_status) { - scan_start - } else { - compensate_from_status - }; - if !effective_from.is_before(&final_status) { + // The action anchor is the only input used to build a recovery path. The + // persisted status is a projection and must not manufacture or suppress + // messages after a restart. Every enqueue below is idempotent. + let Some(events) = compensation_events_from(compensation_anchor_status, final_status) else { tracing::debug!( - "Skip graph compensation for {instance_id}:{graph_id}: effective_from={effective_from:?}, final_status={final_status:?}" + "Skip graph compensation without a canonical observed path for {instance_id}:{graph_id}: anchor={compensation_anchor_status:?}, final={final_status:?}" ); return Ok(()); - } - - let mut rev_path = vec![final_status]; - let mut cursor = final_status; - while cursor != effective_from { - let Some(prev) = cursor.get_previous_status() else { - tracing::debug!( - "Skip graph compensation for {instance_id}:{graph_id}: cannot walk from {final_status:?} back to {effective_from:?}" - ); - return Ok(()); - }; - rev_path.push(prev); - cursor = prev; - } - rev_path.reverse(); - - for window in rev_path.windows(2) { - let [from, to] = window else { - continue; - }; - let Some(event) = map_transition_to_event(*from, *to) else { - continue; - }; + }; + for event in events { match event { GraphCompensateEventKind::PreKickoffSent => { push_graph_compensate_message( @@ -2229,8 +2311,35 @@ pub async fn broadcast_nonstandard_tx(btc_client: &BTCClient, tx: &Transaction) /// - The mempool API URL must be configured. /// - The transaction should already be fully signed. pub async fn broadcast_tx(client: &BTCClient, tx: &Transaction) -> Result<()> { - client.broadcast(tx).await?; - Ok(()) + let txid = tx.compute_txid(); + let started_at = Instant::now(); + match client.broadcast(tx).await { + Ok(()) => { + tracing::info!( + event = "btc_tx_broadcast", + outcome = "broadcasted", + txid = %txid, + input_count = tx.input.len(), + output_count = tx.output.len(), + elapsed_ms = started_at.elapsed().as_millis() as u64, + "bitcoin transaction broadcast accepted by client" + ); + Ok(()) + } + Err(err) => { + tracing::warn!( + event = "btc_tx_broadcast", + outcome = "failed", + txid = %txid, + input_count = tx.input.len(), + output_count = tx.output.len(), + elapsed_ms = started_at.elapsed().as_millis() as u64, + error = %err, + "bitcoin transaction broadcast failed" + ); + Err(err) + } + } } pub async fn broadcast_package( @@ -2238,13 +2347,29 @@ pub async fn broadcast_package( txns: &[Transaction], fallback_on_failure: bool, ) -> Result<()> { + let txids: Vec = txns.iter().map(Transaction::compute_txid).collect(); + let started_at = Instant::now(); match client.broadcast_package(txns).await { - Ok(_) => {} + Ok(_) => { + tracing::info!( + event = "btc_tx_package_broadcast", + outcome = "broadcasted", + transaction_count = txids.len(), + txids = ?txids, + elapsed_ms = started_at.elapsed().as_millis() as u64, + "bitcoin transaction package broadcast accepted by client" + ); + } Err(e) => { if fallback_on_failure { tracing::warn!( - "broadcast_package failed: {}, falling back to broadcasting one by one", - e + event = "btc_tx_package_broadcast", + outcome = "fallback_to_individual_broadcast", + transaction_count = txids.len(), + txids = ?txids, + elapsed_ms = started_at.elapsed().as_millis() as u64, + error = %e, + "bitcoin transaction package broadcast failed; falling back to individual broadcasts" ); for tx in txns { broadcast_tx(client, tx).await?; @@ -3734,6 +3859,7 @@ pub async fn upsert_message( lock_time_until: current_time_secs() + lock_time, state: MessageState::Pending.to_string(), message_version: 0, + created_at: 0, }) .await?; } else { @@ -4185,15 +4311,16 @@ pub async fn get_current_prekickoff_tx( ) .await?; - if !graphs.is_empty() - && let Some(graph_raw_data) = - storage_processor.find_graph_raw_data(&graphs[0].graph_id).await? + if let Some(graph) = graphs.first() + && let Some(simplified_graph) = load_validated_graph_definition( + &mut storage_processor, + graph.instance_id, + graph.graph_id, + ) + .await? { - let simplified_graph = - parse_graph_raw_data(graph_raw_data.raw_data, graphs[0].graph_id).await?; - Ok(Some(( - (graphs[0].kickoff_index + 1) as u64, + (graph.kickoff_index + 1) as u64, BitvmGcGraph::from_simplified(&simplified_graph)?.next_prekickoff, ))) } else { @@ -4329,12 +4456,12 @@ pub async fn get_instance_parameters( } } -fn convert_graph(bitvm_graph: &BitvmGcGraph, current_time: i64) -> Graph { - let mut status = GraphStatus::OperatorPresigned.to_string(); - if bitvm_graph.committee_pre_signed() { - status = GraphStatus::CommitteePresigned.to_string(); - } - +fn convert_graph( + bitvm_graph: &BitvmGcGraph, + current_time: i64, + initial_status: GraphStatus, + definition_hash: String, +) -> Graph { Graph { graph_id: bitvm_graph.parameters.graph_id, instance_id: bitvm_graph.parameters.instance_parameters.instance_id, @@ -4343,9 +4470,10 @@ fn convert_graph(bitvm_graph: &BitvmGcGraph, current_time: i64) -> Graph { to_addr: "".to_string(), amount: bitvm_graph.parameters.instance_parameters.pegin_amount.to_sat() as i64, challenge_amount: bitvm_graph.parameters.challenge_amount.to_sat() as i64, - status, + status: initial_status.to_string(), sub_status: "".to_string(), operator_pubkey: bitvm_graph.parameters.operator_pubkey.to_string(), + definition_hash, cur_prekickoff_txid: Some(bitvm_graph.cur_prekickoff.finalize().compute_txid().into()), next_prekickoff: Some(bitvm_graph.next_prekickoff.finalize().compute_txid().into()), force_skip_kickoff_txid: Some( @@ -4396,37 +4524,202 @@ fn convert_graph(bitvm_graph: &BitvmGcGraph, current_time: i64) -> Graph { } } -pub async fn store_graph(local_db: &LocalDB, simple_graph: &SimplifiedBitvmGcGraph) -> Result<()> { +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum FinalizedGraphStoreOutcome { + NewlyStored, + DefinitionUpgraded, + AlreadyFinalized, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum GraphDefinitionIngestKind { + OperatorPresigned, + Finalized, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum GraphDefinitionIngestOutcome { + Inserted, + FinalizedUpgrade, + Replay, + AlreadyFinalized, +} + +/// Store a verified operator-pre-signed graph definition. +/// +/// This path is intentionally unable to advance `GraphStatus`: receiving a +/// CreateGraph message is not evidence of committee finalization. +pub(crate) async fn store_operator_presigned_graph( + local_db: &LocalDB, + simple_graph: &SimplifiedBitvmGcGraph, +) -> Result<()> { + if !simple_graph.operator_pre_signed() || simple_graph.committee_pre_signed() { + bail!(SpecialError::InvalidGraph(format!( + "graph {} is not an operator-pre-signed proposal", + simple_graph.parameters.graph_id + ))); + } + + let mut tx = local_db.start_transaction().await?; + ingest_graph_definition(&mut tx, simple_graph, GraphDefinitionIngestKind::OperatorPresigned) + .await?; + tx.commit().await?; + Ok(()) +} + +/// Store a verified finalized graph once without replacing a graph whose +/// finalized form is already known. +pub(crate) async fn store_finalized_graph_if_needed( + local_db: &LocalDB, + simple_graph: &SimplifiedBitvmGcGraph, +) -> Result { + let graph_id = simple_graph.parameters.graph_id; + if !simple_graph.operator_pre_signed() || !simple_graph.committee_pre_signed() { + bail!(SpecialError::InvalidGraph(format!("graph {graph_id} is not fully pre-signed"))); + } + let mut tx = local_db.start_transaction().await?; + let outcome = + ingest_graph_definition(&mut tx, simple_graph, GraphDefinitionIngestKind::Finalized) + .await?; + tx.commit().await?; + + Ok(match outcome { + GraphDefinitionIngestOutcome::Inserted => FinalizedGraphStoreOutcome::NewlyStored, + GraphDefinitionIngestOutcome::FinalizedUpgrade => { + FinalizedGraphStoreOutcome::DefinitionUpgraded + } + GraphDefinitionIngestOutcome::AlreadyFinalized => { + FinalizedGraphStoreOutcome::AlreadyFinalized + } + GraphDefinitionIngestOutcome::Replay => { + unreachable!("a finalized graph replay must be classified as already finalized") + } + }) +} + +async fn ingest_graph_definition( + tx: &mut StorageProcessor<'_>, + simple_graph: &SimplifiedBitvmGcGraph, + kind: GraphDefinitionIngestKind, +) -> Result { let bitvm_graph: BitvmGcGraph = BitvmGcGraph::from_simplified(simple_graph)?; let graph_id = simple_graph.parameters.graph_id; - let instance_id = simple_graph.parameters.instance_parameters.instance_id; + verify_graph_operator_pre_signatures(&bitvm_graph).map_err(|error| { + SpecialError::InvalidGraph(format!( + "graph {graph_id} has invalid operator pre-signatures: {error}" + )) + })?; + if kind == GraphDefinitionIngestKind::Finalized { + verify_graph_committee_pre_signatures(&bitvm_graph).map_err(|error| { + SpecialError::InvalidGraph(format!( + "graph {graph_id} has invalid committee pre-signatures: {error}" + )) + })?; + } let current_time = current_time_secs(); - let mut graph = convert_graph(&bitvm_graph, current_time); - let incoming_parameters_hash = simple_graph.parameters_hash()?; - - if let Some(existing_raw_data) = tx.find_graph_raw_data(&graph_id).await? { - let existing_graph = parse_graph_raw_data(existing_raw_data.raw_data, graph_id).await?; - let existing_parameters_hash = existing_graph.parameters_hash()?; - if existing_parameters_hash != incoming_parameters_hash { + let definition_hash = hex::encode(simple_graph.parameters_hash()?); + let existing_graph_row = tx.find_graph(&graph_id).await?; + let existing_raw_data = tx.find_graph_raw_data(&graph_id).await?; + let existing_graph = match (existing_graph_row.as_ref(), existing_raw_data) { + (None, None) => None, + (Some(_), None) => { bail!(SpecialError::InvalidGraph(format!( - "graph parameters changed for graph_id {graph_id}: existing={}, incoming={}", - hex::encode(existing_parameters_hash), - hex::encode(incoming_parameters_hash) + "graph {graph_id} has runtime data but no raw definition; explicit repair is required" ))); } - if existing_graph.operator_pre_signed() && !simple_graph.operator_pre_signed() { + (None, Some(_)) => { bail!(SpecialError::InvalidGraph(format!( - "graph {graph_id} cannot be downgraded after operator pre-signatures are stored" + "graph {graph_id} has raw definition but no graph row; explicit repair is required" ))); } - if existing_graph.committee_pre_signed() && !simple_graph.committee_pre_signed() { + (Some(existing_row), Some(existing_raw_data)) => { + if existing_row.definition_hash.is_empty() { + bail!(SpecialError::InvalidGraph(format!( + "graph {graph_id} is missing its stored definition hash; explicit repair is required" + ))); + } + if existing_row.definition_hash != definition_hash { + bail!(SpecialError::InvalidGraph(format!( + "graph parameters changed for graph_id {graph_id}: existing={}, incoming={definition_hash}", + existing_row.definition_hash + ))); + } + + let existing_graph = parse_graph_raw_data(existing_raw_data.raw_data, graph_id).await?; + let existing_raw_hash = hex::encode(existing_graph.parameters_hash()?); + if existing_raw_hash != existing_row.definition_hash { + bail!(SpecialError::InvalidGraph(format!( + "graph {graph_id} definition hash does not match its stored raw definition" + ))); + } + let existing_full_graph = BitvmGcGraph::from_simplified(&existing_graph)?; + verify_graph_operator_pre_signatures(&existing_full_graph).map_err(|error| { + SpecialError::InvalidGraph(format!( + "stored graph {graph_id} has invalid operator pre-signatures: {error}" + )) + })?; + if existing_full_graph.committee_pre_signed() { + verify_graph_committee_pre_signatures(&existing_full_graph).map_err(|error| { + SpecialError::InvalidGraph(format!( + "stored graph {graph_id} has invalid committee pre-signatures: {error}" + )) + })?; + } + Some(existing_graph) + } + }; + + let existing_is_finalized = + existing_graph.as_ref().is_some_and(SimplifiedBitvmGcGraph::committee_pre_signed); + if existing_is_finalized { + let existing_graph = existing_graph.as_ref().expect("checked above"); + if kind == GraphDefinitionIngestKind::Finalized && existing_graph != simple_graph { bail!(SpecialError::InvalidGraph(format!( - "graph {graph_id} cannot be downgraded after committee pre-signatures are stored" + "conflicting finalized graph for graph_id {graph_id}" ))); } + + // An old CreateGraph replay is harmless once a verified finalized + // graph is stored. Do not replace its raw signatures or runtime. + if kind == GraphDefinitionIngestKind::Finalized { + confirm_finalized_graph_definition( + tx, + bitvm_graph.parameters.instance_parameters.instance_id, + graph_id, + ) + .await?; + } + return Ok(GraphDefinitionIngestOutcome::AlreadyFinalized); } + if kind == GraphDefinitionIngestKind::OperatorPresigned + && let Some(existing_graph) = existing_graph.as_ref() + && existing_graph != simple_graph + { + bail!(SpecialError::InvalidGraph(format!( + "conflicting operator-pre-signed graph for graph_id {graph_id}" + ))); + } + if kind == GraphDefinitionIngestKind::Finalized + && let Some(existing_graph) = existing_graph.as_ref() + && existing_graph.operator_pre_sigs != simple_graph.operator_pre_sigs + { + bail!(SpecialError::InvalidGraph(format!( + "finalized graph {graph_id} changes the stored operator pre-signatures" + ))); + } + + let mut graph = convert_graph( + &bitvm_graph, + current_time, + // A graph row is always created at the operator-presigned baseline. + // Only a verified finalized definition may advance it through the + // dedicated Definition transition below. + GraphStatus::OperatorPresigned, + definition_hash.clone(), + ); + if let Some(node_info) = tx.get_node_by_btc_pub_key(&bitvm_graph.parameters.operator_pubkey.to_string()).await? { @@ -4435,26 +4728,80 @@ pub async fn store_graph(local_db: &LocalDB, simple_graph: &SimplifiedBitvmGcGra node_p2wsh_address(get_network(), &bitvm_graph.parameters.operator_pubkey).to_string(); } - tx.upsert_graph(&graph).await?; - if bitvm_graph.committee_pre_signed() { - tx.update_instance( - &InstanceUpdate::new_with_instance_id(instance_id) - .with_status(InstanceBridgeInStatus::Presigned.to_string()), + tx.upsert_graph_definition(&graph).await?; + + let outcome = if existing_graph.is_none() { + let raw_data = serialize_graph_raw_data(simple_graph, graph_id).await?; + tx.upsert_graph_raw_data( + bitvm_graph.parameters.instance_parameters.instance_id, + GraphRawData { graph_id, raw_data, created_at: current_time, updated_at: current_time }, + &definition_hash, + ) + .await?; + if kind == GraphDefinitionIngestKind::Finalized { + confirm_finalized_graph_definition( + tx, + bitvm_graph.parameters.instance_parameters.instance_id, + graph_id, + ) + .await?; + } + GraphDefinitionIngestOutcome::Inserted + } else if kind == GraphDefinitionIngestKind::Finalized { + let raw_data = serialize_graph_raw_data(simple_graph, graph_id).await?; + tx.upsert_graph_raw_data( + bitvm_graph.parameters.instance_parameters.instance_id, + GraphRawData { graph_id, raw_data, created_at: current_time, updated_at: current_time }, + &definition_hash, ) .await?; - } - let raw_data = serialize_graph_raw_data(simple_graph, graph_id).await?; - tx.upsert_graph_raw_data(GraphRawData { - graph_id, - raw_data, - created_at: current_time, - updated_at: current_time, - }) - .await?; + // Only a verified finalized definition can advance an existing locally + // operator-pre-signed graph. A later chain-observed state is retained. + confirm_finalized_graph_definition( + tx, + bitvm_graph.parameters.instance_parameters.instance_id, + graph_id, + ) + .await?; + GraphDefinitionIngestOutcome::FinalizedUpgrade + } else { + GraphDefinitionIngestOutcome::Replay + }; - tx.commit().await?; - Ok(()) + Ok(outcome) +} + +/// Confirm that a fully verified graph definition has reached the committee +/// pre-signing stage without allowing it to overwrite a later runtime state. +async fn confirm_finalized_graph_definition( + tx: &mut StorageProcessor<'_>, + instance_id: Uuid, + graph_id: Uuid, +) -> Result<()> { + match tx + .transition_graph_status( + instance_id, + graph_id, + GraphStatus::CommitteePresigned, + GraphStatusSource::Definition, + None, + ) + .await? + { + GraphStatusTransitionOutcome::Applied | GraphStatusTransitionOutcome::AlreadyCurrent => { + Ok(()) + } + GraphStatusTransitionOutcome::Rejected { current } => { + tracing::debug!( + "keep later graph status while storing finalized definition: graph={graph_id}, current={current}" + ); + Ok(()) + } + GraphStatusTransitionOutcome::NotFound => { + bail!("graph {graph_id} disappeared while confirming its finalized definition") + } + } } /// Parse raw graph data JSON string to SimplifiedBitvmGcGraph using spawn_blocking @@ -4530,19 +4877,80 @@ pub async fn serialize_graph_raw_data( } } -pub async fn get_graph( - local_db: &LocalDB, - _instance_id: Uuid, +/// Load a graph only after binding its runtime row, raw definition, canonical +/// parameters and pre-signatures together. Callers must use this instead of +/// parsing `graph_raw_data` directly. +pub(crate) async fn load_validated_graph_definition( + storage_processor: &mut StorageProcessor<'_>, + instance_id: Uuid, graph_id: Uuid, ) -> Result> { - let mut storage_process = local_db.acquire().await?; - if let Some(graph_raw_data) = storage_process.find_graph_raw_data(&graph_id).await? - && let Ok(simplified_graph) = parse_graph_raw_data(graph_raw_data.raw_data, graph_id).await + let Some(graph_row) = storage_processor.find_graph(&graph_id).await? else { + return Ok(None); + }; + if graph_row.instance_id != instance_id { + bail!(SpecialError::InvalidGraph(format!( + "graph {graph_id} belongs to instance {}, not {instance_id}", + graph_row.instance_id + ))); + } + if graph_row.definition_hash.is_empty() { + bail!(SpecialError::InvalidGraph(format!( + "graph {graph_id} is missing its definition hash; explicit repair is required" + ))); + } + + let graph_raw_data = storage_processor + .find_graph_raw_data(&graph_id) + .await? + .ok_or_else(|| { + SpecialError::InvalidGraph(format!( + "graph {graph_id} has a runtime row but no raw definition; explicit repair is required" + )) + })?; + let simplified_graph = parse_graph_raw_data(graph_raw_data.raw_data, graph_id).await?; + if simplified_graph.parameters.graph_id != graph_id + || simplified_graph.parameters.instance_parameters.instance_id != instance_id { - Ok(Some(simplified_graph)) - } else { - Ok(None) + bail!(SpecialError::InvalidGraph(format!( + "stored raw definition identifiers do not match graph {instance_id}:{graph_id}" + ))); + } + let definition_hash = hex::encode(simplified_graph.parameters_hash()?); + if definition_hash != graph_row.definition_hash { + bail!(SpecialError::InvalidGraph(format!( + "stored raw definition hash does not match graph row for {instance_id}:{graph_id}" + ))); } + + let full_graph = BitvmGcGraph::from_simplified(&simplified_graph).map_err(|error| { + SpecialError::InvalidGraph(format!( + "stored raw definition cannot rebuild graph for {instance_id}:{graph_id}: {error}" + )) + })?; + verify_graph_operator_pre_signatures(&full_graph).map_err(|error| { + SpecialError::InvalidGraph(format!( + "stored raw definition has invalid operator pre-signatures for {instance_id}:{graph_id}: {error}" + )) + })?; + if full_graph.committee_pre_signed() { + verify_graph_committee_pre_signatures(&full_graph).map_err(|error| { + SpecialError::InvalidGraph(format!( + "stored raw definition has invalid committee pre-signatures for {instance_id}:{graph_id}: {error}" + )) + })?; + } + + Ok(Some(simplified_graph)) +} + +pub async fn get_graph( + local_db: &LocalDB, + instance_id: Uuid, + graph_id: Uuid, +) -> Result> { + let mut storage_processor = local_db.acquire().await?; + load_validated_graph_definition(&mut storage_processor, instance_id, graph_id).await } pub async fn get_graph_by_instance_id_and_operator_pubkey( @@ -4554,10 +4962,8 @@ pub async fn get_graph_by_instance_id_and_operator_pubkey( if let Some(graph_id) = storage_process .get_graph_id_by_instance_id_and_operator_pubkey(&instance_id, &operator_pubkey.to_string()) .await? - && let Some(graph_raw_data) = storage_process.find_graph_raw_data(&graph_id).await? - && let Ok(simplified_graph) = parse_graph_raw_data(graph_raw_data.raw_data, graph_id).await { - Ok(Some(simplified_graph)) + load_validated_graph_definition(&mut storage_process, instance_id, graph_id).await } else { Ok(None) } @@ -4964,24 +5370,58 @@ pub async fn get_verifier_graph_params_endorsements_for_graph( .filter_map(|(k, v)| { v.verifier_index .zip(v.verifier_params_signature.as_ref()) - .map(|(index, signature)| (*k, index, signature.clone())) + .map(|(index, signature)| (*k, index, *signature)) }) .collect()) } pub async fn mark_graph_as_endorsed( local_db: &LocalDB, - _instance_id: Uuid, + instance_id: Uuid, graph_id: Uuid, ) -> Result<()> { let mut storage_processor = local_db.acquire().await?; - storage_processor.update_pegin_graph_endorsed(&graph_id, true).await?; - Ok(()) + let (_, process_data) = find_pegin_graph_process_data(&mut storage_processor, graph_id).await?; + upsert_pegin_graph_process_data( + &mut storage_processor, + graph_id, + instance_id, + true, + &process_data, + ) + .await } pub async fn get_endorsed_graph_count(local_db: &LocalDB, instance_id: Uuid) -> Result { let mut storage_processor = local_db.acquire().await?; Ok(storage_processor.get_pegin_graph_endorsed_len_by_instance_id(&instance_id, true).await? as usize) } + +pub async fn has_required_presigned_graphs(local_db: &LocalDB, instance_id: Uuid) -> Result { + Ok(get_endorsed_graph_count(local_db, instance_id).await? + >= todo_funcs::min_required_operator()) +} + +pub async fn try_transition_instance_to_presigned( + local_db: &LocalDB, + instance_id: Uuid, +) -> Result { + if !has_required_presigned_graphs(local_db, instance_id).await? { + return Ok(false); + } + + let mut storage_processor = local_db.acquire().await?; + let transitioned = storage_processor + .update_instance_status_if_current( + &instance_id, + &InstanceBridgeInStatus::UserBroadcastPeginPrepare.to_string(), + &InstanceBridgeInStatus::Presigned.to_string(), + ) + .await?; + if transitioned { + info!("Instance {instance_id} reached the required finalized graph threshold"); + } + Ok(transitioned) +} pub async fn store_committee_pub_nonce_for_instance( local_db: &LocalDB, instance_id: Uuid, @@ -5174,7 +5614,10 @@ pub async fn try_update_graph_challenge_txid( ); let mut storage_processor = local_db.acquire().await?; storage_processor - .update_graph(&GraphUpdate::new(graph_id).with_challenge_txid(spent_txid.into())) + .update_graph_runtime( + &GraphRuntimeUpdate::new(graph.instance_id, graph_id) + .with_challenge_txid(spent_txid.into()), + ) .await?; } else { info!("try_update_graph_challenge_txid no need to challenge_txid for graph {graph_id}"); @@ -5188,42 +5631,17 @@ pub async fn update_graph_status( graph_id: Uuid, new_status: GraphStatus, sub_status: Option, -) -> Result<()> { +) -> Result { let mut storage_processor = local_db.acquire().await?; - match storage_processor.find_graph(&graph_id).await? { - Some(graph) => { - if graph.status == new_status.to_string() - && let Some(ref sub_status) = sub_status - && *sub_status == ChallengeSubStatus::default() - { - warn!( - "graph: {graph_id}, new_status: {new_status} is equal old status and ChallengeSubStatus is None, so not update" - ); - return Ok(()); - } - } - None => { - warn!("graph: {graph_id} is not update, so not update"); - return Ok(()); - } - } - - if new_status == GraphStatus::CommitteePresigned { - storage_processor - .update_instance( - &InstanceUpdate::new_with_instance_id(instance_id) - .with_status(InstanceBridgeInStatus::Presigned.to_string()), - ) - .await?; - } - - let mut graph_update = GraphUpdate::new(graph_id).with_status(new_status.to_string()); - if let Some(sub_status) = sub_status { - graph_update = graph_update.with_sub_status(serde_json::to_string(&sub_status)?); - } - - storage_processor.update_graph(&graph_update).await?; - Ok(()) + storage_processor + .transition_graph_status( + instance_id, + graph_id, + new_status, + GraphStatusSource::ChainReconcile, + sub_status.map(|sub_status| serde_json::to_string(&sub_status)).transpose()?, + ) + .await } pub async fn get_graph_ids_for_instance( local_db: &LocalDB, diff --git a/node/tla/GraphLifecycle.cfg b/node/tla/GraphLifecycle.cfg new file mode 100644 index 000000000..ff744585c --- /dev/null +++ b/node/tla/GraphLifecycle.cfg @@ -0,0 +1,15 @@ +\* Full system: ChainScanNext (Bitcoin-poll) racing against GoatRaceNext +\* (GoatChain L2-event-watcher), matching the real code where both are +\* independently-scheduled tasks writing the same `graph.status` column +\* with no coordination. Expected result: NoConflictingWithdrawal, +\* TerminalStatusesAreAbsorbing and DisproveIsSticky FAIL - that failure +\* is the bug this spec exists to demonstrate, not a spec error. +SPECIFICATION FairFullSpec + +CHECK_DEADLOCK FALSE + +INVARIANT TypeOK + +PROPERTY NoConflictingWithdrawal +PROPERTY DisproveIsSticky +PROPERTY TerminalStatusesAreAbsorbing diff --git a/node/tla/GraphLifecycle.tla b/node/tla/GraphLifecycle.tla new file mode 100644 index 000000000..cfceac541 --- /dev/null +++ b/node/tla/GraphLifecycle.tla @@ -0,0 +1,188 @@ +---- MODULE GraphLifecycle ---- +(***************************************************************************) +(* Formal model of a single BitVM2 graph's `status` column, built from the *) +(* ACTUAL Rust implementation (not the README, which was found to be *) +(* stale in several ways - see git history / PR description for details). *) +(* Every action below cites the exact code it models. *) +(* *) +(* Ground truth was established by fully reading: *) +(* - scan_graph_chain_state, node/src/utils.rs:1328-1681 (Bitcoin-poll *) +(* status derivation, the "official" state machine) *) +(* - handle_withdraw_paths_events / handle_withdraw_disproved_events, *) +(* node/src/scheduled_tasks/event_watch_task.rs:392-502 (GoatChain *) +(* L2-event-driven status writes - a SEPARATE, independently *) +(* scheduled writer to the same column) *) +(* - StorageProcessor::update_graph / update_graph_status, *) +(* crates/store/src/localdb.rs:1290-1297 and node/src/utils.rs: *) +(* 5185-5223 (both perform a raw `UPDATE graph SET status=?` with NO *) +(* guard against the new status being older/inconsistent with the *) +(* current one - confirmed by reading both implementations) *) +(* *) +(* Known deliberate simplifications (do not affect the properties below): *) +(* - OperatorPresigned -> CommitteePresigned is modeled as one atomic *) +(* action. In reality it fires once an N-of-N committee-signature *) +(* quorum completes (node/src/action.rs:488-540, try_finalize_graph); *) +(* quorum counting across individual committee members is out of *) +(* scope for a single-graph spec. *) +(* - The four distinct Challenge-state disprove detectors (guardian, *) +(* watchtower-flow, per-verifier, connector-d/PubinDisprove - all in *) +(* utils.rs:1490-1670) are collapsed into one `Disprove` action, since *) +(* they differ only in `disprove_type`/`disprove_index` bookkeeping, *) +(* which never gates any further `status` transition (confirmed: the *) +(* ChallengeSubStatus fields and its is_watchtower_challenge_success *) +(* method have zero call sites anywhere in the repo - dead code). *) +(***************************************************************************) +EXTENDS GraphTopology + +VARIABLE status + +vars == <> + +TypeOK == status \in AllStatuses + +Init == status = "OperatorPresigned" + +-------------------------------------------------------------------------- +(* ChainScan actions: the Bitcoin/GoatChain-poll state machine, *) +(* scan_graph_chain_state, utils.rs:1328-1681. This is the ONLY writer *) +(* considered by the CORE-ONLY config. *) + +CommitteePresign == + /\ status = "OperatorPresigned" + /\ status' = "CommitteePresigned" + +OperatorPushL2Data == + /\ status = "CommitteePresigned" + /\ status' = "OperatorDataPushed" \* utils.rs:1368-1373 + +BecomeObsoletedBeforeData == \* utils.rs:1386-1414 + /\ status \in {"OperatorPresigned", "CommitteePresigned"} + /\ status' = "Obsoleted" + +BecomeObsoletedAfterData == \* utils.rs:1375-1384 + /\ status = "OperatorDataPushed" + /\ status' = "Obsoleted" + +ConfirmPreKickoff == \* utils.rs:1386-1405 + /\ status = "OperatorDataPushed" + /\ status' = "PreKickoff" + +\* utils.rs:1418-1434 - fires from PreKickoff OR Obsoleted (resurrection). +BroadcastKickoff == + /\ status \in {"PreKickoff", "Obsoleted"} + /\ status' = "OperatorKickOff" + +ForceSkip == \* utils.rs:1418-1434 + /\ status \in {"PreKickoff", "Obsoleted"} + /\ status' = "Skipped" + +Take1 == \* utils.rs:1462-1489 + /\ status = "OperatorKickOff" + /\ status' = "OperatorTake1" + +OpenChallenge == \* utils.rs:1462-1489 + /\ status = "OperatorKickOff" + /\ status' = "Challenge" + +\* utils.rs:1448-1461 - guardian disprove bypasses Challenge entirely; this +\* edge is absent from the README's Fig-04-1/4-2 diagram. +GuardianDisproveFromKickoff == + /\ status = "OperatorKickOff" + /\ status' = "Disprove" + +Take2 == \* utils.rs:1661-1670 + /\ status = "Challenge" + /\ status' = "OperatorTake2" + +\* Collapses all 4 real disprove detectors - see header comment. +Disprove == \* utils.rs:1490-1660 + /\ status = "Challenge" + /\ status' = "Disprove" + +ChainScanNext == + \/ CommitteePresign + \/ OperatorPushL2Data + \/ BecomeObsoletedBeforeData + \/ BecomeObsoletedAfterData + \/ ConfirmPreKickoff + \/ BroadcastKickoff + \/ ForceSkip + \/ Take1 + \/ OpenChallenge + \/ GuardianDisproveFromKickoff + \/ Take2 + \/ Disprove + +-------------------------------------------------------------------------- +(* GoatRace actions: node/src/scheduled_tasks/event_watch_task.rs, a *) +(* SEPARATE scheduled task that reacts to GoatChain (L2) events and *) +(* writes `status` via StorageProcessor::update_graph - a raw SQL UPDATE *) +(* with no WHERE-clause guard on the current status *) +(* (crates/store/src/localdb.rs:1290-1297) and no coordination with *) +(* ChainScanNext above. Modeled exactly as read: no precondition on the *) +(* current status at all, matching "any -> X" in the real code. *) + +GoatPostGraphData == status' = "OperatorDataPushed" \* event_watch_task.rs:930-947 +GoatWithdrawHappy == status' = "OperatorTake1" \* event_watch_task.rs:392-449, WithdrawHappyEvent +GoatWithdrawUnhappy == status' = "OperatorTake2" \* event_watch_task.rs:392-449, WithdrawUnhappyEvent +GoatWithdrawDisproved == status' = "Disprove" \* event_watch_task.rs:451-502 + +GoatRaceNext == + \/ GoatPostGraphData + \/ GoatWithdrawHappy + \/ GoatWithdrawUnhappy + \/ GoatWithdrawDisproved + +\* The actual fix (update_graph_status_guarded, node/src/utils.rs) has TWO +\* guards, not one - a first draft with only guard (1) still let TLC find a +\* liveness bug: GoatPostGraphData could still knock an OperatorKickOff graph +\* back to OperatorDataPushed, which combined with Obsoleted's resurrection +\* edges cycles forever and never reaches a terminal status. +\* (1) refuse to write OFF a closed/terminal status onto something else. +\* (2) refuse OperatorDataPushed specifically unless the graph hasn't yet +\* progressed past it (OperatorDataPushed only causally precedes +\* PreKickoff in the real system - a correctly functioning node never +\* needs to "re-push" data after kickoff has already happened). +GoatRaceNextGuarded == + \/ (status \notin TerminalStatuses + /\ status \in {"OperatorPresigned", "CommitteePresigned", "OperatorDataPushed"} + /\ GoatPostGraphData) + \/ (status \notin TerminalStatuses /\ GoatWithdrawHappy) + \/ (status \notin TerminalStatuses /\ GoatWithdrawUnhappy) + \/ (status \notin TerminalStatuses /\ GoatWithdrawDisproved) + +-------------------------------------------------------------------------- +Next == ChainScanNext \/ GoatRaceNext +NextFixed == ChainScanNext \/ GoatRaceNextGuarded + +CoreSpec == Init /\ [][ChainScanNext]_vars +FairCoreSpec == CoreSpec /\ WF_vars(ChainScanNext) + +FullSpec == Init /\ [][Next]_vars +FairFullSpec == FullSpec /\ WF_vars(Next) + +FullSpecFixed == Init /\ [][NextFixed]_vars +FairFullSpecFixed == FullSpecFixed /\ WF_vars(NextFixed) + +-------------------------------------------------------------------------- +(* Properties. Checked against FairCoreSpec (GraphLifecycleCoreOnly.cfg, *) +(* expected to hold - the chain-scan state machine alone is sound) and *) +(* against FairFullSpec (GraphLifecycle.cfg, expected to VIOLATE - that *) +(* is the actual bug this spec was built to demonstrate). *) + +NoStatusSkipping == [][(status # status' => <> \in AllowedTransitions)]_vars + +TerminalStatusesAreAbsorbing == [][(status \in TerminalStatuses => status' = status)]_vars + +\* The sharpest statement of the real risk: the two payout paths (happy-path +\* Take1 vs challenge-path Take2) must never both be recorded for one graph, +\* and a recorded Disprove (evidence an operator cheated) must never be lost. +NoConflictingWithdrawal == + /\ [][(status = "OperatorTake1" => status' # "OperatorTake2")]_vars + /\ [][(status = "OperatorTake2" => status' # "OperatorTake1")]_vars + +DisproveIsSticky == [][(status = "Disprove" => status' = "Disprove")]_vars + +EventuallyTerminal == <>(status \in TerminalStatuses) + +==== diff --git a/node/tla/GraphLifecycleCoreOnly.cfg b/node/tla/GraphLifecycleCoreOnly.cfg new file mode 100644 index 000000000..e47d7efbc --- /dev/null +++ b/node/tla/GraphLifecycleCoreOnly.cfg @@ -0,0 +1,15 @@ +\* Baseline: the chain-scan state machine alone (ChainScanNext), no GoatChain +\* event-watcher race. Expected result: all properties hold. This isolates +\* whether the CORE lifecycle logic is sound by itself, so that any failure +\* in GraphLifecycle.cfg can be attributed specifically to the second writer. +SPECIFICATION FairCoreSpec + +CHECK_DEADLOCK FALSE + +INVARIANT TypeOK + +PROPERTY NoStatusSkipping +PROPERTY TerminalStatusesAreAbsorbing +PROPERTY NoConflictingWithdrawal +PROPERTY DisproveIsSticky +PROPERTY EventuallyTerminal diff --git a/node/tla/GraphLifecycleFineGrained.cfg b/node/tla/GraphLifecycleFineGrained.cfg new file mode 100644 index 000000000..90e471899 --- /dev/null +++ b/node/tla/GraphLifecycleFineGrained.cfg @@ -0,0 +1,5 @@ +SPECIFICATION Spec + +INVARIANT TypeOK +PROPERTY TerminalStatusesAreAbsorbing +PROPERTY NoConflictingWithdrawal diff --git a/node/tla/GraphLifecycleFineGrained.tla b/node/tla/GraphLifecycleFineGrained.tla new file mode 100644 index 000000000..08f0bb315 --- /dev/null +++ b/node/tla/GraphLifecycleFineGrained.tla @@ -0,0 +1,125 @@ +---- MODULE GraphLifecycleFineGrained ---- +(***************************************************************************) +(* GraphLifecycle.tla / GraphLifecycleFixed.cfg model update_graph_status_guarded's *) +(* guard-check-then-write as ONE atomic TLA+ step. The real function is NOT *) +(* atomic: `find_graph(...).await` (the read) and `update_graph(...).await` *) +(* (the write) are two separate yield points (node/src/utils.rs). This *) +(* module exposes that gap explicitly with PlusCal: each writer reads a *) +(* snapshot, decides using the SAME guard logic as the real fix, and only *) +(* writes later - so TLC can explore another writer's full read-decide- *) +(* write completing in between. *) +(***************************************************************************) +EXTENDS GraphTopology + +GoatTargets == {"OperatorDataPushed", "OperatorTake1", "OperatorTake2", "Disprove"} + +(* The exact guard from node/src/utils.rs's update_graph_status_guarded, + applied to a (possibly stale) `current` reading. *) +GuardOK(current, target) == + /\ ~(current \in TerminalStatuses /\ current # target) + /\ (target = "OperatorDataPushed" => + current \in {"OperatorPresigned", "CommitteePresigned", "OperatorDataPushed"}) + +(*--algorithm GraphWriteRace +variables status = "OperatorPresigned"; + +process ChainScan = "ChainScan" +variables csSnap = "", csTarget = ""; +begin + CSRead: + while TRUE do + csSnap := status; + with t \in ({t2 \in AllStatuses : <> \in AllowedTransitions} \cup {csSnap}) do + csTarget := t; + end with; + CSWrite: + if GuardOK(csSnap, csTarget) then + status := csTarget; + end if; + end while; +end process; + +process GoatRace = "GoatRace" +variables grSnap = "", grTarget = ""; +begin + GRRead: + while TRUE do + grSnap := status; + with t \in GoatTargets do + grTarget := t; + end with; + GRWrite: + if GuardOK(grSnap, grTarget) then + status := grTarget; + end if; + end while; +end process; + +end algorithm; *) +\* BEGIN TRANSLATION +VARIABLES status, pc, csSnap, csTarget, grSnap, grTarget + +vars == << status, pc, csSnap, csTarget, grSnap, grTarget >> + +ProcSet == {"ChainScan"} \cup {"GoatRace"} + +Init == (* Global variables *) + /\ status = "OperatorPresigned" + (* Process ChainScan *) + /\ csSnap = "" + /\ csTarget = "" + (* Process GoatRace *) + /\ grSnap = "" + /\ grTarget = "" + /\ pc = [self \in ProcSet |-> CASE self = "ChainScan" -> "CSRead" + [] self = "GoatRace" -> "GRRead"] + +CSRead == /\ pc["ChainScan"] = "CSRead" + /\ csSnap' = status + /\ \E t \in ({t2 \in AllStatuses : <> \in AllowedTransitions} \cup {csSnap'}): + csTarget' = t + /\ pc' = [pc EXCEPT !["ChainScan"] = "CSWrite"] + /\ UNCHANGED << status, grSnap, grTarget >> + +CSWrite == /\ pc["ChainScan"] = "CSWrite" + /\ IF GuardOK(csSnap, csTarget) + THEN /\ status' = csTarget + ELSE /\ TRUE + /\ UNCHANGED status + /\ pc' = [pc EXCEPT !["ChainScan"] = "CSRead"] + /\ UNCHANGED << csSnap, csTarget, grSnap, grTarget >> + +ChainScan == CSRead \/ CSWrite + +GRRead == /\ pc["GoatRace"] = "GRRead" + /\ grSnap' = status + /\ \E t \in GoatTargets: + grTarget' = t + /\ pc' = [pc EXCEPT !["GoatRace"] = "GRWrite"] + /\ UNCHANGED << status, csSnap, csTarget >> + +GRWrite == /\ pc["GoatRace"] = "GRWrite" + /\ IF GuardOK(grSnap, grTarget) + THEN /\ status' = grTarget + ELSE /\ TRUE + /\ UNCHANGED status + /\ pc' = [pc EXCEPT !["GoatRace"] = "GRRead"] + /\ UNCHANGED << csSnap, csTarget, grSnap, grTarget >> + +GoatRace == GRRead \/ GRWrite + +Next == ChainScan \/ GoatRace + +Spec == Init /\ [][Next]_vars + +\* END TRANSLATION + +TypeOK == status \in AllStatuses + +TerminalStatusesAreAbsorbing == [][(status \in TerminalStatuses => status' = status)]_status + +NoConflictingWithdrawal == + /\ [][(status = "OperatorTake1" => status' # "OperatorTake2")]_status + /\ [][(status = "OperatorTake2" => status' # "OperatorTake1")]_status + +==== diff --git a/node/tla/GraphLifecycleFineGrainedFixed.cfg b/node/tla/GraphLifecycleFineGrainedFixed.cfg new file mode 100644 index 000000000..b39fd13e2 --- /dev/null +++ b/node/tla/GraphLifecycleFineGrainedFixed.cfg @@ -0,0 +1,4 @@ +SPECIFICATION Spec +INVARIANT TypeOK +PROPERTY TerminalStatusesAreAbsorbing +PROPERTY NoConflictingWithdrawal diff --git a/node/tla/GraphLifecycleFineGrainedFixed.tla b/node/tla/GraphLifecycleFineGrainedFixed.tla new file mode 100644 index 000000000..0fe4c5b77 --- /dev/null +++ b/node/tla/GraphLifecycleFineGrainedFixed.tla @@ -0,0 +1,107 @@ +---- MODULE GraphLifecycleFineGrainedFixed ---- +(***************************************************************************) +(* Companion to GraphLifecycleFineGrained.tla, which demonstrated that *) +(* splitting the guard-check from the write into two separate steps (read, *) +(* then later write - exactly what a `SELECT` followed by an `UPDATE` in *) +(* application code does) leaves a real gap, even with the correct guard *) +(* logic on both sides. *) +(* *) +(* The actual fix (node/src/utils.rs + crates/store/src/localdb.rs) moves *) +(* the guard into the SQL statement itself: `GraphUpdate::only_if_status_in` *) +(* becomes a `WHERE status IN (...)` clause on the same `UPDATE` that sets *) +(* the new status, so the read-the-current-value-and-decide step and the *) +(* write happen as ONE indivisible database statement - there is no gap *) +(* for another writer's statement to land inside. *) +(* *) +(* This module models exactly that: each writer's read+decide+write is ONE *) +(* PlusCal label (one atomic step), reusing the identical guard and target *) +(* sets from GraphLifecycleFineGrained.tla. If the properties that failed *) +(* there hold here, that confirms the SQL-level fix - not just the *) +(* application-level guard logic - is what actually closes the gap. *) +(***************************************************************************) +EXTENDS GraphTopology + +GoatTargets == {"OperatorDataPushed", "OperatorTake1", "OperatorTake2", "Disprove"} + +GuardOK(current, target) == + /\ ~(current \in TerminalStatuses /\ current # target) + /\ (target = "OperatorDataPushed" => + current \in {"OperatorPresigned", "CommitteePresigned", "OperatorDataPushed"}) + +(*--algorithm GraphWriteRaceFixed +variables status = "OperatorPresigned"; + +process ChainScan = "ChainScan" +variables csTarget = ""; +begin + CSStep: + while TRUE do + with t \in ({t2 \in AllStatuses : <> \in AllowedTransitions} \cup {status}) do + csTarget := t; + end with; + if GuardOK(status, csTarget) then + status := csTarget; + end if; + end while; +end process; + +process GoatRace = "GoatRace" +variables grTarget = ""; +begin + GRStep: + while TRUE do + with t \in GoatTargets do + grTarget := t; + end with; + if GuardOK(status, grTarget) then + status := grTarget; + end if; + end while; +end process; + +end algorithm; *) +\* BEGIN TRANSLATION +VARIABLES status, csTarget, grTarget + +vars == << status, csTarget, grTarget >> + +ProcSet == {"ChainScan"} \cup {"GoatRace"} + +Init == (* Global variables *) + /\ status = "OperatorPresigned" + (* Process ChainScan *) + /\ csTarget = "" + (* Process GoatRace *) + /\ grTarget = "" + +ChainScan == /\ \E t \in ({t2 \in AllStatuses : <> \in AllowedTransitions} \cup {status}): + csTarget' = t + /\ IF GuardOK(status, csTarget') + THEN /\ status' = csTarget' + ELSE /\ TRUE + /\ UNCHANGED status + /\ UNCHANGED grTarget + +GoatRace == /\ \E t \in GoatTargets: + grTarget' = t + /\ IF GuardOK(status, grTarget') + THEN /\ status' = grTarget' + ELSE /\ TRUE + /\ UNCHANGED status + /\ UNCHANGED csTarget + +Next == ChainScan \/ GoatRace + +Spec == Init /\ [][Next]_vars + +\* END TRANSLATION + +TypeOK == status \in AllStatuses + +TerminalStatusesAreAbsorbing == [][(status \in TerminalStatuses => status' = status)]_status + +NoConflictingWithdrawal == + /\ [][(status = "OperatorTake1" => status' # "OperatorTake2")]_status + /\ [][(status = "OperatorTake2" => status' # "OperatorTake1")]_status + +==== diff --git a/node/tla/GraphLifecycleFixed.cfg b/node/tla/GraphLifecycleFixed.cfg new file mode 100644 index 000000000..a94b23760 --- /dev/null +++ b/node/tla/GraphLifecycleFixed.cfg @@ -0,0 +1,16 @@ +\* Same full system as GraphLifecycle.cfg (ChainScanNext racing the GoatChain +\* event-watcher), but with the actual fix applied: GoatRaceNextGuarded +\* mirrors update_graph_status_guarded's closed-status check +\* (node/src/utils.rs). Expected result: all properties now hold - this +\* config is the proof that the code fix closes the gap GraphLifecycle.cfg +\* found, not just a claim. +SPECIFICATION FairFullSpecFixed + +CHECK_DEADLOCK FALSE + +INVARIANT TypeOK + +PROPERTY NoConflictingWithdrawal +PROPERTY DisproveIsSticky +PROPERTY TerminalStatusesAreAbsorbing +PROPERTY EventuallyTerminal diff --git a/node/tla/GraphTopology.tla b/node/tla/GraphTopology.tla new file mode 100644 index 000000000..0f792802e --- /dev/null +++ b/node/tla/GraphTopology.tla @@ -0,0 +1,46 @@ +---- MODULE GraphTopology ---- +(***************************************************************************) +(* The real Graph.status state machine, shared by every spec that models *) +(* it (GraphLifecycle.tla, GraphLifecycleFineGrained.tla, *) +(* GraphLifecycleFineGrainedFixed.tla) - a single source of truth for the *) +(* status set and the real transition edge set, instead of three *) +(* independent copies that could silently drift out of sync with each *) +(* other and with the real code. *) +(* *) +(* Ground truth: scan_graph_chain_state, node/src/utils.rs:1328-1681. *) +(***************************************************************************) + +AllStatuses == { + "OperatorPresigned", "CommitteePresigned", "OperatorDataPushed", + "PreKickoff", "OperatorKickOff", "Challenge", + "OperatorTake1", "OperatorTake2", "Skipped", "Obsoleted", "Disprove" +} + +\* Obsoleted is NOT terminal in the real code. utils.rs:1418-1434 re-checks +\* `matches!(current_status, PreKickoff | Obsoleted)` and can move an +\* Obsoleted graph to OperatorKickOff or Skipped if a kickoff tx is later +\* observed on-chain. +TerminalStatuses == {"OperatorTake1", "OperatorTake2", "Skipped", "Disprove"} + +\* The real edge set from scan_graph_chain_state - used directly by +\* GraphLifecycleCoreOnly.cfg to check the chain-scan state machine is +\* internally sound in isolation, and by NoStatusSkipping in GraphLifecycle.tla. +AllowedTransitions == { + <<"OperatorPresigned", "CommitteePresigned">>, + <<"CommitteePresigned", "OperatorDataPushed">>, + <<"OperatorDataPushed", "PreKickoff">>, + <<"OperatorPresigned", "Obsoleted">>, + <<"CommitteePresigned", "Obsoleted">>, + <<"OperatorDataPushed", "Obsoleted">>, + <<"PreKickoff", "OperatorKickOff">>, + <<"Obsoleted", "OperatorKickOff">>, + <<"PreKickoff", "Skipped">>, + <<"Obsoleted", "Skipped">>, + <<"OperatorKickOff", "OperatorTake1">>, + <<"OperatorKickOff", "Challenge">>, + <<"OperatorKickOff", "Disprove">>, + <<"Challenge", "OperatorTake2">>, + <<"Challenge", "Disprove">> +} + +==== diff --git a/node/tla/InstanceBridgeOutRace.cfg b/node/tla/InstanceBridgeOutRace.cfg new file mode 100644 index 000000000..78e410071 --- /dev/null +++ b/node/tla/InstanceBridgeOutRace.cfg @@ -0,0 +1,6 @@ +\* Models the CURRENT, actual code (all writers unconditional) - expected +\* to FAIL. This is a live bug, not a historical artifact. +SPECIFICATION FairSpec +CHECK_DEADLOCK FALSE +INVARIANT TypeOK +PROPERTY TerminalStatusesAreAbsorbing diff --git a/node/tla/InstanceBridgeOutRace.tla b/node/tla/InstanceBridgeOutRace.tla new file mode 100644 index 000000000..febc366dc --- /dev/null +++ b/node/tla/InstanceBridgeOutRace.tla @@ -0,0 +1,80 @@ +---- MODULE InstanceBridgeOutRace ---- +(***************************************************************************) +(* Formal model of a race found while auditing every remaining stateful *) +(* enum after GraphStatus/InstanceBridgeInStatus (see audit/TLAPlus-*.md). *) +(* *) +(* `InstanceBridgeOutStatus` (crates/store/src/schema.rs:223-229) is *) +(* written from three independently-scheduled, uncoordinated tasks with *) +(* no shared transaction spanning read+decide+write in any of them: *) +(* - the RPC-service task (node/src/rpc_service/handler/bitvm2_handler.rs, *) +(* `bridge_out_init_tag`) - stale-read-then-full-row-upsert, sets *) +(* Initialize. *) +(* - the GoatChain L2-event watcher (node/src/scheduled_tasks/ *) +(* event_watch_task.rs), 5s tokio task - unconditional targeted *) +(* `update_instance` on SwapClaimEvent/SwapRefundEvent, sets *) +(* Claim/Refund with NO status precondition (`InstanceUpdate`'s WHERE *) +(* clause is only `hex(instance_id)=?`, confirmed via *) +(* crates/store/src/localdb.rs - no `with_only_if_status_in` exists *) +(* anywhere in the codebase for this entity). *) +(* - the maintenance task (node/src/scheduled_tasks/ *) +(* instance_maintenance_tasks.rs, `instance_bridge_out_monitor`), 10s *) +(* tokio task - batch-reads a stale snapshot, then per-row does an *) +(* unconditional targeted `update_instance` to Timeout with no re- *) +(* check at write time. *) +(* *) +(* Modeled the same way GraphLifecycle.tla models its GoatRace actions: *) +(* every writer is unconditional on the CURRENT status - that's the *) +(* verified real behavior, not a simplification of it. *) +(***************************************************************************) +Statuses == {"Initialize", "Claim", "Timeout", "Refund"} + +\* Once a bridge-out instance is claimed, timed out, or refunded, that is +\* meant to be the final outcome - Initialize is the only non-terminal +\* status. +TerminalStatuses == {"Claim", "Timeout", "Refund"} + +VARIABLE status +vars == <> + +TypeOK == status \in Statuses + +Init == status = "Initialize" + +-------------------------------------------------------------------------- +(* Every action below is unconditional on the current status - confirmed *) +(* real behavior for all three writers, not a modeling simplification. *) + +RpcStaleInit == status' = "Initialize" \* bitvm2_handler.rs bridge_out_init_tag, stale full-row upsert +WatchEventClaim == status' = "Claim" \* event_watch_task.rs handle_swap_claim_events +WatchEventRefund == status' = "Refund" \* event_watch_task.rs handle_swap_refund_events +MaintenanceTimeout == status' = "Timeout" \* instance_maintenance_tasks.rs instance_bridge_out_monitor + +Next == + \/ RpcStaleInit + \/ WatchEventClaim + \/ WatchEventRefund + \/ MaintenanceTimeout + +Spec == Init /\ [][Next]_vars +FairSpec == Spec /\ WF_vars(Next) + +\* Proposed fix design (not applied to code - same atomic-CAS pattern as +\* the GraphStatus/InstanceBridgeInStatus fixes: only write if the row +\* isn't already in a terminal status, folded into the UPDATE's WHERE +\* clause so there's no read-then-write gap). +NextFixed == + \/ (status \notin TerminalStatuses /\ RpcStaleInit) + \/ (status \notin TerminalStatuses /\ WatchEventClaim) + \/ (status \notin TerminalStatuses /\ WatchEventRefund) + \/ (status \notin TerminalStatuses /\ MaintenanceTimeout) + +SpecFixed == Init /\ [][NextFixed]_vars +FairSpecFixed == SpecFixed /\ WF_vars(NextFixed) + +-------------------------------------------------------------------------- +(* Safety property: once a bridge-out instance reaches a final outcome, *) +(* it never changes again - e.g. a successfully Claimed instance must *) +(* never be silently reset by a stale maintenance-task Timeout write. *) +TerminalStatusesAreAbsorbing == [][(status \in TerminalStatuses => status' = status)]_status + +==== diff --git a/node/tla/InstanceBridgeOutRaceFixed.cfg b/node/tla/InstanceBridgeOutRaceFixed.cfg new file mode 100644 index 000000000..ffdbda46c --- /dev/null +++ b/node/tla/InstanceBridgeOutRaceFixed.cfg @@ -0,0 +1,5 @@ +\* Proposed fix design (verified, NOT applied to code) - expected to pass. +SPECIFICATION FairSpecFixed +CHECK_DEADLOCK FALSE +INVARIANT TypeOK +PROPERTY TerminalStatusesAreAbsorbing diff --git a/node/tla/InstancePresigned.tla b/node/tla/InstancePresigned.tla new file mode 100644 index 000000000..b984c0453 --- /dev/null +++ b/node/tla/InstancePresigned.tla @@ -0,0 +1,76 @@ +---- MODULE InstancePresigned ---- +(***************************************************************************) +(* Formal model of the SECOND race found by the same audit that produced *) +(* GraphLifecycle.tla: `Instance.status` (InstanceBridgeInStatus, *) +(* crates/store/src/schema.rs:194-217) has NO terminal-status protection *) +(* anywhere in the codebase (confirmed: `grep -n "impl InstanceBridgeInStatus"` *) +(* has zero hits, unlike GraphStatus which has is_closed()). *) +(* *) +(* This is deliberately NOT a full Instance-lifecycle spec - InstanceBridgeInStatus's *) +(* full transition graph has not been ground-truth-traced with the same *) +(* rigor GraphLifecycle.tla's was. It models exactly the one confirmed *) +(* regression: `store_graph` (node/src/utils.rs, ~4438) and *) +(* `update_graph_status_guarded` (node/src/utils.rs, ~5195) both write *) +(* `InstanceBridgeInStatus::Presigned` unconditionally as a side effect of *) +(* a graph reaching `CommitteePresigned`, reachable from independent P2P- *) +(* message handlers (handle.rs) and chain-rescans (graph_maintenance_tasks) *) +(* with no coordination - so a stale/replayed event could revert an *) +(* instance that already progressed past Presigned (e.g. to *) +(* RelayerL1Broadcasted or RelayerL2Minted, written by instance_btc_tx_monitor *) +(* and handle_bridge_in_events respectively) back down to Presigned. *) +(* *) +(* "Advanced" below is an abstract stand-in for every real status that *) +(* causally follows Presigned (RelayerL1Broadcasted, RelayerL2Minted, *) +(* RelayerL2MintedFailed, PresignedFailed, Timeout, UserCanceled, *) +(* UserDiscarded, NoEnoughCommitteesAnswered) - the guard treats all of *) +(* them identically (refuse), so collapsing them loses no precision for *) +(* the property being checked here. *) +(***************************************************************************) + +Statuses == {"Early", "Presigned", "Advanced"} + +VARIABLE status +vars == <> + +TypeOK == status \in Statuses + +Init == status = "Early" + +-------------------------------------------------------------------------- +(* The real, legitimate forward progression (instance_btc_tx_monitor, *) +(* handle_bridge_in_events, etc. - the various writers that correctly *) +(* advance status once real on-chain/L2 events are observed). *) + +AdvancePastPresigned == + /\ status = "Presigned" + /\ status' = "Advanced" + +-------------------------------------------------------------------------- +(* The race: store_graph / update_graph_status_guarded writing Presigned *) +(* as a side effect, reachable from an independent P2P/rescan path. *) + +\* As originally written: unconditional, matching the confirmed bug. +SetPresignedUnguarded == + status' = "Presigned" + +\* The actual fix: set_instance_presigned_guarded, node/src/utils.rs - +\* only writes Presigned if the instance hasn't already moved past it. +SetPresignedGuarded == + /\ status \in {"Early", "Presigned"} + /\ status' = "Presigned" + +-------------------------------------------------------------------------- +NextBug == AdvancePastPresigned \/ SetPresignedUnguarded +NextFixed == AdvancePastPresigned \/ SetPresignedGuarded + +SpecBug == Init /\ [][NextBug]_vars +SpecFixed == Init /\ [][NextFixed]_vars + +-------------------------------------------------------------------------- +(* The property: once an instance is Advanced, it must never regress. This *) +(* is deliberately not "terminal is absorbing" in the GraphStatus sense - *) +(* Advanced isn't necessarily terminal in reality, it just must never be *) +(* undone by the Presigned side-effect write specifically. *) +NeverRegressPastPresigned == [][(status = "Advanced" => status' = "Advanced")]_vars + +==== diff --git a/node/tla/InstancePresignedBug.cfg b/node/tla/InstancePresignedBug.cfg new file mode 100644 index 000000000..d701c5dd1 --- /dev/null +++ b/node/tla/InstancePresignedBug.cfg @@ -0,0 +1,4 @@ +SPECIFICATION SpecBug +CHECK_DEADLOCK FALSE +INVARIANT TypeOK +PROPERTY NeverRegressPastPresigned diff --git a/node/tla/InstancePresignedFixed.cfg b/node/tla/InstancePresignedFixed.cfg new file mode 100644 index 000000000..4aaf1854d --- /dev/null +++ b/node/tla/InstancePresignedFixed.cfg @@ -0,0 +1,4 @@ +SPECIFICATION SpecFixed +CHECK_DEADLOCK FALSE +INVARIANT TypeOK +PROPERTY NeverRegressPastPresigned diff --git a/node/tla/MessageStateRace.cfg b/node/tla/MessageStateRace.cfg new file mode 100644 index 000000000..13a129bf2 --- /dev/null +++ b/node/tla/MessageStateRace.cfg @@ -0,0 +1,6 @@ +\* Models the CURRENT, actual code (upsert_message resurrects unconditionally) +\* - expected to FAIL. This is a live bug, not a historical artifact. +SPECIFICATION FairSpec +CHECK_DEADLOCK FALSE +INVARIANT TypeOK +PROPERTY TerminalStatusesAreAbsorbing diff --git a/node/tla/MessageStateRace.tla b/node/tla/MessageStateRace.tla new file mode 100644 index 000000000..9ab76eaba --- /dev/null +++ b/node/tla/MessageStateRace.tla @@ -0,0 +1,78 @@ +---- MODULE MessageStateRace ---- +(***************************************************************************) +(* Formal model of a race found while auditing every remaining stateful *) +(* enum after GraphStatus/InstanceBridgeInStatus (see audit/TLAPlus-*.md). *) +(* *) +(* `MessageState` (crates/store/src/schema.rs:431-437: Pending, Processed, *) +(* Failed, Expired, Cancelled) tracks P2P message delivery/processing *) +(* status. Unlike the GraphStatus/InstanceBridgeOutStatus findings, the *) +(* "cancel" writer here IS correctly guarded - `update_messages_state_by_ *) +(* business_id` (crates/store/src/localdb.rs) does a real CAS: `UPDATE ... *) +(* WHERE business_id=? AND state='Pending'`, called from *) +(* node/src/scheduled_tasks/event_watch_task.rs's handle_withdraw_paths_/ *) +(* disproved_events when a graph reaches a closed on-chain status *) +(* (OperatorTake1/OperatorTake2/Disprove) - bulk-cancelling any still- *) +(* Pending message for that graph as moot. *) +(* *) +(* The bug is on the OTHER side: `upsert_message` (node/src/utils.rs, *) +(* called by push_local_unhandled_messages - the generic "defer/retry *) +(* this p2p message" primitive used ~30 times across node/src/handle.rs) *) +(* with `is_update=true` unconditionally sets state back to Pending via *) +(* `INSERT ... ON CONFLICT(message_id) DO UPDATE SET state=excluded.state` *) +(* - no WHERE clause is possible on an upsert, so a message the system *) +(* just administratively marked Cancelled (because its graph is already *) +(* finalized) can be silently resurrected to Pending and re-dispatched *) +(* the next time a handler in the swarm-message task calls a retry/defer *) +(* on it, unrelated to the cancellation. *) +(***************************************************************************) +Statuses == {"Pending", "Cancelled"} + +\* Cancelled is an administrative "this message is moot, stop touching it" +\* marker tied to its graph reaching a closed status - it must stay final. +TerminalStatuses == {"Cancelled"} + +VARIABLE status +vars == <> + +TypeOK == status \in Statuses + +Init == status = "Pending" + +-------------------------------------------------------------------------- +\* event_watch_task.rs's handle_withdraw_paths_events / handle_withdraw_ +\* disproved_events, via update_messages_state_by_business_id - a genuine +\* CAS, correctly guarded in the real code. +BulkCancelOnGraphClose == + /\ status = "Pending" + /\ status' = "Cancelled" + +\* push_local_unhandled_messages -> utils::upsert_message(is_update=true) +\* -> store upsert_message's `ON CONFLICT DO UPDATE SET state=excluded.state` +\* - confirmed NO guard of any kind. Fires from ~30 call sites in +\* node/src/handle.rs whenever a message handler needs to defer/retry, +\* with no awareness of whether the message was since cancelled. +ResurrectPendingUnconditional == status' = "Pending" + +Next == + \/ BulkCancelOnGraphClose + \/ ResurrectPendingUnconditional + +Spec == Init /\ [][Next]_vars +FairSpec == Spec /\ WF_vars(Next) + +\* Proposed fix design (not applied to code): guard the resurrect-to-Pending +\* write the same way - only apply it if the message isn't already in a +\* terminal status, folded into the UPDATE/upsert's WHERE clause. +NextFixed == + \/ BulkCancelOnGraphClose + \/ (status \notin TerminalStatuses /\ ResurrectPendingUnconditional) + +SpecFixed == Init /\ [][NextFixed]_vars +FairSpecFixed == SpecFixed /\ WF_vars(NextFixed) + +-------------------------------------------------------------------------- +\* Safety property: once a message is administratively Cancelled, it must +\* never be resurrected and re-dispatched. +TerminalStatusesAreAbsorbing == [][(status \in TerminalStatuses => status' = status)]_status + +==== diff --git a/node/tla/MessageStateRaceFixed.cfg b/node/tla/MessageStateRaceFixed.cfg new file mode 100644 index 000000000..ffdbda46c --- /dev/null +++ b/node/tla/MessageStateRaceFixed.cfg @@ -0,0 +1,5 @@ +\* Proposed fix design (verified, NOT applied to code) - expected to pass. +SPECIFICATION FairSpecFixed +CHECK_DEADLOCK FALSE +INVARIANT TypeOK +PROPERTY TerminalStatusesAreAbsorbing diff --git a/node/tla/MultiActorRace.cfg b/node/tla/MultiActorRace.cfg new file mode 100644 index 000000000..ae064dfc8 --- /dev/null +++ b/node/tla/MultiActorRace.cfg @@ -0,0 +1,6 @@ +SPECIFICATION Spec +CHECK_DEADLOCK FALSE +INVARIANT WatchtowerAckWindowAlwaysPositive +INVARIANT NackAlwaysBeatsTake2ViaF +INVARIANT CommitTimeoutAlwaysBeatsTake2ViaF +INVARIANT DisproveAlwaysBeatsTake2ViaD diff --git a/node/tla/MultiActorRace.tla b/node/tla/MultiActorRace.tla new file mode 100644 index 000000000..91b76e3ad --- /dev/null +++ b/node/tla/MultiActorRace.tla @@ -0,0 +1,114 @@ +---- MODULE MultiActorRace ---- +(***************************************************************************) +(* Take2DisproveRace.tla abstracted "the watchtower" and "the verifier" as *) +(* single actors. The real protocol has N independent watchtowers and M *) +(* independent verifiers (node/tla/Take2DisproveRace.tla's header / *) +(* node/README.md document the single-actor version). This module checks *) +(* whether N=2/M=2 introduces anything the single-actor abstraction *) +(* couldn't see - the concern being: with multiple independent actors *) +(* timing their own actions, could the "any ONE honest actor suffices" *) +(* property (verified structurally true by a dedicated research pass - *) +(* see below) actually fail on the TIMING axis even though it holds *) +(* logically? *) +(* *) +(* Ground truth (verified by reading goat/src/{connectors,transactions}/*.rs *) +(* directly, not assumed): *) +(* - WATCHTOWERS: all N watchtowers' WatchtowerChallengeConnector[i]/ *) +(* AckConnector[i] pairs are outputs of the SAME, single *) +(* WatchtowerChallengeInitTransaction. So every watchtower's challenge *) +(* window AND the operator's ack window for that slot are measured *) +(* from the SAME shared height, regardless of which watchtower acts or *) +(* when. There is no genuine multi-clock complexity on this side - N *) +(* watchtowers share ONE clock. If watchtower i is un-acked, its Nack *) +(* spends the shared ConnectorF leaf1, permanently blocking Take2's *) +(* leaf0 (first-confirmed-wins UTXO semantics) - true 1-of-N by *) +(* construction, confirmed via detect_watchtower_flow_disprove *) +(* (node/src/utils.rs:1299-1326), which returns Disprove on the FIRST *) +(* nack found, no counting/quorum. *) +(* - VERIFIERS: structurally identical EXCEPT each verifier's clock *) +(* genuinely is independent - VerifierAssertTransaction[i] confirms at *) +(* verifier i's own pace (whenever THEY finish detecting fraud), not a *) +(* shared height. This is the one place N/M actually could matter. *) +(***************************************************************************) +EXTENDS Integers, ShippedTimelocks + +VARIABLES + net, + wtChallengeHeight1, wtChallengeHeight2, \* watchtower i challenges at this height within [0, WatchtowerChallenge[net]], or NEVER + verifierDelta1, verifierDelta2 \* verifier i's own independent assert-confirmation gap after OperatorAssert (assertHeight = 0 WLOG), or NEVER + +vars == <> + +NEVER == -1 + +Init == + /\ net \in Networks + /\ wtChallengeHeight1 \in ({NEVER} \cup 0..WatchtowerChallenge[net]) + /\ wtChallengeHeight2 \in ({NEVER} \cup 0..WatchtowerChallenge[net]) + /\ verifierDelta1 \in ({NEVER} \cup 0..MinReactionBlocks[net]) + /\ verifierDelta2 \in ({NEVER} \cup 0..MinReactionBlocks[net]) + +Next == FALSE \* exhaustive check over Init, not a process model - see Take2DisproveRace.tla +Spec == Init /\ [][Next]_vars + +-------------------------------------------------------------------------- +(* Watchtower side: both slots share ONE clock (WatchtowerChallengeInit's *) +(* confirmation, height 0 here), independent of when each watchtower acts. *) + +Challenged(h) == h # NEVER +NackDeadline == OperatorAck[net] \* shared, same for every watchtower slot +Take2ReadyHeightViaF == ConnectorF[net] +OperatorAckWindow(challengeHeight) == NackDeadline - challengeHeight + +\* For every possible timing of both watchtowers - including both waiting +\* until the literal last block of their window - an operator notified the +\* instant a challenge lands still has a strictly positive number of blocks +\* to construct and confirm their Ack before the shared deadline. Checked +\* per watchtower since neither's window depends on the other's timing (no +\* shared clock to contend over on THIS axis). +WatchtowerAckWindowAlwaysPositive == + /\ (Challenged(wtChallengeHeight1) => OperatorAckWindow(wtChallengeHeight1) > 0) + /\ (Challenged(wtChallengeHeight2) => OperatorAckWindow(wtChallengeHeight2) > 0) + +\* If a watchtower's Nack ever becomes available (operator failed to ack in +\* time - the worst case for the operator), it is always available strictly +\* before Take2 could fire via connector_f, independent of the OTHER +\* watchtower's behavior or timing. +NackAlwaysBeatsTake2ViaF == + /\ (Challenged(wtChallengeHeight1) => NackDeadline < Take2ReadyHeightViaF) + /\ (Challenged(wtChallengeHeight2) => NackDeadline < Take2ReadyHeightViaF) + +-------------------------------------------------------------------------- +(* ConnectorF leaf 1 (the "committee blocks Take2" leaf) has TWO *) +(* alternative spenders, not one - confirmed by reading *) +(* goat/src/transactions/watchtower_challenge.rs directly: *) +(* OperatorChallengeNackTransaction (leaf1, checked above via *) +(* NackAlwaysBeatsTake2ViaF) AND OperatorCommitTimeoutTransaction (also *) +(* leaf1, watchtower_challenge.rs:714-747, jointly spending ConnectorE *) +(* leaf1 as its other input). The Rust side already enforces *) +(* operator_commit < connector_f (timelocks.rs's *) +(* ensure_lt("operator_commit", ..., "connector_f", ...)) but - unlike the *) +(* operator_ack/Nack case - that comparison was never independently *) +(* confirmed here using real shipped values. Same shared clock root as *) +(* every other property in this module (WatchtowerChallengeInit's *) +(* confirmation - operator_commit's clock starts there via ConnectorE's *) +(* own construction, symmetric to operator_ack's), and not a multi-actor *) +(* quantity (one operator, one commit-or-not decision per graph, not per *) +(* watchtower), so no new VARIABLE is needed - just the real numbers. *) +CommitTimeoutDeadline == OperatorCommit[net] +CommitTimeoutAlwaysBeatsTake2ViaF == CommitTimeoutDeadline < Take2ReadyHeightViaF + +-------------------------------------------------------------------------- +(* Verifier side: genuinely independent clocks - each verifier's own delta. *) + +DisproveDeadline(delta) == delta + ProverConnector[net] +Take2ReadyHeightViaD == ConnectorD[net] + +\* Symmetric to the watchtower property: EACH verifier's disprove deadline +\* (rooted at THEIR OWN assert height) must beat Take2's connector_d +\* deadline, regardless of the other verifier's independent timing. +DisproveAlwaysBeatsTake2ViaD == + /\ (verifierDelta1 # NEVER => DisproveDeadline(verifierDelta1) < Take2ReadyHeightViaD) + /\ (verifierDelta2 # NEVER => DisproveDeadline(verifierDelta2) < Take2ReadyHeightViaD) + +==== diff --git a/node/tla/ShippedTimelocks.tla b/node/tla/ShippedTimelocks.tla new file mode 100644 index 000000000..563bad5f2 --- /dev/null +++ b/node/tla/ShippedTimelocks.tla @@ -0,0 +1,33 @@ +---- MODULE ShippedTimelocks ---- +(***************************************************************************) +(* Real per-network timelock values, transcribed from *) +(* crates/bitvm-gc/src/timelocks.rs, shared by every margin-arithmetic *) +(* spec in this directory (Take2DisproveRace.tla, MultiActorRace.tla, *) +(* Take1ChallengeRace.tla) - single source of truth instead of each spec *) +(* re-transcribing its own copy. *) +(* *) +(* As of commit 991faaa ("Dev fix #418"), Finding 4's fix has actually been *) +(* applied to crates/bitvm-gc/src/timelocks.rs - these are now the REAL *) +(* shipped values (re-verified via TLC against these exact numbers, not *) +(* just the originally-proposed 35/testnet4 figure this file used to hold *) +(* before the real fix landed with a wider margin than proposed). *) +(* connector_a is NOT here - Take1ChallengeRace.tla defines its own *) +(* ConnectorA/ConnectorAFixed locally, since that value pair is the *) +(* historical subject under test in that spec, not a settled shared *) +(* constant. *) +(***************************************************************************) + +Networks == {"Bitcoin", "Testnet4", "Signet", "Regtest"} + +ProverConnector == [Bitcoin |-> 144, Testnet4 |-> 20, Signet |-> 6, Regtest |-> 1] +ConnectorD == [Bitcoin |-> 432, Testnet4 |-> 40, Signet |-> 18, Regtest |-> 3] +WatchtowerChallenge == [Bitcoin |-> 144, Testnet4 |-> 20, Signet |-> 6, Regtest |-> 1] +OperatorAck == [Bitcoin |-> 288, Testnet4 |-> 32, Signet |-> 12, Regtest |-> 2] +OperatorCommit == [Bitcoin |-> 432, Testnet4 |-> 40, Signet |-> 18, Regtest |-> 3] +ConnectorF == [Bitcoin |-> 576, Testnet4 |-> 52, Signet |-> 24, Regtest |-> 4] + +\* Policy floor, not itself a timelocks.rs field: the assumed real-world +\* reaction-time bound (~1 hour) used across every margin-race spec. +MinReactionBlocks == [Bitcoin |-> 6, Testnet4 |-> 12, Signet |-> 1, Regtest |-> 1] + +==== diff --git a/node/tla/Take1ChallengeRace.cfg b/node/tla/Take1ChallengeRace.cfg new file mode 100644 index 000000000..6873f166b --- /dev/null +++ b/node/tla/Take1ChallengeRace.cfg @@ -0,0 +1,6 @@ +\* Models the CURRENT, shipped connector_a values (no margin check exists +\* anywhere in the codebase for this field) - expected to FAIL on at least +\* one network. This is a live gap, not a historical artifact. +SPECIFICATION Spec +CHECK_DEADLOCK FALSE +INVARIANT ChallengeHasSufficientReactionMargin diff --git a/node/tla/Take1ChallengeRace.tla b/node/tla/Take1ChallengeRace.tla new file mode 100644 index 000000000..9c4abd297 --- /dev/null +++ b/node/tla/Take1ChallengeRace.tla @@ -0,0 +1,75 @@ +---- MODULE Take1ChallengeRace ---- +(***************************************************************************) +(* Formal model of a new bottleneck connector found while extending the *) +(* transaction-graph audit beyond ConnectorD (see Take2DisproveRace.tla *) +(* and audit/TLAPlus-*.md's "Cross-check"/connector-sweep follow-up). *) +(* *) +(* Ground truth, read directly from the external goat crate *) +(* (checkout e369b2a, goat/src/connectors/connector_a.rs, *) +(* goat/src/transactions/{kickoff,take1,challenge}.rs): *) +(* - KickoffTransaction creates ConnectorA's output (kickoff.rs:67-71, *) +(* output_0 = connector_a.generate_taproot_address()) - this is the *) +(* single shared clock root for both leaves below. *) +(* - ConnectorA leaf 0: operator key + CSV(connector_a) (connector_a.rs: *) +(* 37-46) - spent by Take1Transaction (take1.rs:73-74, input_1_leaf=0),*) +(* the operator's uncontested fast-exit path. *) +(* - ConnectorA leaf 1: n-of-n committee key, NO CSV at all *) +(* (connector_a.rs:48-54) - spent by ChallengeTransaction *) +(* (challenge.rs:56-57, input_0_leaf=1) using SinglePlusAnyoneCanPay *) +(* (challenge.rs:93), so any third-party challenger can add their own *) +(* fee input and force this tx through the instant it's noticed. *) +(* Same ConnectorA output, different leaves - ordinary UTXO semantics mean *) +(* whichever of Take1/Challenge confirms first wins it permanently. Unlike *) +(* ConnectorD (Take2 vs Disprove), this is a single-clock race: Challenge *) +(* is spendable immediately once Kickoff confirms, so the entire margin a *) +(* challenger has to detect fraud and get Challenge confirmed IS *) +(* connector_a's own CSV value, in blocks, from Kickoff's confirmation. *) +(* *) +(* crates/bitvm-gc/src/timelocks.rs's validate_timelock_config (69-101) *) +(* checks connector_a is merely nonzero (73-81) but - unlike every other *) +(* named timelock field - never puts it in any ensure_lt/ensure_lte *) +(* comparison (91-99). There is no enforcement anywhere in the codebase *) +(* that connector_a leaves a challenger enough real-world reaction time, *) +(* on ANY network, not just the boundary case this module happens to find. *) +(* MinReactionBlocks is the same ~1-hour-of-reaction-time policy floor *) +(* Take2DisproveRace.tla already established and validated for the *) +(* analogous ConnectorD/ProverConnector race - reused here, not re-derived,*) +(* since it's the same real-world assumption (off-chain fraud detection + *) +(* tx construction + broadcast + confirmation lag) applied to a different *) +(* connector. *) +(* *) +(* STATUS UPDATE (commit 991faaa, "Dev fix #418"): this finding has since *) +(* been fixed - validate_timelock_config now has an ensure_gt("connector_a",*) +(* ..., "min_reaction_blocks", ...) check, and NODE_REGTEST_TIMELOCK_CONFIG*) +(* .connector_a was bumped 1 -> 2, exactly matching ConnectorAFixed below. *) +(* ConnectorA (unchanged from this file's original discovery run) is kept *) +(* as a historical record of the exact pre-fix shipped value/counterexample*) +(* - it deliberately does NOT track crates/bitvm-gc/src/timelocks.rs's *) +(* current numbers the way ShippedTimelocks.tla's tables do. *) +(***************************************************************************) +EXTENDS Integers, ShippedTimelocks + +\* Historical: the shipped value at the time this bug was found (pre-fix). +\* Kept as-is rather than updated to track current timelocks.rs - see the +\* STATUS UPDATE note above. +ConnectorA == [Bitcoin |-> 144, Testnet4 |-> 16, Signet |-> 6, Regtest |-> 1] + +\* The fix design, since actually applied verbatim in commit 991faaa. +ConnectorAFixed == [Bitcoin |-> 144, Testnet4 |-> 16, Signet |-> 6, Regtest |-> 2] + +VARIABLE net +vars == <> + +Init == net \in Networks +Next == FALSE \* no transitions - exhaustive check over Init, same idiom as Take2DisproveRace.tla +Spec == Init /\ [][Next]_vars + +\* The property connector_a's margin exists to guarantee but nothing in +\* the code currently checks: a challenger has strictly more than the +\* assumed real-world reaction-time floor, from Kickoff's confirmation, to +\* detect fraud and get a Challenge transaction confirmed before the +\* operator's Take1 CSV (rooted at that same confirmation) elapses. +ChallengeHasSufficientReactionMargin == ConnectorA[net] > MinReactionBlocks[net] +ChallengeHasSufficientReactionMarginFixed == ConnectorAFixed[net] > MinReactionBlocks[net] + +==== diff --git a/node/tla/Take1ChallengeRaceFixed.cfg b/node/tla/Take1ChallengeRaceFixed.cfg new file mode 100644 index 000000000..431c23396 --- /dev/null +++ b/node/tla/Take1ChallengeRaceFixed.cfg @@ -0,0 +1,4 @@ +\* Proposed fix design (verified, NOT applied to code) - expected to pass. +SPECIFICATION Spec +CHECK_DEADLOCK FALSE +INVARIANT ChallengeHasSufficientReactionMarginFixed diff --git a/node/tla/Take2DisproveRace.cfg b/node/tla/Take2DisproveRace.cfg new file mode 100644 index 000000000..78651b872 --- /dev/null +++ b/node/tla/Take2DisproveRace.cfg @@ -0,0 +1,4 @@ +SPECIFICATION Spec +CHECK_DEADLOCK FALSE +INVARIANT DisproveWinsConnectorDRace +INVARIANT ConnectorFNeverShortcutsConnectorD diff --git a/node/tla/Take2DisproveRace.tla b/node/tla/Take2DisproveRace.tla new file mode 100644 index 000000000..1c91cbd47 --- /dev/null +++ b/node/tla/Take2DisproveRace.tla @@ -0,0 +1,83 @@ +---- MODULE Take2DisproveRace ---- +(***************************************************************************) +(* Formal model of the "gap 3" race flagged when auditing timelock *) +(* reasonableness (see crates/bitvm-gc/src/timelocks.rs): whether Take2's *) +(* two-clock readiness condition can be exploited to preempt the *) +(* committee's cooperative Disprove fallback on the same UTXO. *) +(* *) +(* Ground truth, read directly from the external goat crate *) +(* (checkout e369b2a, goat/src/transactions/{take2,assert}.rs): *) +(* - Take2Transaction spends ConnectorD leaf 0 (operator + CSV(connector_d), *) +(* clock starts at OperatorAssert's confirmation) AND ConnectorF leaf 0 *) +(* (operator + CSV(connector_f), clock starts at WatchtowerChallengeInit's *) +(* confirmation) - both required simultaneously (take2.rs:56-90). *) +(* - DisproveTransaction spends ConnectorD leaf 1 (n-of-n, immediate, no *) +(* CSV) AND ProverConnector leaf 1 (n-of-n + CSV(prover_connector), *) +(* clock starts at VerifierAssert's confirmation) (assert.rs:317-357). *) +(* Same ConnectorD output, different leaves - first tx confirmed wins it *) +(* permanently (ordinary Bitcoin UTXO semantics, no special ordering rule *) +(* needed - modeled here as first-deadline-reached-wins). *) +(* *) +(* NOTE (found by the later connector-sweep, audit/TLAPlus-*.md): ConnectorD *) +(* actually has a THIRD leaf (connector_d.rs:20-22,80-88) spent by a free *) +(* function pubin_disprove() (assert.rs:571-599), the node's tracked *) +(* DisproveTxType::PubinDisprove outcome. This module does not model it - *) +(* deliberately, not by oversight: leaf 2's script has NO CSV at all, so it *) +(* can only ever be at least as fast as the already-proven-safe Disprove *) +(* path modeled below, never slower - it cannot introduce a new exploitable *) +(* margin. Noted here so this header no longer implies leaf 2 doesn't exist.*) +(* *) +(* VerifierAssert spends OperatorAssert's output, so it can only confirm *) +(* strictly after OperatorAssert - `delta` below is that real-world gap. *) +(* It is NOT a protocol constant; nothing on-chain bounds it. The fix in *) +(* timelocks.rs assumes it is bounded by min_reaction_blocks(network) (the *) +(* same ~1-hour policy floor used elsewhere). This module checks that *) +(* assumption is sufficient, using the actual shipped per-network values, *) +(* and separately confirms (rather than just hand-argues) that Take2's *) +(* second clock (connector_f, rooted at WatchtowerChallengeInit) can never *) +(* be used to shortcut this specific race, regardless of when the operator *) +(* chooses to broadcast WatchtowerChallengeInit. *) +(***************************************************************************) +EXTENDS Integers, ShippedTimelocks + +Max(a, b) == IF a >= b THEN a ELSE b + +VARIABLES net, delta, wci + +vars == <> + +\* assertHeight is fixed at 0 WLOG - every other height is expressed relative +\* to it. delta ranges up to (and including) the assumed reaction-time bound +\* for each network - this is exactly the guarantee the timelocks.rs fix is +\* supposed to provide, checked here rather than re-derived by hand. wci +\* ranges over a wide symmetric window since the operator fully controls +\* when WatchtowerChallengeInit is broadcast (it is their own signed, +\* immediately-spendable transaction, independent of OperatorAssert - see +\* watchtower_challenge.rs:178-256, both connector_b and connector_c are +\* plain, unrelated Kickoff outputs with no relative ordering between them). +Init == + /\ net \in Networks + /\ delta \in 0..MinReactionBlocks[net] + /\ wci \in -300..300 + +vars_unchanged == UNCHANGED vars +Next == FALSE \* no transitions - this is an exhaustive check over Init, not a process model +Spec == Init /\ [][Next]_vars + +DisproveDeadline == delta + ProverConnector[net] +Take2ConnectorDDeadline == ConnectorD[net] +Take2ConnectorFDeadline == wci + ConnectorF[net] +Take2Deadline == Max(Take2ConnectorDDeadline, Take2ConnectorFDeadline) + +\* The property the margin fix in validate_timelock_config exists to +\* guarantee: within the assumed real-world confirmation-gap bound, Disprove +\* always has a strictly earlier deadline than Take2's ConnectorD leaf, for +\* every choice the operator could make for wci. +DisproveWinsConnectorDRace == DisproveDeadline < Take2ConnectorDDeadline + +\* Independent confirmation (not just a hand-argument) that ConnectorF can +\* only ever add delay to Take2, never let the operator preempt the +\* ConnectorD-specific race early via a clever choice of wci. +ConnectorFNeverShortcutsConnectorD == Take2Deadline >= Take2ConnectorDDeadline + +==== diff --git a/proof-builder-rpc/src/api/proof_handler.rs b/proof-builder-rpc/src/api/proof_handler.rs index e86b1f4e8..d55eb949a 100644 --- a/proof-builder-rpc/src/api/proof_handler.rs +++ b/proof-builder-rpc/src/api/proof_handler.rs @@ -67,7 +67,7 @@ pub(super) async fn get_chain_proof_task_desc( block_end: proof.block_end, proof_type: payload.proof_type.to_string(), state: ProofState::from_i64(proof.proof_state) - .unwrap_or_else(|| ProofState::New) + .unwrap_or(ProofState::New) .to_string(), proving_cycles: proof.cycles, proving_time: proof.proving_time, @@ -110,7 +110,7 @@ pub(super) async fn get_operator_proof_task_desc( block_end: operator_proof.execution_layer_block_number + 1, proof_type: "Operator".to_string(), state: ProofState::from_i64(operator_proof.proof_state) - .unwrap_or_else(|| ProofState::New) + .unwrap_or(ProofState::New) .to_string(), proving_cycles: operator_proof.cycles, proving_time: operator_proof.proving_time, diff --git a/proof-builder-rpc/src/config.rs b/proof-builder-rpc/src/config.rs index 7c137c5b8..95eb73861 100644 --- a/proof-builder-rpc/src/config.rs +++ b/proof-builder-rpc/src/config.rs @@ -18,7 +18,7 @@ impl ProofBuilderConfig { fn load(url: &str) -> anyhow::Result { let content = - std::fs::read_to_string(&url).context(format!("Failed to read config file: {url}"))?; + std::fs::read_to_string(url).context(format!("Failed to read config file: {url}"))?; Ok(toml::from_str(&content)?) } diff --git a/proof-builder-rpc/src/task/commit_chain_proof.rs b/proof-builder-rpc/src/task/commit_chain_proof.rs index 942da4811..5cb4be1a1 100644 --- a/proof-builder-rpc/src/task/commit_chain_proof.rs +++ b/proof-builder-rpc/src/task/commit_chain_proof.rs @@ -86,7 +86,7 @@ pub(crate) fn spawn_commit_chain_proof_task( let zkm_version = proof.zkm_version.clone(); let (public_value_hex, proof_size) = builder.save_proof(&ctx, &input, cycles, proof)?; - create_commit_chain_proof(&local_db, block_start, 0xFFffFFff as i64 - block_start, args.output_proof.clone(), public_value_hex, proof_size as i64, cycles, CommitChainProofBuilder::name(), proving_duration as i64, proving_time as i64, store::ProofState::Proven,zkm_version).await?; + create_commit_chain_proof(&local_db, block_start, 0xffffffff_i64 - block_start, args.output_proof.clone(), public_value_hex, proof_size as i64, cycles, CommitChainProofBuilder::name(), proving_duration as i64, proving_time as i64, store::ProofState::Proven,zkm_version).await?; args = ProofBuilderConfig::run_next(args, CommitChainProofBuilder::name())?; } _ = cancellation_token.cancelled() => { diff --git a/proof-builder-rpc/src/task/mod.rs b/proof-builder-rpc/src/task/mod.rs index a6f795f34..4cc88c0b5 100644 --- a/proof-builder-rpc/src/task/mod.rs +++ b/proof-builder-rpc/src/task/mod.rs @@ -1,3 +1,5 @@ +#![allow(clippy::too_many_arguments)] // Task persistence APIs mirror the proof metadata schema. + mod commit_chain_proof; mod header_chain_proof; mod operator_proof; @@ -273,14 +275,14 @@ async fn read_watchtower_challenge_details<'a>( task.graph_id ); } - if let Some(first) = challenge_init_txids.first() { - if !challenge_init_txids.iter().all(|x| first == x) { - anyhow::bail!( - "Inconsistent watchtower challenge info from instance {} and graph_id {}", - task.instance_id, - task.graph_id - ); - } + if let Some(first) = challenge_init_txids.first() + && !challenge_init_txids.iter().all(|x| first == x) + { + anyhow::bail!( + "Inconsistent watchtower challenge info from instance {} and graph_id {}", + task.instance_id, + task.graph_id + ); } let challenge_txids: Vec> = watchtower_info .iter() @@ -529,9 +531,9 @@ pub(crate) async fn create_long_running_task( zkm_version: String, ) -> anyhow::Result { let mut storage_processor = local_db.acquire().await?; - Ok(storage_processor + storage_processor .create_long_running_task_proof(&LongRunningTaskProof { - block_start: start as i64, + block_start: start, block_end: start + batch_size, chain_name, path_to_proof: Some(path_to_proof), @@ -546,7 +548,7 @@ pub(crate) async fn create_long_running_task( created_at: current_time_secs(), updated_at: current_time_secs(), }) - .await?) + .await } /// This is a special function to add a new record while updating the previous record's block_end. @@ -567,11 +569,11 @@ pub(crate) async fn create_commit_chain_proof( let mut storage_processor = local_db.start_transaction().await?; // we use start directly since it's block_end is initialized by u64::MAX let previous_proof = storage_processor - .find_long_running_task_proof_including_block_number(start as i64, chain_name.clone()) + .find_long_running_task_proof_including_block_number(start, chain_name.clone()) .await?; tracing::info!("previous_proof: {previous_proof:?}"); if let Some(previous_proof) = previous_proof { - let prev_batch_size = start as i64 - previous_proof.block_start; + let prev_batch_size = start - previous_proof.block_start; tracing::info!("update previous proof from {start} batch_size: {prev_batch_size}"); storage_processor .update_long_running_task_proof_state( @@ -584,7 +586,7 @@ pub(crate) async fn create_commit_chain_proof( } let affected = storage_processor .create_long_running_task_proof(&LongRunningTaskProof { - block_start: start as i64, + block_start: start, block_end: start + batch_size, chain_name, path_to_proof: Some(path_to_proof), @@ -619,7 +621,7 @@ pub(crate) async fn update_long_running_task( ) -> anyhow::Result { let mut storage_processor = local_db.acquire().await?; let task = storage_processor - .find_long_running_task_proof_including_block_number(start_index as i64, chain_name.clone()) + .find_long_running_task_proof_including_block_number(start_index, chain_name.clone()) .await?; if task.is_none() { anyhow::bail!( @@ -627,7 +629,7 @@ pub(crate) async fn update_long_running_task( ); } let total_time_to_proof = (current_time_secs() - task.unwrap().created_at) * 1000; - Ok(storage_processor + storage_processor .update_long_running_task_proof_success( start_index, &chain_name, @@ -641,13 +643,14 @@ pub(crate) async fn update_long_running_task( proving_time, &zkm_version, ) - .await?) + .await } /// table schema: (index, instance_id, graph_id, public_key, challenge_txid, challenge_init_txid, path_to_proof, cycles, state, update_time) /// * index: incremental id /// * state: 0-new, 1-doing, 2-done, 3-failed -/// Invocated by API +/// +/// Invoked by API. pub(crate) async fn add_watchtower_task( local_db: &LocalDB, instance_id: Uuid, @@ -657,7 +660,7 @@ pub(crate) async fn add_watchtower_task( execution_layer_block_number: i64, ) -> anyhow::Result { let mut storage_processor = local_db.acquire().await?; - Ok(storage_processor + storage_processor .create_watchtower_proof(&WatchtowerProof { id: 1, instance_id, @@ -671,7 +674,7 @@ pub(crate) async fn add_watchtower_task( included: true, ..Default::default() }) - .await?) + .await } pub(crate) async fn find_watchtower_task( @@ -719,7 +722,7 @@ pub(crate) async fn update_watchtower_task( zkm_version: String, ) -> anyhow::Result { let mut storage_processor = local_db.acquire().await?; - Ok(storage_processor + storage_processor .update_watchtower_proof( index, path_to_proof, @@ -731,7 +734,7 @@ pub(crate) async fn update_watchtower_task( proving_time, &zkm_version, ) - .await?) + .await } /// table schema: (index, instance_id, graph_id, execution_layer_block_number, path_to_proof, cycles, state, update_time) @@ -819,7 +822,7 @@ pub(crate) async fn add_operator_task( id: 1, instance_id, graph_id, - execution_layer_block_number: execution_layer_block_number as i64, + execution_layer_block_number, proof_state: ProofState::New.to_i64(), created_at: current_time_secs(), updated_at: current_time_secs(), @@ -905,7 +908,7 @@ pub(crate) async fn update_operator_task( zkm_version: String, ) -> anyhow::Result { let mut storage_processor = local_db.acquire().await?; - Ok(storage_processor + storage_processor .update_operator_proof( index, path_to_proof, @@ -917,7 +920,7 @@ pub(crate) async fn update_operator_task( proving_time, &zkm_version, ) - .await?) + .await } #[inline(always)] diff --git a/proof-builder-rpc/src/task/state_chain_proof.rs b/proof-builder-rpc/src/task/state_chain_proof.rs index 5f7302bdb..8912c6bb0 100644 --- a/proof-builder-rpc/src/task/state_chain_proof.rs +++ b/proof-builder-rpc/src/task/state_chain_proof.rs @@ -82,8 +82,8 @@ async fn spawn_state_chain_ctx_builder( let snap_path = std::path::Path::new(&args.input_proof).parent().unwrap().to_str().unwrap(); tracing::info!("fetch snap_path: {snap_path:?}"); - std::fs::write(&format!("{}/{}.args", snap_path, args.start), serde_json::to_string(&args)?)?; - std::fs::write(&format!("{}/{}.ctx", snap_path, args.start), serde_json::to_string(&ctx)?)?; + std::fs::write(format!("{}/{}.args", snap_path, args.start), serde_json::to_string(&args)?)?; + std::fs::write(format!("{}/{}.ctx", snap_path, args.start), serde_json::to_string(&ctx)?)?; let affected = match create_long_running_task( &local_db, @@ -133,7 +133,7 @@ async fn spawn_state_chain_prover( tracing::info!("prover: start proving, block_start: {start_index}, batch_size: {batch_size}"); let snap_path = std::path::Path::new(&input_proof).parent().unwrap().to_str().unwrap(); - let args: state_chain_proof::Args = match std::fs::read(&format!("{}/{}.args", snap_path, start_index)) { + let args: state_chain_proof::Args = match std::fs::read(format!("{}/{}.args", snap_path, start_index)) { Ok(x) => match serde_json::from_slice(&x) { Ok(args) => args, Err(e) => { @@ -148,7 +148,7 @@ async fn spawn_state_chain_prover( } }; - let ctx: ProofRequest = match &std::fs::read(&format!("{}/{}.ctx", snap_path, start_index)) { + let ctx: ProofRequest = match std::fs::read(format!("{}/{}.ctx", snap_path, start_index)) { Ok(x) => match serde_json::from_slice(&x) { Ok(ctx) => ctx, Err(e) => { @@ -262,18 +262,16 @@ pub(crate) fn spawn_state_chain_proof_task( ) .await?; - if let Some(task_failed) = cur_task_failed { - if let Some(ref task) = cur_task { - if task.block_start == task_failed.block_start - && task.block_end == task_failed.block_end - { - cur_task = Some(task_failed); - } else { - if task.block_start < args.start as i64 { - // load from config - cur_task = None; - } - } + if let Some(task_failed) = cur_task_failed + && let Some(ref task) = cur_task + { + if task.block_start == task_failed.block_start + && task.block_end == task_failed.block_end + { + cur_task = Some(task_failed); + } else if task.block_start < args.start as i64 { + // load from config + cur_task = None; } };