diff --git a/.github/workflows/go_build.yml b/.github/workflows/go_build.yml index 7b7d0115..837e0e09 100644 --- a/.github/workflows/go_build.yml +++ b/.github/workflows/go_build.yml @@ -39,6 +39,7 @@ jobs: run: >- go test -race -p 2 -count=1 ./pkg/alpenglow ./pkg/consensus ./pkg/replay ./pkg/rewards + ./pkg/rpcserver ./pkg/epochrewards ./pkg/turbine ./pkg/sigverify ./pkg/blockprod/... ./cmd/mithril/node ./cmd/mithril/configcmd diff --git a/README.md b/README.md index dcbb27af..f37405e9 100644 --- a/README.md +++ b/README.md @@ -222,8 +222,20 @@ curl http://YOUR_MITHRIL_IP:8899 -X POST -H "Content-Type: application/json" -d - `getBankHash` - Get bankhash for a slot (Mithril extension, not standard Solana RPC) - `getBlockHeight` - Get current block height - `getEpochInfo` - Get current epoch info +- `getInflationReward` - Get retained inflation rewards for an epoch - `getLatestBlockhash` - Get recent blockhash +On Alpenglow, `getInflationReward` can publish verified canonical rewards before +the account checkpoint reaches disk. `confirmed` and `finalized` share that view; +`processed` is not supported. The default epoch and `minContextSlot` check use the +same reward-read slot. Other RPC methods may still use an older durable view. +Recovery withdraws the in-memory view, and a restart restores only +durable history. Retention follows durable checkpoints. Advisory or disabled +execution verification keeps reward reads on the durable view. +Credited rewards are readable while other staking partitions are still pending. +Requests for pending partitions return `-32017` with verified distribution +progress; pruned history returns `-32001`, and unknown history returns `-32004`. + We're actively expanding RPC method coverage. Upcoming methods include transaction simulation, send transaction, and get leader schedule. ### Current Limitations diff --git a/cmd/mithril/node/epoch_reward_recovery_test.go b/cmd/mithril/node/epoch_reward_recovery_test.go new file mode 100644 index 00000000..8a73fc73 --- /dev/null +++ b/cmd/mithril/node/epoch_reward_recovery_test.go @@ -0,0 +1,139 @@ +package node + +import ( + "bytes" + "encoding/json" + "io" + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/Overclock-Validator/mithril/pkg/accountsdb" + "github.com/Overclock-Validator/mithril/pkg/block" + "github.com/Overclock-Validator/mithril/pkg/rpcserver" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/Overclock-Validator/mithril/pkg/state" + "github.com/gagliardetto/solana-go" + "github.com/gagliardetto/solana-go/rpc" + "github.com/stretchr/testify/require" +) + +func TestEpochRewardHTTPAfterDurableRewind(t *testing.T) { + for _, name := range []string{"rewind", "prune failure", "accounts cleanup failure", "accounts precommit failure"} { + t.Run(name, func(t *testing.T) { + failPrune := name == "prune failure" + dir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(dir, "accounts"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "largest_file_id"), make([]byte, 8), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "bootstrap_high_file_id"), make([]byte, 8), 0o644)) + db, err := accountsdb.OpenDb(dir) + require.NoError(t, err) + db.RootedDurable = true + db.InitCaches() + t.Cleanup(db.CloseDb) + address := solana.PublicKey{91} + var tip *state.ResumeContext + for _, slot := range []uint64{2, 4} { + tip = testDurableStatusContext(t, dir, slot) + encoded, err := json.Marshal(tip) + require.NoError(t, err) + _, err = db.CommitBatch([]accounts.SlotDelta{{Slot: slot, Delta: []*accounts.Account{{Key: address, Lamports: slot}}}}, slot, nil, encoded) + require.NoError(t, err) + } + server := rpcserver.NewRpcServer(db, 0, &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 3}, solana.Hash{1}) + require.NoError(t, server.EnableEpochRewards(filepath.Join(dir, "rpc-epoch-rewards"), 100, 0)) + record := &block.Block{Slot: 4, Epoch: 1, Rewards: []rpc.BlockReward{{Pubkey: address, Lamports: 2, PostBalance: 4, RewardType: rpc.RewardTypeVoting}}} + require.NoError(t, server.RecordEpochRewards(record)) + require.NoError(t, server.PrepareEpochRewards(4)) + require.NoError(t, server.SetRootedEpochRewardsSlot(4)) + endpoint := httptest.NewServer(server) + t.Cleanup(endpoint.Close) + endpoint.Client().Timeout = time.Second + call := func(minContextSlot uint64) map[string]json.RawMessage { + t.Helper() + body, err := json.Marshal(map[string]any{"jsonrpc": "2.0", "id": 1, "method": "getInflationReward", "params": []any{[]string{address.String()}, map[string]any{"epoch": 0, "minContextSlot": minContextSlot}}}) + require.NoError(t, err) + response, err := endpoint.Client().Post(endpoint.URL, "application/json", bytes.NewReader(body)) + require.NoError(t, err) + defer response.Body.Close() + payload, err := io.ReadAll(response.Body) + require.NoError(t, err) + t.Logf("HTTP=%d response=%s", response.StatusCode, payload) + var result map[string]json.RawMessage + require.NoError(t, json.Unmarshal(payload, &result)) + return result + } + want := `[{"epoch":0,"effectiveSlot":4,"amount":2,"postBalance":4,"commission":null}]` + unavailable := `{"code":-32004,"message":"Epoch reward history is unavailable for epoch 0","data":{"epoch":0,"contextSlot":2}}` + require.JSONEq(t, want, string(call(4)["result"])) + // An in-memory fork switch above the durable root must retain its rewards. + require.NoError(t, server.RewindEpochRewards(5)) + require.JSONEq(t, want, string(call(4)["result"])) + + s := &state.MithrilState{LastRootedSlot: 4, LastRootedContext: tip, LastSlot: 4} + if failPrune { + batchPath := filepath.Join(dir, "rpc-epoch-rewards", "batch-00000000000000000004.json") + require.NoError(t, os.Remove(batchPath)) + require.NoError(t, os.Mkdir(batchPath, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(batchPath, "blocker"), nil, 0o644)) + } + var accountFaultPath string + if name == "accounts cleanup failure" { + accountFaultPath = filepath.Join(db.AcctsDir, "rewound") + require.NoError(t, os.WriteFile(accountFaultPath, nil, 0o644)) + } + if name == "accounts precommit failure" { + manifests, err := accountsdb.ListFoldManifests(db.AcctsDir) + require.NoError(t, err) + for _, manifest := range manifests { + if manifest.ThroughSlot == 4 { + accountFaultPath = manifest.Path + ".rewound" + } + } + require.NotEmpty(t, accountFaultPath) + require.NoError(t, os.Mkdir(accountFaultPath, 0o755)) + } + require.Equal(t, name == "rewind", rewindStoreBelowDivergence(dir, db, s, 3, 100, server)) + account, err := db.GetAccount(2, address) + require.NoError(t, err) + if name == "accounts precommit failure" { + require.Equal(t, uint64(4), account.Lamports, "failed manifest parking must not alter accounts") + } else { + require.Equal(t, uint64(2), account.Lamports) + } + if name != "rewind" { + for _, minContextSlot := range []uint64{2, 4} { + reply := call(minContextSlot) + require.Empty(t, reply["result"]) + require.Contains(t, string(reply["error"]), "RPC history is unavailable during account recovery") + } + if accountFaultPath == "" { + // Inspect the side-store watermark independently of the gate; recovery remains halted in production. + server.SetHistoryRecoveryPending(false) + require.JSONEq(t, `{"code":-32016,"message":"Minimum context slot has not been reached","data":{"contextSlot":2}}`, string(call(4)["error"])) + require.JSONEq(t, unavailable, string(call(2)["error"])) + return + } + require.FileExists(t, filepath.Join(dir, "rpc-epoch-rewards", "batch-00000000000000000004.json"), "failed account rewind must retain rewards") + require.NoError(t, os.Remove(accountFaultPath)) + require.True(t, rewindStoreBelowDivergence(dir, db, s, 3, 100, server)) + } + require.Equal(t, uint64(2), s.LastRootedSlot) + // Replay's startup hook must not restore the discarded durable reward. + require.NoError(t, server.RewindEpochRewards(s.GetResumeSlot())) + require.JSONEq(t, `{"code":-32016,"message":"Minimum context slot has not been reached","data":{"contextSlot":2}}`, string(call(4)["error"])) + require.JSONEq(t, unavailable, string(call(2)["error"])) + + // A replacement on the selected fork becomes visible only after its fold. + record.Rewards[0].Lamports, record.Rewards[0].PostBalance = 3, 5 + require.NoError(t, server.RecordEpochRewards(record)) + require.NoError(t, server.PrepareEpochRewards(4)) + require.JSONEq(t, unavailable, string(call(2)["error"])) + require.NoError(t, server.SetRootedEpochRewardsSlot(4)) + require.JSONEq(t, `[{"epoch":0,"effectiveSlot":4,"amount":3,"postBalance":5,"commission":null}]`, string(call(4)["result"])) + }) + } +} diff --git a/cmd/mithril/node/node.go b/cmd/mithril/node/node.go index eab065b6..1f60b6aa 100644 --- a/cmd/mithril/node/node.go +++ b/cmd/mithril/node/node.go @@ -2574,7 +2574,16 @@ postBootstrap: if rpcPort < 0 || rpcPort > 65535 { klog.Fatalf("invalid port: %d", rpcPort) } else if rpcPort != 0 { - rpcServer = rpcserver.NewRpcServer(accountsDb, uint16(rpcPort), epochScheduleFromState(mithrilState), solana.MustHashFromBase58(networkGenesisHash)) + schedule := epochScheduleFromState(mithrilState) + rpcServer = rpcserver.NewRpcServer(accountsDb, uint16(rpcPort), schedule, solana.MustHashFromBase58(networkGenesisHash)) + if alpenglowMode { + if schedule == nil || schedule.SlotsPerEpoch == 0 || schedule.SlotsPerEpoch > math.MaxUint64/3 { + klog.Fatalf("Alpenglow RPC requires a valid epoch schedule") + } + if err := rpcServer.EnableEpochRewards(filepath.Join(accountsPath, "rpc-epoch-rewards"), 3*schedule.SlotsPerEpoch, mithrilState.LastRootedSlot); err != nil { + klog.Fatalf("unable to load RPC epoch rewards: %v", err) + } + } rpcServer.Start() mlog.Log.Infof("Started RPC server on port %d", rpcPort) } @@ -4342,7 +4351,11 @@ func completeInterruptedRewind(accountsDbPath string, accountsDb *accountsdb.Acc // boundary strictly below divSlot and adopts that boundary's context as the // rooted checkpoint. Returns false (caller halts) when no boundary below the // divergence is retained or the rewind/reconcile fails. -func rewindStoreBelowDivergence(accountsDbPath string, accountsDb *accountsdb.AccountsDb, s *state.MithrilState, divSlot uint64, rewindHorizon uint64) bool { +func rewindStoreBelowDivergence(accountsDbPath string, accountsDb *accountsdb.AccountsDb, s *state.MithrilState, divSlot uint64, rewindHorizon uint64, rpcServer replay.SlotCtxSetter) bool { + historyGate, hasHistoryGate := rpcServer.(interface{ SetHistoryRecoveryPending(bool) }) + if hasHistoryGate { + historyGate.SetHistoryRecoveryPending(true) + } points, err := accountsDb.ListRewindPoints() if err != nil { mlog.Log.Errorf("fork switch: cannot list rewind boundaries: %v; halting", err) @@ -4373,12 +4386,21 @@ func rewindStoreBelowDivergence(accountsDbPath string, accountsDb *accountsdb.Ac mlog.Log.Errorf("fork switch: rewind to boundary %d failed: %v; halting", target, err) return false } + if rewards, ok := rpcServer.(interface{ SetRootedEpochRewardsSlot(uint64) error }); ok { + if err := rewards.SetRootedEpochRewardsSlot(res.NewThrough); err != nil { + mlog.Log.Errorf("fork switch: rewind RPC epoch rewards to slot %d: %v; halting", res.NewThrough, err) + return false + } + } if err := adoptRewindResult(accountsDbPath, s, res); err != nil { mlog.Log.Errorf("fork switch: %v; halting", err) return false } mlog.Log.Warnf("fork switch: superseded slot %d was already folded (R=%d); rewound durable state to boundary %d (%d batches / %d keys undone)", divSlot, oldRooted, res.NewThrough, res.UndoneBatches, res.UndoneKeys) + if hasHistoryGate { + historyGate.SetHistoryRecoveryPending(false) + } return true } @@ -4607,7 +4629,7 @@ func runReplayWithRecovery( // (this is exactly why durable state deliberately lags and undo pointers // are retained for a horizon). Beyond the horizon -> fail closed. if divSlot <= mithrilState.LastRootedSlot { - if !rewindStoreBelowDivergence(accountsDbPath, accountsDb, mithrilState, divSlot, rewindHorizon) { + if !rewindStoreBelowDivergence(accountsDbPath, accountsDb, mithrilState, divSlot, rewindHorizon, rpcServer) { break } } diff --git a/pkg/blockprod/scheduler/scheduler_test.go b/pkg/blockprod/scheduler/scheduler_test.go index f8371131..f25b731c 100644 --- a/pkg/blockprod/scheduler/scheduler_test.go +++ b/pkg/blockprod/scheduler/scheduler_test.go @@ -233,6 +233,8 @@ func TestSchedulerSkipsWhenSlotEntryBytesExceeded(t *testing.T) { for sched.Stats().DroppedCost == 0 && time.Now().Before(deadline) { time.Sleep(5 * time.Millisecond) } + // Wait for the drain loop to return any temporarily popped entries. + sched.Stop() require.Equal(t, uint64(1), sched.Stats().DroppedCost) require.Equal(t, uint64(1), sched.Stats().DroppedBatchBytes) require.Equal(t, uint64(0), sched.Stats().Accepted) diff --git a/pkg/epochrewards/coverage_test.go b/pkg/epochrewards/coverage_test.go new file mode 100644 index 00000000..c1f78301 --- /dev/null +++ b/pkg/epochrewards/coverage_test.go @@ -0,0 +1,109 @@ +package epochrewards + +import ( + "os" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestEpochCoverageEmptyDistributionAndRetention(t *testing.T) { + dir := t.TempDir() + store, err := Open(dir, 128) + require.NoError(t, err) + store.RecordCoverage(0, 128, true, false) + require.NoError(t, store.Prepare(128)) + require.NoError(t, store.SetRooted(128)) + require.False(t, store.EpochAvailable(0), "an active distribution is incomplete") + store.RecordCoverage(0, 129, false, true) + require.NoError(t, store.Prepare(129)) + require.False(t, store.EpochAvailable(0), "prepared completion cannot precede account commit") + require.NoError(t, store.SetRooted(129)) + require.True(t, store.EpochAvailable(0), "a complete epoch may have no payouts") + reopened, err := Open(dir, 128) + require.NoError(t, err) + require.NoError(t, reopened.SetRooted(129)) + require.True(t, reopened.EpochAvailable(0)) + require.NoError(t, reopened.SetRooted(256)) + require.False(t, reopened.EpochAvailable(0), "a retained completion cannot replace a pruned boundary") + _, err = os.Stat(reopened.batchPath(129)) + require.NoError(t, err, "the completion batch itself remains retained") +} + +func TestEpochCoverageRewindAndPartialFoldRecovery(t *testing.T) { + dir := t.TempDir() + store, err := Open(dir, 512) + require.NoError(t, err) + store.RecordCoverage(0, 128, true, false) + require.NoError(t, store.Prepare(128)) + store.RecordCoverage(0, 129, false, true) + require.NoError(t, store.Prepare(130)) + + // Only the first account fold committed. The later duplicate boundary and + // completion must not hide or replace the committed boundary on reopen. + store, err = Open(dir, 512) + require.NoError(t, err) + require.NoError(t, store.SetRooted(128)) + require.False(t, store.EpochAvailable(0)) + store.RecordCoverage(0, 131, false, true) + require.NoError(t, store.Prepare(132)) + require.NoError(t, store.Rewind(131)) + require.False(t, store.EpochAvailable(0)) + store.RecordCoverage(0, 133, false, true) + require.NoError(t, store.Prepare(134)) + require.NoError(t, store.SetRooted(134)) + require.True(t, store.EpochAvailable(0)) + store, err = Open(dir, 512) + require.NoError(t, err) + require.NoError(t, store.SetRooted(134)) + require.True(t, store.EpochAvailable(0)) +} + +func TestEpochCoverageExpiresEvenWhenBothMarkersRemain(t *testing.T) { + store, err := Open(t.TempDir(), 128) + require.NoError(t, err) + store.RecordCoverage(0, 128, true, false) + store.RecordCoverage(0, 140, false, true) + require.NoError(t, store.Prepare(200)) + require.NoError(t, store.SetRooted(200)) + require.True(t, store.EpochAvailable(0)) + require.NoError(t, store.SetRooted(256)) + _, err = os.Stat(store.batchPath(200)) + require.NoError(t, err) + require.False(t, store.EpochAvailable(0), "coverage cannot extend beyond the epoch's retained boundary") +} + +func TestEpochCoverageDoesNotInferSnapshotHistory(t *testing.T) { + store, err := Open(t.TempDir(), 512) + require.NoError(t, err) + store.RecordCoverage(0, 150, false, true) + require.NoError(t, store.Prepare(150)) + require.NoError(t, store.SetRooted(150)) + require.False(t, store.EpochAvailable(0), "inactive rewards after a snapshot do not prove the missing boundary") + files, err := batchFiles(store.dir) + require.NoError(t, err) + require.Empty(t, files) + + // Even a surviving completion event in an older batch is insufficient. + require.NoError(t, os.WriteFile(store.batchPath(150), []byte(`{"version":1,"through":150,"records":[],"coverage":[{"epoch":0,"slot":150,"boundarySlot":128,"complete":true}]}`), 0o600)) + store, err = Open(store.dir, 512) + require.NoError(t, err) + require.NoError(t, store.SetRooted(150)) + require.False(t, store.EpochAvailable(0)) +} + +func TestEpochCoverageRejectsMalformedBatches(t *testing.T) { + for _, coverage := range []string{ + `[{"epoch":0,"slot":151,"boundarySlot":128,"complete":true}]`, + `[{"epoch":0,"slot":150,"boundarySlot":151,"complete":true}]`, + `[{"epoch":0,"slot":150,"boundarySlot":128,"complete":false}]`, + `[{"epoch":0,"slot":128,"boundarySlot":128},{"epoch":0,"slot":129,"boundarySlot":129}]`, + } { + dir := t.TempDir() + store, err := Open(dir, 512) + require.NoError(t, err) + require.NoError(t, os.WriteFile(store.batchPath(150), []byte(`{"version":1,"through":150,"records":[],"coverage":`+coverage+`}`), 0o600)) + _, err = Open(dir, 512) + require.ErrorContains(t, err, "invalid epoch reward coverage") + } +} diff --git a/pkg/epochrewards/read_view.go b/pkg/epochrewards/read_view.go new file mode 100644 index 00000000..0eb9af70 --- /dev/null +++ b/pkg/epochrewards/read_view.go @@ -0,0 +1,116 @@ +package epochrewards + +import "slices" + +// Publish advances the process-local read view through a verified canonical +// prefix. The caller must bound slot by the currently executed bank and apply +// the execution and finality gates before calling it. +// This does not commit accounts or change the durable recovery watermark. +func (s *Store) Publish(slot uint64) { + s.mu.Lock() + defer s.mu.Unlock() + if slot <= s.readable { + return + } + var slots []uint64 + for pendingSlot := range s.unpublished { + if pendingSlot <= slot { + slots = append(slots, pendingSlot) + } + } + slices.Sort(slots) + for _, pendingSlot := range slots { + for _, record := range s.unpublished[pendingSlot] { + s.published[rewardKey{epoch: record.Epoch, address: record.Address}] = record + } + delete(s.unpublished, pendingSlot) + } + for _, event := range s.pendingCoverage { + if event.Slot <= slot { + key := event.key() + if previous, ok := s.publishedCoverage[key]; !ok || event.Slot > previous.Slot { + s.publishedCoverage[key] = event + } + } + } + s.readable = slot +} + +// ResetPublished discards the verified overlay. Recovered replay must verify +// pending capture again before it can be published; durable history is kept. +func (s *Store) ResetPublished() { + s.mu.Lock() + defer s.mu.Unlock() + s.resetPublishedLocked() +} + +func (s *Store) resetPublishedLocked() { + s.readable = s.rooted.Load() + clear(s.published) + clear(s.publishedCoverage) + clear(s.unpublished) + for _, record := range s.pending { + s.unpublished[record.EffectiveSlot] = append(s.unpublished[record.EffectiveSlot], record) + } +} + +// PublishedSlot returns the verified read watermark, including the durable +// checkpoint when no newer process-local view is available. +func (s *Store) PublishedSlot() uint64 { + s.mu.RLock() + defer s.mu.RUnlock() + return s.readable +} + +// GetPublished returns a detached reward from the verified view. RPC callers +// serialize publication and recovery across the entire multi-address request. +func (s *Store) GetPublished(epoch uint64, address string) (Record, bool) { + s.mu.RLock() + defer s.mu.RUnlock() + key := rewardKey{epoch: epoch, address: address} + record, ok := s.published[key] + if !ok { + record, ok = s.persisted[key] + ok = ok && record.EffectiveSlot <= s.rooted.Load() && s.sourceBatch[key] <= s.rooted.Load() + } + if !ok || record.EffectiveSlot > s.readable { + return Record{}, false + } + record.Commission = cloneUint8(record.Commission) + return record, true +} + +// EpochPublished reports complete retained coverage in the verified read view. +func (s *Store) EpochPublished(epoch uint64) bool { + s.mu.RLock() + defer s.mu.RUnlock() + return s.epochAvailableLocked(epoch, s.readable, true) +} + +// PublishedDistribution returns progress bound to an observed, retained epoch +// boundary. A mid-epoch snapshot alone cannot establish reward history. +func (s *Store) PublishedDistribution(epoch uint64) (Distribution, bool) { + s.mu.RLock() + defer s.mu.RUnlock() + start, started := s.coverageAtLocked(coverageKey{epoch: epoch}, s.readable, true) + progress, observed := s.coverageAtLocked(coverageKey{epoch: epoch, progress: true}, s.readable, true) + if !started || !observed || progress.Slot != s.readable || progress.Distribution == nil || progress.BoundarySlot != start.Slot || start.Slot < s.firstRetainedSlotLocked() { + return Distribution{}, false + } + return *progress.Distribution, true +} + +// FirstRetainedSlot returns the reward retention lower bound, not the first +// block in a ledger. Read publication does not move this durable pruning bound. +func (s *Store) FirstRetainedSlot() uint64 { + s.mu.RLock() + defer s.mu.RUnlock() + return s.firstRetainedSlotLocked() +} + +func (s *Store) firstRetainedSlotLocked() uint64 { + if root := s.rooted.Load(); root > s.retentionSlots { + return root - s.retentionSlots + 1 + } + return 0 +} diff --git a/pkg/epochrewards/read_view_test.go b/pkg/epochrewards/read_view_test.go new file mode 100644 index 00000000..77f5d55d --- /dev/null +++ b/pkg/epochrewards/read_view_test.go @@ -0,0 +1,204 @@ +package epochrewards + +import ( + "fmt" + "os" + "testing" + + b "github.com/Overclock-Validator/mithril/pkg/block" + "github.com/gagliardetto/solana-go" + "github.com/gagliardetto/solana-go/rpc" + "github.com/stretchr/testify/require" +) + +func TestVerifiedRewardViewLifecycle(t *testing.T) { + dir := t.TempDir() + s, err := Open(dir, 100) + require.NoError(t, err) + require.NoError(t, s.SetRooted(10)) + address := solana.PublicKey{1} + commission := uint8(25) + record := func(slot, amount uint64) { + t.Helper() + require.NoError(t, s.RecordBlock(&b.Block{Slot: slot, Epoch: 1, Rewards: []rpc.BlockReward{{ + Pubkey: address, Lamports: int64(amount), PostBalance: 100 + amount, + RewardType: rpc.RewardTypeVoting, Commission: &commission, + }}})) + } + record(11, 7) + s.RecordCoverage(0, 11, true, true) + _, ok := s.GetPublished(0, address.String()) + require.False(t, ok) + require.False(t, s.EpochPublished(0)) + s.Publish(11) + require.Equal(t, uint64(10), s.RootedSlot()) + require.Equal(t, uint64(11), s.PublishedSlot()) + reward, ok := s.GetPublished(0, address.String()) + require.True(t, ok) + require.Equal(t, uint64(7), reward.Amount) + require.True(t, s.EpochPublished(0)) + *reward.Commission = 99 + reward, _ = s.GetPublished(0, address.String()) + require.Equal(t, uint8(25), *reward.Commission, "returned values must not alias the published view") + + record(12, 9) + reward, _ = s.GetPublished(0, address.String()) + require.Equal(t, uint64(7), reward.Amount, "speculative overwrite cannot change a published reward") + require.NoError(t, s.Prepare(12)) + reward, _ = s.GetPublished(0, address.String()) + require.Equal(t, uint64(7), reward.Amount, "prepared files are not read publication") + reopened, err := Open(dir, 100) + require.NoError(t, err) + require.NoError(t, reopened.SetRooted(10)) + _, ok = reopened.GetPublished(0, address.String()) + require.False(t, ok, "restart trusts only the durable account root") + require.Equal(t, uint64(10), reopened.PublishedSlot()) + + require.NoError(t, s.Rewind(12)) + require.Equal(t, uint64(10), s.PublishedSlot(), "rewind withdraws the entire uncommitted view") + _, ok = s.GetPublished(0, address.String()) + require.False(t, ok) + record(11, 7) + s.Publish(11) + require.NoError(t, s.Prepare(11)) + require.NoError(t, s.SetRooted(11)) + require.Empty(t, s.published, "durable history takes ownership") + require.Empty(t, s.unpublished) + reward, ok = s.GetPublished(0, address.String()) + require.True(t, ok) + require.Equal(t, uint64(7), reward.Amount) + s.ResetPublished() + reward, ok = s.GetPublished(0, address.String()) + require.True(t, ok, "reset does not delete durable rewards") + reopened, err = Open(dir, 100) + require.NoError(t, err) + require.NoError(t, reopened.SetRooted(11)) + restored, ok := reopened.GetPublished(0, address.String()) + require.True(t, ok) + require.Equal(t, reward, restored) + require.NoError(t, s.SetRooted(112)) + _, ok = s.GetPublished(0, address.String()) + require.False(t, ok, "retention applies after publication too") + require.False(t, s.EpochPublished(0)) +} + +func TestVerifiedViewSurvivesFailedDurableMaintenance(t *testing.T) { + s, err := Open(t.TempDir(), 100) + require.NoError(t, err) + address := solana.PublicKey{1} + require.NoError(t, s.SetRooted(10)) + require.NoError(t, s.RecordBlock(&b.Block{Slot: 11, Epoch: 1, Rewards: []rpc.BlockReward{{ + Pubkey: address, Lamports: 7, RewardType: rpc.RewardTypeVoting, + }}})) + s.RecordCoverage(0, 11, true, true) + s.Publish(11) + require.NoError(t, s.Prepare(11)) + require.NoError(t, s.Prepare(12)) + path := s.batchPath(11) + require.NoError(t, os.Rename(path, path+".saved")) + require.Error(t, s.SetRooted(11), "missing committed batch makes orphan recovery fail") + require.Equal(t, uint64(10), s.RootedSlot()) + require.Equal(t, uint64(11), s.PublishedSlot()) + reward, ok := s.GetPublished(0, address.String()) + require.True(t, ok, "failed durable maintenance must not erase verified capture") + require.Equal(t, uint64(7), reward.Amount) + require.True(t, s.EpochPublished(0)) + require.NoError(t, os.Rename(path+".saved", path)) + require.NoError(t, s.SetRooted(11)) + require.Equal(t, uint64(11), s.RootedSlot()) + require.Empty(t, s.published) + reward, ok = s.GetPublished(0, address.String()) + require.True(t, ok) + require.Equal(t, uint64(7), reward.Amount) +} + +func TestVerifiedViewInvalidatedBeforeFailedRewind(t *testing.T) { + s, err := Open(t.TempDir(), 100) + require.NoError(t, err) + require.NoError(t, s.SetRooted(10)) + address := solana.PublicKey{1} + require.NoError(t, s.RecordBlock(&b.Block{Slot: 11, Epoch: 1, Rewards: []rpc.BlockReward{{ + Pubkey: address, Lamports: 7, RewardType: rpc.RewardTypeVoting, + }}})) + s.Publish(11) + require.NoError(t, s.Prepare(11)) + path := s.batchPath(11) + require.NoError(t, os.Remove(path)) + require.NoError(t, os.Mkdir(path, 0o700)) + require.NoError(t, os.WriteFile(path+"/occupied", []byte("fixture"), 0o600)) + require.Error(t, s.Rewind(11)) + require.Equal(t, uint64(10), s.PublishedSlot()) + _, ok := s.GetPublished(0, address.String()) + require.False(t, ok) + s.Publish(11) + _, ok = s.GetPublished(0, address.String()) + require.False(t, ok, "failed cleanup cannot republish the discarded suffix") +} + +func TestVerifiedPublicationOrdersCapturedSlots(t *testing.T) { + s, err := Open(t.TempDir(), 100) + require.NoError(t, err) + address := solana.PublicKey{1} + for _, slot := range []uint64{13, 11, 12} { + require.NoError(t, s.RecordBlock(&b.Block{Slot: slot, Epoch: 1, Rewards: []rpc.BlockReward{{ + Pubkey: address, Lamports: int64(slot), RewardType: rpc.RewardTypeVoting, + }}})) + } + s.Publish(12) + reward, ok := s.GetPublished(0, address.String()) + require.True(t, ok) + require.Equal(t, uint64(12), reward.Amount) + s.Publish(13) + reward, ok = s.GetPublished(0, address.String()) + require.True(t, ok) + require.Equal(t, uint64(13), reward.Amount) +} + +func TestVerifiedCoverageRetentionFollowsDurableCheckpoint(t *testing.T) { + s, err := Open(t.TempDir(), 10) + require.NoError(t, err) + s.RecordCoverage(0, 1, true, true) + s.Publish(1) + require.NoError(t, s.Prepare(1)) + require.NoError(t, s.SetRooted(1)) + s.Publish(20) + require.True(t, s.EpochPublished(0), "read publication does not prune retained coverage") + require.NoError(t, s.SetRooted(20)) + require.False(t, s.EpochPublished(0), "durable pruning invalidates old coverage") +} + +func BenchmarkVerifiedRewardPublication(bm *testing.B) { + for _, count := range []int{1000, 100000} { + bm.Run(fmt.Sprintf("steady/%d", count), func(bm *testing.B) { + s, err := Open(bm.TempDir(), 1000000) + require.NoError(bm, err) + for i := 0; i < count; i++ { + key := rewardKey{epoch: 0, address: fmt.Sprint(i)} + s.published[key] = Record{Epoch: 0, Address: key.address, EffectiveSlot: 1} + } + bm.ReportAllocs() + bm.ResetTimer() + for i := 0; i < bm.N; i++ { + s.Publish(uint64(i + 2)) + } + }) + bm.Run(fmt.Sprintf("boundary/%d", count), func(bm *testing.B) { + s, err := Open(bm.TempDir(), 1000000) + require.NoError(bm, err) + records := make([]Record, count) + for i := range records { + records[i] = Record{Epoch: 0, Address: fmt.Sprint(i), EffectiveSlot: 1} + } + bm.ReportAllocs() + bm.ResetTimer() + for i := 0; i < bm.N; i++ { + bm.StopTimer() + s.readable = 0 + s.published = make(map[rewardKey]Record) + s.unpublished[1] = records + bm.StartTimer() + s.Publish(1) + } + }) + } +} diff --git a/pkg/epochrewards/recovery_test.go b/pkg/epochrewards/recovery_test.go new file mode 100644 index 00000000..21346431 --- /dev/null +++ b/pkg/epochrewards/recovery_test.go @@ -0,0 +1,35 @@ +package epochrewards + +import ( + "os" + "testing" + + b "github.com/Overclock-Validator/mithril/pkg/block" + "github.com/gagliardetto/solana-go" + "github.com/gagliardetto/solana-go/rpc" + "github.com/stretchr/testify/require" +) + +func TestStoreRestoresCommittedRewardAfterOrphanReadRetry(t *testing.T) { + store, err := Open(t.TempDir(), 100) + require.NoError(t, err) + address := solana.PublicKey{5} + for _, through := range []uint64{6, 8} { + require.NoError(t, store.RecordBlock(&b.Block{Slot: 5, Epoch: 1, Rewards: []rpc.BlockReward{{ + Pubkey: address, Lamports: int64(through), RewardType: rpc.RewardTypeStaking, + }}})) + require.NoError(t, store.Prepare(through)) + } + + committed := store.batchPath(6) + require.NoError(t, os.Rename(committed, committed+".saved")) + require.Error(t, store.SetRooted(6)) + require.FileExists(t, store.batchPath(8), "a failed recovery must leave the orphan available for retry") + _, ok := store.Get(0, address.String(), false) + require.False(t, ok) + require.NoError(t, os.Rename(committed+".saved", committed)) + require.NoError(t, store.SetRooted(6)) + record, ok := store.Get(0, address.String(), false) + require.True(t, ok) + require.Equal(t, uint64(6), record.Amount) +} diff --git a/pkg/epochrewards/store.go b/pkg/epochrewards/store.go new file mode 100644 index 00000000..bb8979b1 --- /dev/null +++ b/pkg/epochrewards/store.go @@ -0,0 +1,597 @@ +package epochrewards + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + "sync/atomic" + + b "github.com/Overclock-Validator/mithril/pkg/block" + "github.com/gagliardetto/solana-go/rpc" +) + +const recordVersion = 1 + +// Record describes one account's inflation reward for an epoch. +type Record struct { + Epoch uint64 `json:"epoch"` + EffectiveSlot uint64 `json:"effectiveSlot"` + Address string `json:"address"` + Amount uint64 `json:"amount"` + PostBalance uint64 `json:"postBalance"` + Commission *uint8 `json:"commission,omitempty"` +} + +type rewardKey struct { + epoch uint64 + address string +} + +type batch struct { + Version uint32 `json:"version"` + Through uint64 `json:"through"` + Records []Record `json:"records"` + Coverage []epochCoverage `json:"coverage,omitempty"` + Progress []epochCoverage `json:"progress,omitempty"` +} + +type coverageKey struct { + epoch uint64 + complete bool + progress bool + slot uint64 +} + +type epochCoverage struct { + Epoch uint64 `json:"epoch"` + Slot uint64 `json:"slot"` + BoundarySlot uint64 `json:"boundarySlot"` + Complete bool `json:"complete"` + Distribution *Distribution `json:"distribution,omitempty"` +} + +// Distribution is the reward-partition progress in one successfully executed bank. +type Distribution struct { + BlockHeight uint64 `json:"blockHeight"` + StartingBlockHeight uint64 `json:"startingBlockHeight"` + NumPartitions uint64 `json:"numPartitions"` + ParentBlockhash [32]byte `json:"parentBlockhash"` +} + +func (e epochCoverage) key() coverageKey { + return coverageKey{epoch: e.Epoch, complete: e.Complete, progress: e.Distribution != nil} +} + +// Store keeps speculative rewards in memory and persists them with the same +// rooted fold that commits the credited account balances. +type Store struct { + dir string + retentionSlots uint64 + rooted atomic.Uint64 + mu sync.RWMutex + pending map[rewardKey]Record + persisted map[rewardKey]Record + sourceBatch map[rewardKey]uint64 + batches map[uint64][]rewardKey + pendingCoverage map[coverageKey]epochCoverage + coverage map[coverageKey]epochCoverage + coverageBatch map[coverageKey]uint64 + readable uint64 + unpublished map[uint64][]Record + published map[rewardKey]Record + publishedCoverage map[coverageKey]epochCoverage +} + +// Open loads or creates an epoch reward store. +func Open(dir string, retentionSlots uint64) (*Store, error) { + if retentionSlots == 0 { + return nil, errors.New("epoch reward retention must be greater than zero") + } + if err := os.MkdirAll(dir, 0o755); err != nil { + return nil, fmt.Errorf("create epoch reward directory: %w", err) + } + store := &Store{ + dir: dir, + retentionSlots: retentionSlots, + pending: make(map[rewardKey]Record), + persisted: make(map[rewardKey]Record), + sourceBatch: make(map[rewardKey]uint64), + batches: make(map[uint64][]rewardKey), + pendingCoverage: make(map[coverageKey]epochCoverage), + coverage: make(map[coverageKey]epochCoverage), + coverageBatch: make(map[coverageKey]uint64), + unpublished: make(map[uint64][]Record), + published: make(map[rewardKey]Record), + publishedCoverage: make(map[coverageKey]epochCoverage), + } + entries, err := os.ReadDir(dir) + if err != nil { + return nil, fmt.Errorf("read epoch reward directory: %w", err) + } + for _, entry := range entries { + if entry.IsDir() || !strings.HasPrefix(entry.Name(), "batch-") || !strings.HasSuffix(entry.Name(), ".json") { + continue + } + through, err := strconv.ParseUint(strings.TrimSuffix(strings.TrimPrefix(entry.Name(), "batch-"), ".json"), 10, 64) + if err != nil { + continue + } + loaded, err := readBatch(filepath.Join(dir, entry.Name())) + if err != nil { + return nil, fmt.Errorf("load epoch reward batch %d: %w", through, err) + } + if loaded.Through != through { + return nil, fmt.Errorf("load epoch reward batch %d: record contains through %d", through, loaded.Through) + } + store.install(loaded) + } + return store, nil +} + +// RecordBlock adds inflation rewards from a successfully replayed block. +func (s *Store) RecordBlock(block *b.Block) error { + if block == nil { + return errors.New("record epoch rewards: nil block") + } + if block.Epoch == 0 { + return nil + } + rewardedEpoch := block.Epoch - 1 + s.mu.Lock() + defer s.mu.Unlock() + var captured []Record + for _, reward := range block.Rewards { + if reward.Lamports < 0 || (reward.RewardType != rpc.RewardTypeVoting && reward.RewardType != rpc.RewardTypeStaking) { + continue + } + address := reward.Pubkey.String() + commission := cloneUint8(reward.Commission) + record := Record{ + Epoch: rewardedEpoch, EffectiveSlot: block.Slot, Address: address, + Amount: uint64(reward.Lamports), PostBalance: reward.PostBalance, Commission: commission, + } + s.pending[rewardKey{epoch: rewardedEpoch, address: address}] = record + captured = append(captured, record) + } + if len(captured) != 0 { + s.unpublished[block.Slot] = captured + } + return nil +} + +// RecordCoverage records an observed epoch boundary and reward completion. +// A completion without the matching boundary cannot establish full history. +func (s *Store) RecordCoverage(epoch, slot uint64, boundary, complete bool) { + s.mu.Lock() + defer s.mu.Unlock() + startKey := coverageKey{epoch: epoch} + if boundary { + s.pendingCoverage[startKey] = epochCoverage{Epoch: epoch, Slot: slot, BoundarySlot: slot} + } + if !complete { + return + } + start, ok := s.pendingCoverage[startKey] + if !ok { + start, ok = s.coverage[startKey] + } + if !ok || start.Slot > slot { + return + } + endKey := coverageKey{epoch: epoch, complete: true} + if end, ok := s.pendingCoverage[endKey]; ok && end.BoundarySlot == start.Slot { + return + } + if end, ok := s.coverage[endKey]; ok && end.BoundarySlot == start.Slot { + return + } + s.pendingCoverage[endKey] = epochCoverage{Epoch: epoch, Slot: slot, BoundarySlot: start.Slot, Complete: true} +} + +// RecordDistribution captures progress without making the executed bank readable. +// Each pending bank is kept until publication so a later speculative bank cannot +// make an earlier verified partition appear complete. +func (s *Store) RecordDistribution(epoch, slot uint64, progress Distribution) { + s.mu.Lock() + defer s.mu.Unlock() + start, ok := s.pendingCoverage[coverageKey{epoch: epoch}] + if !ok { + start, ok = s.coverage[coverageKey{epoch: epoch}] + } + if !ok || start.Slot > slot || progress.NumPartitions == 0 { + return + } + s.pendingCoverage[coverageKey{epoch: epoch, progress: true, slot: slot}] = epochCoverage{ + Epoch: epoch, Slot: slot, BoundarySlot: start.Slot, Distribution: &progress, + } +} + +// Prepare writes rewards through a fold boundary before the fold is committed. +func (s *Store) Prepare(through uint64) error { + s.mu.RLock() + records := make([]Record, 0) + for _, record := range s.pending { + if record.EffectiveSlot <= through { + records = append(records, record) + } + } + latest := make(map[coverageKey]epochCoverage) + for _, event := range s.pendingCoverage { + if event.Slot <= through { + key := event.key() + if previous, ok := latest[key]; !ok || event.Slot > previous.Slot { + latest[key] = event + } + } + } + s.mu.RUnlock() + coverage, progress := make([]epochCoverage, 0), make([]epochCoverage, 0) + for _, event := range latest { + if event.Distribution != nil { + progress = append(progress, event) + } else { + coverage = append(coverage, event) + } + } + if len(records) == 0 && len(coverage) == 0 && len(progress) == 0 { + return nil + } + sort.Slice(records, func(i, j int) bool { + if records[i].Epoch != records[j].Epoch { + return records[i].Epoch < records[j].Epoch + } + return records[i].Address < records[j].Address + }) + sort.Slice(coverage, func(i, j int) bool { + if coverage[i].Epoch != coverage[j].Epoch { + return coverage[i].Epoch < coverage[j].Epoch + } + return !coverage[i].Complete && coverage[j].Complete + }) + sort.Slice(progress, func(i, j int) bool { return progress[i].Epoch < progress[j].Epoch }) + payload := batch{Version: recordVersion, Through: through, Records: records, Coverage: coverage, Progress: progress} + tmp, err := os.CreateTemp(s.dir, ".epoch-rewards-*") + if err != nil { + return fmt.Errorf("create epoch reward temp file: %w", err) + } + tmpPath := tmp.Name() + defer os.Remove(tmpPath) + writeErr := json.NewEncoder(tmp).Encode(payload) + if writeErr == nil { + writeErr = tmp.Sync() + } + closeErr := tmp.Close() + if writeErr != nil { + return fmt.Errorf("write epoch rewards through slot %d: %w", through, writeErr) + } + if closeErr != nil { + return fmt.Errorf("close epoch rewards through slot %d: %w", through, closeErr) + } + if err := os.Rename(tmpPath, s.batchPath(through)); err != nil { + return fmt.Errorf("publish epoch rewards through slot %d: %w", through, err) + } + if err := syncDir(s.dir); err != nil { + return err + } + s.mu.Lock() + s.install(payload) + s.mu.Unlock() + return nil +} + +func (s *Store) install(value batch) { + keys := make([]rewardKey, 0, len(value.Records)) + for _, record := range value.Records { + key := rewardKey{epoch: record.Epoch, address: record.Address} + s.persisted[key] = record + s.sourceBatch[key] = value.Through + keys = append(keys, key) + } + s.batches[value.Through] = keys + for _, event := range append(value.Coverage, value.Progress...) { + key := event.key() + s.coverage[key] = event + s.coverageBatch[key] = value.Through + } +} + +// SetRooted sets the durable reward view and prunes stale batches. +func (s *Store) SetRooted(slot uint64) error { + s.mu.Lock() + if slot < s.rooted.Load() { + // A failed prune must not expose rewards above the recovered account root. + s.rooted.Store(slot) + s.resetPublishedLocked() + } + for key, record := range s.pending { + if record.EffectiveSlot <= slot { + delete(s.pending, key) + } + } + for key, event := range s.pendingCoverage { + if event.Slot <= slot { + delete(s.pendingCoverage, key) + } + } + pruneBefore := uint64(0) + if slot > s.retentionSlots { + pruneBefore = slot - s.retentionSlots + } + removed := false + for through := range s.batches { + if through > slot { + if err := s.restoreRetainedRecords(pruneBefore, slot); err != nil { + s.mu.Unlock() + return err + } + break + } + } + for through, keys := range s.batches { + orphaned := through > slot + expired := pruneBefore > 0 && through <= pruneBefore + if !orphaned && !expired { + continue + } + if err := os.Remove(s.batchPath(through)); err != nil && !os.IsNotExist(err) { + s.mu.Unlock() + return fmt.Errorf("prune epoch reward batch %d: %w", through, err) + } + removed = true + for _, key := range keys { + if s.sourceBatch[key] == through { + delete(s.persisted, key) + delete(s.sourceBatch, key) + } + } + delete(s.batches, through) + s.removeCoverageBatch(through) + } + s.mu.Unlock() + if removed { + if err := syncDir(s.dir); err != nil { + return err + } + } + s.mu.Lock() + s.rooted.Store(slot) + s.readable = max(s.readable, slot) + for key, record := range s.published { + if record.EffectiveSlot <= slot { + delete(s.published, key) + } + } + for key, event := range s.publishedCoverage { + if event.Slot <= slot { + delete(s.publishedCoverage, key) + } + } + for pendingSlot := range s.unpublished { + if pendingSlot <= slot { + delete(s.unpublished, pendingSlot) + } + } + s.mu.Unlock() + return nil +} + +// Rewind removes speculative rewards produced by a discarded fork suffix. +func (s *Store) Rewind(fromSlot uint64) error { + s.mu.Lock() + // Invalidate the read overlay before any fallible disk cleanup. + s.resetPublishedLocked() + for slot := range s.unpublished { + if slot >= fromSlot { + delete(s.unpublished, slot) + } + } + for key, record := range s.pending { + if record.EffectiveSlot >= fromSlot { + delete(s.pending, key) + } + } + for key, event := range s.pendingCoverage { + if event.Slot >= fromSlot { + delete(s.pendingCoverage, key) + } + } + removed := false + for through := range s.batches { + if through >= fromSlot && through > s.rooted.Load() { + // Keep every committed batch, including when fromSlot is below the root. + limit := max(fromSlot, s.rooted.Load()+1) - 1 + if err := s.restoreRetainedRecords(0, limit); err != nil { + s.mu.Unlock() + return err + } + break + } + } + for through, keys := range s.batches { + if through < fromSlot || through <= s.rooted.Load() { + continue + } + if err := os.Remove(s.batchPath(through)); err != nil && !os.IsNotExist(err) { + s.mu.Unlock() + return fmt.Errorf("rewind epoch reward batch %d: %w", through, err) + } + removed = true + for _, key := range keys { + if s.sourceBatch[key] == through { + delete(s.persisted, key) + delete(s.sourceBatch, key) + } + } + delete(s.batches, through) + s.removeCoverageBatch(through) + } + s.mu.Unlock() + if removed { + return syncDir(s.dir) + } + return nil +} + +// An interrupted multi-chunk fold can leave the same reward in several batches. +// Removing an uncommitted batch must reveal the preceding committed record. +func (s *Store) restoreRetainedRecords(pruneBefore, throughLimit uint64) error { + throughs := make([]uint64, 0, len(s.batches)) + for through := range s.batches { + if through <= throughLimit && (pruneBefore == 0 || through > pruneBefore) { + throughs = append(throughs, through) + } + } + sort.Slice(throughs, func(i, j int) bool { return throughs[i] < throughs[j] }) + persisted := make(map[rewardKey]Record) + sources := make(map[rewardKey]uint64) + coverage := make(map[coverageKey]epochCoverage) + coverageBatches := make(map[coverageKey]uint64) + for _, through := range throughs { + value, err := readBatch(s.batchPath(through)) + if err != nil { + return fmt.Errorf("restore epoch reward batch %d: %w", through, err) + } + if value.Through != through { + return fmt.Errorf("restore epoch reward batch %d: record contains through %d", through, value.Through) + } + for _, record := range value.Records { + key := rewardKey{epoch: record.Epoch, address: record.Address} + persisted[key] = record + sources[key] = through + } + for _, event := range append(value.Coverage, value.Progress...) { + key := event.key() + coverage[key], coverageBatches[key] = event, through + } + } + s.persisted, s.sourceBatch = persisted, sources + s.coverage, s.coverageBatch = coverage, coverageBatches + return nil +} + +func (s *Store) removeCoverageBatch(through uint64) { + for key, source := range s.coverageBatch { + if source == through { + delete(s.coverage, key) + delete(s.coverageBatch, key) + } + } +} + +// EpochAvailable reports complete, retained history at the durable account root. +func (s *Store) EpochAvailable(epoch uint64) bool { + s.mu.RLock() + defer s.mu.RUnlock() + return s.epochAvailableLocked(epoch, s.rooted.Load(), false) +} + +func (s *Store) epochAvailableLocked(epoch, root uint64, published bool) bool { + startKey, endKey := coverageKey{epoch: epoch}, coverageKey{epoch: epoch, complete: true} + start, started := s.coverageAtLocked(startKey, root, published) + end, completed := s.coverageAtLocked(endKey, root, published) + if !started || !completed || start.Slot != start.BoundarySlot || end.BoundarySlot != start.Slot || end.Slot < start.Slot { + return false + } + if start.Slot > root || end.Slot > root { + return false + } + // Retention follows durable folds, not process-local read publication. A + // later completion marker cannot certify an expired boundary batch. + durable := s.rooted.Load() + return durable <= s.retentionSlots || start.Slot > durable-s.retentionSlots +} + +func (s *Store) coverageAtLocked(key coverageKey, root uint64, published bool) (epochCoverage, bool) { + if published { + if event, ok := s.publishedCoverage[key]; ok { + return event, event.Slot <= root + } + } + event, ok := s.coverage[key] + return event, ok && event.Slot <= root && event.Slot <= s.rooted.Load() && s.coverageBatch[key] <= s.rooted.Load() +} + +// Get returns an address's reward for an epoch. +func (s *Store) Get(epoch uint64, address string, includePending bool) (Record, bool) { + key := rewardKey{epoch: epoch, address: address} + s.mu.RLock() + defer s.mu.RUnlock() + if includePending { + if record, ok := s.pending[key]; ok { + return record, true + } + } + record, ok := s.persisted[key] + if !ok || record.EffectiveSlot > s.rooted.Load() || s.sourceBatch[key] > s.rooted.Load() { + return Record{}, false + } + return record, true +} + +// RootedSlot returns the durable reward watermark. +func (s *Store) RootedSlot() uint64 { + return s.rooted.Load() +} + +func (s *Store) batchPath(through uint64) string { + return filepath.Join(s.dir, fmt.Sprintf("batch-%020d.json", through)) +} + +func readBatch(path string) (batch, error) { + data, err := os.ReadFile(path) + if err != nil { + return batch{}, err + } + var value batch + if err := json.Unmarshal(data, &value); err != nil { + return batch{}, err + } + if value.Version != recordVersion { + return batch{}, fmt.Errorf("unsupported epoch reward version %d", value.Version) + } + seen := make(map[coverageKey]bool, len(value.Coverage)) + for _, event := range value.Coverage { + key := coverageKey{epoch: event.Epoch, complete: event.Complete} + if event.Distribution != nil || event.Slot > value.Through || event.BoundarySlot > event.Slot || (!event.Complete && event.BoundarySlot != event.Slot) || seen[key] { + return batch{}, fmt.Errorf("invalid epoch reward coverage in batch %d", value.Through) + } + seen[key] = true + } + for _, event := range value.Progress { + key := event.key() + if event.Distribution == nil || event.Distribution.NumPartitions == 0 || event.Complete || event.Slot > value.Through || event.BoundarySlot > event.Slot || seen[key] { + return batch{}, fmt.Errorf("invalid epoch reward progress in batch %d", value.Through) + } + seen[key] = true + } + return value, nil +} + +func syncDir(path string) error { + dir, err := os.Open(path) + if err != nil { + return fmt.Errorf("open epoch reward directory: %w", err) + } + if err := dir.Sync(); err != nil { + _ = dir.Close() + return fmt.Errorf("sync epoch reward directory: %w", err) + } + if err := dir.Close(); err != nil { + return fmt.Errorf("close epoch reward directory: %w", err) + } + return nil +} + +func cloneUint8(value *uint8) *uint8 { + if value == nil { + return nil + } + copy := *value + return © +} diff --git a/pkg/epochrewards/store_test.go b/pkg/epochrewards/store_test.go new file mode 100644 index 00000000..fc24c661 --- /dev/null +++ b/pkg/epochrewards/store_test.go @@ -0,0 +1,170 @@ +package epochrewards + +import ( + "os" + "testing" + + b "github.com/Overclock-Validator/mithril/pkg/block" + "github.com/gagliardetto/solana-go" + "github.com/gagliardetto/solana-go/rpc" + "github.com/stretchr/testify/require" +) + +func TestStoreKeepsPendingAndRootedViewsSeparate(t *testing.T) { + dir := t.TempDir() + store, err := Open(dir, 256) + require.NoError(t, err) + + address := solana.NewWallet().PublicKey() + commission := uint8(7) + require.NoError(t, store.RecordBlock(&b.Block{ + Slot: 130, Epoch: 2, + Rewards: []rpc.BlockReward{{ + Pubkey: address, Lamports: 42, PostBalance: 1_042, + RewardType: rpc.RewardTypeVoting, Commission: &commission, + }}, + })) + + pending, ok := store.Get(1, address.String(), true) + require.True(t, ok) + require.Equal(t, uint64(42), pending.Amount) + require.Equal(t, uint64(130), pending.EffectiveSlot) + require.Equal(t, uint64(1_042), pending.PostBalance) + require.Equal(t, uint8(7), *pending.Commission) + _, ok = store.Get(1, address.String(), false) + require.False(t, ok) + + require.NoError(t, store.Prepare(130)) + require.NoError(t, store.SetRooted(130)) + rooted, ok := store.Get(1, address.String(), false) + require.True(t, ok) + require.Equal(t, pending, rooted) + + reopened, err := Open(dir, 256) + require.NoError(t, err) + require.NoError(t, reopened.SetRooted(130)) + restarted, ok := reopened.Get(1, address.String(), false) + require.True(t, ok) + require.Equal(t, rooted, restarted) +} + +func TestStoreHidesPreparedRewardsUntilTheirFoldCommits(t *testing.T) { + store, err := Open(t.TempDir(), 100) + require.NoError(t, err) + address := solana.PublicKey{1} + require.NoError(t, store.RecordBlock(&b.Block{Slot: 11, Epoch: 2, Rewards: []rpc.BlockReward{{ + Pubkey: address, Lamports: 5, RewardType: rpc.RewardTypeStaking, + }}})) + require.NoError(t, store.Prepare(12)) + store.rooted.Store(11) // The account fold through slot 12 has not committed. + _, ok := store.Get(1, address.String(), false) + require.False(t, ok) + require.NoError(t, store.SetRooted(12)) + _, ok = store.Get(1, address.String(), false) + require.True(t, ok) +} + +func TestStoreKeepsZeroInflationRewards(t *testing.T) { + store, err := Open(t.TempDir(), 32) + require.NoError(t, err) + address := solana.NewWallet().PublicKey() + require.NoError(t, store.RecordBlock(&b.Block{ + Slot: 33, Epoch: 1, + Rewards: []rpc.BlockReward{ + {Pubkey: address, Lamports: 10, RewardType: rpc.RewardTypeFee}, + {Pubkey: address, Lamports: 0, RewardType: rpc.RewardTypeStaking}, + }, + })) + reward, ok := store.Get(0, address.String(), true) + require.True(t, ok) + require.Zero(t, reward.Amount) + require.Equal(t, uint64(33), reward.EffectiveSlot) + require.NoError(t, store.Prepare(33)) + entries, err := batchFiles(store.dir) + require.NoError(t, err) + require.Len(t, entries, 1) +} + +func TestStoreRecordsLiveAlpenglowVotingRewardAfterVATDebit(t *testing.T) { + store, err := Open(t.TempDir(), 108_000) + require.NoError(t, err) + vote := solana.MustPublicKeyFromBase58("TitanB6gCvNeb5RLM1RuGNT1mgmqxR6Q8hXMALQpLuJ") + // A VAT debit must not hide the voting inflation credit in the same block. + require.NoError(t, store.RecordBlock(&b.Block{ + Slot: 7_236_004, Epoch: 134, + Rewards: []rpc.BlockReward{ + {Pubkey: vote, Lamports: -800_000_000, PostBalance: 19_022_056_333_577, RewardType: rpc.RewardType("VATDebit")}, + {Pubkey: vote, Lamports: 413_137_680_751, PostBalance: 19_435_194_014_328, RewardType: rpc.RewardTypeVoting}, + }, + })) + require.NoError(t, store.Prepare(7_236_004)) + require.NoError(t, store.SetRooted(7_236_004)) + reward, ok := store.Get(133, vote.String(), false) + require.True(t, ok) + require.Equal(t, uint64(7_236_004), reward.EffectiveSlot) + require.Equal(t, uint64(413_137_680_751), reward.Amount) + require.Equal(t, uint64(19_435_194_014_328), reward.PostBalance) + require.Nil(t, reward.Commission) +} + +func TestStoreDropsUnselectedAndExpiredBatches(t *testing.T) { + dir := t.TempDir() + store, err := Open(dir, 10) + require.NoError(t, err) + address := solana.NewWallet().PublicKey() + require.NoError(t, store.RecordBlock(&b.Block{ + Slot: 20, Epoch: 2, + Rewards: []rpc.BlockReward{{Pubkey: address, Lamports: 5, RewardType: rpc.RewardTypeStaking}}, + })) + require.NoError(t, store.Prepare(20)) + + reopened, err := Open(dir, 10) + require.NoError(t, err) + require.NoError(t, reopened.SetRooted(19)) + _, ok := reopened.Get(1, address.String(), false) + require.False(t, ok, "a batch ahead of the recovered root is an orphan") + + require.NoError(t, reopened.RecordBlock(&b.Block{ + Slot: 21, Epoch: 2, + Rewards: []rpc.BlockReward{{Pubkey: address, Lamports: 6, RewardType: rpc.RewardTypeStaking}}, + })) + require.NoError(t, reopened.Prepare(21)) + require.NoError(t, reopened.SetRooted(21)) + _, ok = reopened.Get(1, address.String(), false) + require.True(t, ok) + require.NoError(t, reopened.SetRooted(32)) + _, ok = reopened.Get(1, address.String(), false) + require.False(t, ok, "records outside retention are removed") +} + +func TestStoreRewindsSpeculativeRewards(t *testing.T) { + store, err := Open(t.TempDir(), 32) + require.NoError(t, err) + address := solana.NewWallet().PublicKey() + require.NoError(t, store.RecordBlock(&b.Block{ + Slot: 33, Epoch: 1, + Rewards: []rpc.BlockReward{{Pubkey: address, Lamports: 5, RewardType: rpc.RewardTypeVoting}}, + })) + require.NoError(t, store.Prepare(33)) + + require.NoError(t, store.Rewind(33)) + _, ok := store.Get(0, address.String(), true) + require.False(t, ok) + files, err := batchFiles(store.dir) + require.NoError(t, err) + require.Empty(t, files) +} + +func batchFiles(dir string) ([]string, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return nil, err + } + files := make([]string, 0, len(entries)) + for _, entry := range entries { + if !entry.IsDir() { + files = append(files, entry.Name()) + } + } + return files, nil +} diff --git a/pkg/replay/block.go b/pkg/replay/block.go index 9b7bf651..8436ee24 100644 --- a/pkg/replay/block.go +++ b/pkg/replay/block.go @@ -51,6 +51,18 @@ type SlotCtxSetter interface { SetSlotCtx(slotCtx *sealevel.SlotCtx) } +type epochRewardPublisher interface { + RecordEpochRewards(block *b.Block) error + PrepareEpochRewards(through uint64) error + SetRootedEpochRewardsSlot(slot uint64) error + RewindEpochRewards(fromSlot uint64) error +} + +type epochRewardReadPublisher interface { + PublishEpochRewards(slot uint64) + ResetEpochRewardView() +} + // BlockFetchOpts contains options for parallel block fetching type BlockFetchOpts struct { MaxRPS int // Rate limit (requests per second), 0 = use default @@ -1706,6 +1718,21 @@ func ReplayBlocks( ) *ReplayResult { result := &ReplayResult{} alpenglowMode := consensusOpts != nil && consensusOpts.Alpenglow + rewardPublisher, _ := rpcServer.(epochRewardPublisher) + rewardReadPublisher, _ := rpcServer.(epochRewardReadPublisher) + if rewardReadPublisher != nil { + rewardReadPublisher.ResetEpochRewardView() + defer rewardReadPublisher.ResetEpochRewardView() + } + rewardStatusPublisher, _ := rpcServer.(interface { + RecordEpochRewardStatus(*b.Block, *sealevel.BankSysvars, bool) + }) + if rewardPublisher != nil { + if err := rewardPublisher.RewindEpochRewards(startSlot); err != nil { + result.Error = fmt.Errorf("rewind epoch rewards from replay slot %d: %w", startSlot, err) + return result + } + } replayFrontier := uint64(0) if startSlot > 0 { replayFrontier = startSlot - 1 @@ -2023,9 +2050,28 @@ func ReplayBlocks( // on the existing fold worker, after releasing the live cache lock. Capture: transactionStatuses.CaptureSnapshotThrough, Install: func(through uint64, payload []byte) (*state.TransactionStatusCheckpointRef, error) { - return PrepareTransactionStatusCheckpoint(acctsDbPath, through, payload) + ref, err := PrepareTransactionStatusCheckpoint(acctsDbPath, through, payload) + if err != nil { + return nil, err + } + if rewardPublisher != nil { + if err := rewardPublisher.PrepareEpochRewards(through); err != nil { + return nil, fmt.Errorf("prepare epoch rewards: %w", err) + } + } + return ref, nil + }, + AfterCommit: func(ref *state.TransactionStatusCheckpointRef) error { + var err error + if rewardPublisher != nil { + // Forced flushes may commit several chunks before replay applies their results. + err = rewardPublisher.SetRootedEpochRewardsSlot(ref.Root) + } + if checkpointAfterCommit != nil { + err = errors.Join(err, checkpointAfterCommit(ref)) + } + return err }, - AfterCommit: checkpointAfterCommit, }); hookErr != nil { result.Error = fmt.Errorf("configure durable transaction status checkpoints: %w", hookErr) return result @@ -2093,6 +2139,11 @@ func ReplayBlocks( mithrilState.LastRootedSlot = promotedThrough mithrilState.LastRootedBankhash = rootedCtx.Bankhash mithrilState.LastRootedContext = rootedCtx + if rewardPublisher != nil { + if err := rewardPublisher.SetRootedEpochRewardsSlot(promotedThrough); err != nil { + mlog.Log.Errorf("failed to advance rooted epoch rewards through slot %d: %v", promotedThrough, err) + } + } if rewardsCompletion.retire(&partitionedRewardsInfo, promotedThrough) { rewardsHoldBelowSlot = 0 mlog.Log.Infof("epoch rewards bookkeeping retired through durable slot %d; later fork switches may unwind in memory", promotedThrough) @@ -2161,6 +2212,11 @@ func ReplayBlocks( // force-folds the trailing partial chunk (epoch-boundary settlement and // shutdown); force=false folds full chunks only. foldRootedPrefix := func(force bool) (halt bool) { + defer func() { + if halt && rewardReadPublisher != nil { + rewardReadPublisher.ResetEpochRewardView() + } + }() if unrootedTailState == nil { return false } @@ -2203,13 +2259,9 @@ func ReplayBlocks( verifiedWM = trailingVerifier.VerifiedWatermark() } promoteThrough := safePromoteTarget(lastRootedWatermark, verifierRequired, verifiedWM, replayDivergenceFloor) - // Partitioned-rewards window: hold promotion below the boundary block - // until the completion bank verifies and is eligible to fold, so a - // failed distribution re-runs the boundary and rebuilds its bookkeeping. - promoteThrough = rewardsCompletion.limitPromotion(partitionedRewardsInfo, rewardsHoldBelowSlot, promoteThrough) if promoteThrough <= mithrilState.LastRootedSlot { // Operator signal: promotion is fully stalled (verifier lag, - // divergence floor, or rewards hold) while finality has run at + // or divergence floor) while finality has run at // least a whole fold chunk ahead. Healthy steady state stays 0. if lastRootedWatermark >= mithrilState.LastRootedSlot+uint64(FoldBatchSlots) { promotionHolds++ @@ -2237,9 +2289,22 @@ func ReplayBlocks( mlog.Log.FileOnlyf("alpenglow gate: checked=%d matched=%d no_finality=%d no_local_id=%d", gateStats.checked, gateStats.matched, gateStats.noFinality, gateStats.noLocalID) } - // The finality gate can stop before the verified completion bank. - promoteThrough = rewardsCompletion.limitPromotion(partitionedRewardsInfo, rewardsHoldBelowSlot, promoteThrough) + // Read publication uses the same verified prefix without waiting for disk. + // Advisory/disabled verification retains the durable-only read contract. + // Select an actual retained bank, not a high-water mark or skipped slot: + // finality can end in a skipped prefix before the next epoch's first bank. + readThrough, durableThrough := unrootedTailState.rewardPublicationTargets(&rewardsCompletion, partitionedRewardsInfo, rewardsHoldBelowSlot, promoteThrough) + if !force && result.Error == nil && alpenglowMode && rewardReadPublisher != nil && (verifierRequired || TrailingVerifierCfg.ValidatorFooterHash) { + rewardReadPublisher.PublishEpochRewards(readThrough) + } + // Only disk checkpoints wait for the whole distribution, so recovery can + // replay the boundary and reconstruct its consumed reward spools. Reads + // above have already passed execution verification and exact finality. + promoteThrough = durableThrough if promoteThrough <= mithrilState.LastRootedSlot { + if lastRootedWatermark >= mithrilState.LastRootedSlot+uint64(FoldBatchSlots) { + promotionHolds++ + } return false } @@ -2461,6 +2526,9 @@ func ReplayBlocks( // identical; only the source rewind differs (certified sibling re-fetch vs // retaining an already assembled alternate child). handleAlpenglowSwitch := func(sw *CertifiedSwitch, rewindSource func() bool) bool { + if rewardReadPublisher != nil { + rewardReadPublisher.ResetEpochRewardView() + } windowSwitches++ if unrootedTailState == nil || promoter == nil { windowSwitchFallback++ @@ -2520,6 +2588,13 @@ func ReplayBlocks( mlog.Log.Warnf("%v", result.Error) return false } + if rewardPublisher != nil { + if err := rewardPublisher.RewindEpochRewards(sw.Slot); err != nil { + result.Error = fmt.Errorf("%w: rewind epoch rewards: %v", sw, err) + mlog.Log.Warnf("%v", result.Error) + return false + } + } for slot := range alpenglowExecutedBlockIDs { if slot >= sw.Slot { @@ -3124,7 +3199,12 @@ func ReplayBlocks( // post-epoch boundary rewards distribution if partitionedEpochRewardsEnabled && partitionedRewardsInfo != nil && currentSlot >= partitionedRewardsInfo.FirstStakingRewardSlot && partitionedRewardsInfo.NumRewardPartitionsRemaining > 0 { - distributedAccts, parentDistributedAccts := distributePartitionedEpochRewardsForSlot(acctsDb, lastSlotCtx, block.EpochUpdatedAccts, replayCtx, partitionedRewardsInfo, currentSlot, block.BlockHeight) + distributedAccts, parentDistributedAccts, stakeCommissions := distributePartitionedEpochRewardsForSlot(acctsDb, lastSlotCtx, block.EpochUpdatedAccts, replayCtx, partitionedRewardsInfo, currentSlot, block.BlockHeight) + if err := replaceInflationRewardRecords(block, distributedAccts, parentDistributedAccts, rpc.RewardTypeStaking, stakeCommissions); err != nil { + result.Error = err + mlog.Log.Errorf("%v", err) + break + } block.EpochUpdatedAccts = append(block.EpochUpdatedAccts, distributedAccts...) block.ParentEpochUpdatedAccts = append(block.ParentEpochUpdatedAccts, parentDistributedAccts...) } @@ -3172,6 +3252,15 @@ func ReplayBlocks( global.ClearPendingStakePubkeys() break } + if rewardPublisher != nil { + if err := rewardPublisher.RecordEpochRewards(block); err != nil { + result.Error = fmt.Errorf("record epoch rewards for slot %d: %w", block.Slot, err) + break + } + if rewardStatusPublisher != nil { + rewardStatusPublisher.RecordEpochRewardStatus(block, lastSlotCtx.BankSysvars(), partitionedEpochRewardsEnabled) + } + } // The successful child now owns its derived snapshot. Any later bank uses // lastSlotCtx; the one-shot retained unwind bridge is no longer needed. rewardsCompletion.observeBank(partitionedRewardsInfo, lastSlotCtx.BankSysvars()) @@ -3651,6 +3740,10 @@ func ReplayBlocks( } + if rewardReadPublisher != nil { + rewardReadPublisher.ResetEpochRewardView() + } + // Check if block source stalled (this provides explicit error info) if blockStream.Stalled() && result.Error == nil { result.Error = fmt.Errorf("block fetch stalled - no progress for %v", blockStream.StallTimeout()) diff --git a/pkg/replay/epoch_commission_history_test.go b/pkg/replay/epoch_commission_history_test.go index 51956808..70fbbfc9 100644 --- a/pkg/replay/epoch_commission_history_test.go +++ b/pkg/replay/epoch_commission_history_test.go @@ -169,8 +169,8 @@ func TestEpochBoundaryDelayedCommissionForReactivatedStake(t *testing.T) { require.Equal(t, tc.wantVoter, votes[0].Lamports-voteParents[0].Lamports) var paidStake uint64 for partition := uint64(0); partition < result.NumPartitions; partition++ { - stakes, parents, paid, burned := rewards.DistributeStakingRewardsFromSpool( - db, result.SpoolDir, result.SpoolSlot, partition, distributionSlot+1+partition, nil) + stakes, parents, paid, burned, _ := rewards.DistributeStakingRewardsFromSpool( + db, result.SpoolDir, result.SpoolSlot, partition, distributionSlot+1+partition, nil, result.VoteCommissions) require.Zero(t, burned) paidStake += paid for i, acct := range stakes { diff --git a/pkg/replay/epoch_reward_publisher_test.go b/pkg/replay/epoch_reward_publisher_test.go new file mode 100644 index 00000000..8f014e54 --- /dev/null +++ b/pkg/replay/epoch_reward_publisher_test.go @@ -0,0 +1,20 @@ +package replay + +import ( + "testing" + + b "github.com/Overclock-Validator/mithril/pkg/block" +) + +type legacyEpochRewardPublisher struct{} + +func (legacyEpochRewardPublisher) RecordEpochRewards(*b.Block) error { return nil } +func (legacyEpochRewardPublisher) PrepareEpochRewards(uint64) error { return nil } +func (legacyEpochRewardPublisher) SetRootedEpochRewardsSlot(uint64) error { return nil } +func (legacyEpochRewardPublisher) RewindEpochRewards(uint64) error { return nil } + +func TestEpochRewardPublisherDoesNotRequireCoverage(t *testing.T) { + if _, ok := any(legacyEpochRewardPublisher{}).(epochRewardPublisher); !ok { + t.Fatal("existing reward publishers must retain their callbacks without the optional coverage hook") + } +} diff --git a/pkg/replay/epoch_rewards_flush_test.go b/pkg/replay/epoch_rewards_flush_test.go new file mode 100644 index 00000000..ae69dc3a --- /dev/null +++ b/pkg/replay/epoch_rewards_flush_test.go @@ -0,0 +1,71 @@ +package replay + +import ( + "testing" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + b "github.com/Overclock-Validator/mithril/pkg/block" + "github.com/Overclock-Validator/mithril/pkg/epochrewards" + "github.com/Overclock-Validator/mithril/pkg/state" + "github.com/gagliardetto/solana-go" + "github.com/gagliardetto/solana-go/rpc" + "github.com/stretchr/testify/require" +) + +func TestEpochRewardsSurvivePartialForcedFlush(t *testing.T) { + for _, publishEachCommit := range []bool{false, true} { + for _, restart := range []bool{false, true} { + name := "running" + if publishEachCommit { + name += "_publish_each_commit" + } + if restart { + name += "_restart" + } + t.Run(name, func(t *testing.T) { + dir := t.TempDir() + store, err := epochrewards.Open(dir, 100) + require.NoError(t, err) + require.NoError(t, store.SetRooted(4)) + key := solana.PublicKey{5} + require.NoError(t, store.RecordBlock(&b.Block{Slot: 5, Epoch: 1, Rewards: []rpc.BlockReward{{Pubkey: key, Lamports: 7, PostBalance: 107, RewardType: rpc.RewardTypeStaking}}})) + committer := &fakeCommitter{durable: accounts.NewMemAccounts(), failOn: 7} + tail := asyncTestTail(committer, 5, 6, 7, 8) + statuses := NewTransactionStatusCache() + checkpointDir := t.TempDir() + var prepared []uint64 + require.NoError(t, tail.SetTransactionStatusCheckpointHooks(TransactionStatusCheckpointHooks{ + Capture: statuses.CaptureSnapshotThrough, + AfterCommit: func(ref *state.TransactionStatusCheckpointRef) error { + if publishEachCommit { + return store.SetRooted(ref.Root) + } + return nil + }, + Install: func(through uint64, payload []byte) (*state.TransactionStatusCheckpointRef, error) { + ref, err := PrepareTransactionStatusCheckpoint(checkpointDir, through, payload) + if err != nil { + return nil, err + } + prepared = append(prepared, through) + return ref, store.Prepare(through) + }, + })) + through, rooted, err := tail.flush(8) + require.ErrorContains(t, err, "commit boom at slot 7") + require.Equal(t, []uint64{6, 8}, prepared) + require.Equal(t, []uint64{6}, committer.throughs) + require.Equal(t, uint64(6), through) + require.Equal(t, uint64(6), rooted.Slot) + if restart { + store, err = epochrewards.Open(dir, 100) + require.NoError(t, err) + } + require.NoError(t, store.SetRooted(through)) + got, ok := store.Get(0, key.String(), false) + require.True(t, ok, "first fold committed reward before second fold failed") + require.Equal(t, uint64(7), got.Amount) + }) + } + } +} diff --git a/pkg/replay/inflation_reward_records_test.go b/pkg/replay/inflation_reward_records_test.go new file mode 100644 index 00000000..fb33ff38 --- /dev/null +++ b/pkg/replay/inflation_reward_records_test.go @@ -0,0 +1,328 @@ +package replay + +import ( + "bytes" + "math" + "os" + "path/filepath" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/Overclock-Validator/mithril/pkg/accountsdb" + a "github.com/Overclock-Validator/mithril/pkg/addresses" + b "github.com/Overclock-Validator/mithril/pkg/block" + "github.com/Overclock-Validator/mithril/pkg/epochrewards" + "github.com/Overclock-Validator/mithril/pkg/epochstakes" + "github.com/Overclock-Validator/mithril/pkg/features" + "github.com/Overclock-Validator/mithril/pkg/global" + "github.com/Overclock-Validator/mithril/pkg/rewards" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/Overclock-Validator/mithril/pkg/state" + bin "github.com/gagliardetto/binary" + "github.com/gagliardetto/solana-go" + "github.com/gagliardetto/solana-go/rpc" + "github.com/stretchr/testify/require" +) + +func TestInflationRewardsSurviveCompletionFoldRetirementAndRestart(t *testing.T) { + dir := t.TempDir() + store, err := epochrewards.Open(dir, 100) + require.NoError(t, err) + require.NoError(t, store.SetRooted(4)) + key := solana.PublicKey{1} + require.NoError(t, store.RecordBlock(&b.Block{Slot: 5, Epoch: 1, Rewards: []rpc.BlockReward{ + {Pubkey: key, Lamports: 7, PostBalance: 107, RewardType: rpc.RewardTypeVoting}, + }})) + committer := &fakeCommitter{durable: accounts.NewMemAccounts(), failOn: 7} + tail := asyncTestTail(committer, 5, 6, 7, 8) + statuses := NewTransactionStatusCache() + checkpointDir := t.TempDir() + require.NoError(t, tail.SetTransactionStatusCheckpointHooks(TransactionStatusCheckpointHooks{ + Capture: statuses.CaptureSnapshotThrough, + Install: func(through uint64, payload []byte) (*state.TransactionStatusCheckpointRef, error) { + ref, err := PrepareTransactionStatusCheckpoint(checkpointDir, through, payload) + if err != nil { + return nil, err + } + return ref, store.Prepare(through) + }, + })) + info := &rewards.PartitionedRewardDistributionInfo{} + var completion partitionedRewardsCompletion + completion.observeBank(info, testUnwindBankSysvars(t, 7, 50)) + job, err := tail.buildRewardsCompletionFoldJob(completion.slot) + require.NoError(t, err) + require.Error(t, runFoldJob(committer, job)) + _, visible := store.Get(0, key.String(), false) + require.False(t, visible, "a prepared reward must not precede its committed bank") + require.False(t, completion.retire(&info, store.RootedSlot())) + + committer.failOn = 0 + job, err = tail.buildRewardsCompletionFoldJob(completion.slot) + require.NoError(t, err) + require.NoError(t, runFoldJob(committer, job)) + require.NotNil(t, tail.applyFoldJob(job)) + require.NoError(t, store.SetRooted(job.through)) + require.True(t, completion.retire(&info, job.through)) + want, visible := store.Get(0, key.String(), false) + require.True(t, visible) + require.Equal(t, uint64(7), want.Amount) + + reopened, err := epochrewards.Open(dir, 100) + require.NoError(t, err) + require.NoError(t, reopened.SetRooted(job.through)) + got, visible := reopened.Get(0, key.String(), false) + require.True(t, visible) + require.Equal(t, want, got) +} + +func TestReplaceInflationRewardRecordsUsesAppliedBalanceDelta(t *testing.T) { + key := solana.NewWallet().PublicKey() + block := &b.Block{Slot: 12, Rewards: []rpc.BlockReward{ + {Pubkey: key, Lamports: 9, RewardType: rpc.RewardTypeVoting}, + {Pubkey: key, Lamports: 2, RewardType: rpc.RewardTypeFee}, + }} + parent := &accounts.Account{Key: key, Lamports: 100} + updated := &accounts.Account{Key: key, Lamports: 107} + + require.NoError(t, replaceInflationRewardRecords( + block, + []*accounts.Account{updated, nil}, + []*accounts.Account{parent, nil}, + rpc.RewardTypeVoting, + nil, + )) + require.Len(t, block.Rewards, 2) + require.Equal(t, rpc.RewardTypeFee, block.Rewards[0].RewardType) + require.Equal(t, rpc.RewardTypeVoting, block.Rewards[1].RewardType) + require.Equal(t, int64(7), block.Rewards[1].Lamports) + require.Equal(t, uint64(107), block.Rewards[1].PostBalance) +} + +func TestReplaceInflationRewardRecordsRejectsInvalidAccountPairs(t *testing.T) { + key := solana.NewWallet().PublicKey() + block := &b.Block{Slot: 12} + require.Error(t, replaceInflationRewardRecords( + block, + []*accounts.Account{{Key: key, Lamports: 99}}, + []*accounts.Account{{Key: key, Lamports: 100}}, + rpc.RewardTypeStaking, + nil, + )) + require.Error(t, replaceInflationRewardRecords( + block, + []*accounts.Account{{Key: key, Lamports: 101}}, + nil, + rpc.RewardTypeStaking, + nil, + )) +} + +func TestReplaceInflationRewardRecordsKeepsZeroVotingReward(t *testing.T) { + key := solana.NewWallet().PublicKey() + block := &b.Block{Slot: 12} + parent := &accounts.Account{Key: key, Lamports: 100} + updated := &accounts.Account{Key: key, Lamports: 100} + + require.NoError(t, replaceInflationRewardRecords( + block, + []*accounts.Account{updated}, + []*accounts.Account{parent}, + rpc.RewardTypeVoting, + nil, + )) + require.Len(t, block.Rewards, 1) + require.Zero(t, block.Rewards[0].Lamports) + require.Equal(t, uint64(100), block.Rewards[0].PostBalance) +} + +func TestReplaceInflationRewardRecordsOmitsZeroStakingReward(t *testing.T) { + key := solana.PublicKey{1} + fee := rpc.BlockReward{Pubkey: key, Lamports: 2, RewardType: rpc.RewardTypeFee} + block := &b.Block{Slot: 12, Rewards: []rpc.BlockReward{ + fee, {Pubkey: key, Lamports: 9, RewardType: rpc.RewardTypeStaking}, + }} + parent := &accounts.Account{Key: key, Lamports: 100, Data: []byte{1, 2, 3}} + updated := parent.Clone() + + require.NoError(t, replaceInflationRewardRecords(block, + []*accounts.Account{updated}, []*accounts.Account{parent}, rpc.RewardTypeStaking, nil)) + require.Equal(t, []rpc.BlockReward{fee}, block.Rewards) + require.Equal(t, parent, updated) +} + +func TestReplaceInflationRewardRecordsOmitsEpochRewardsSysvar(t *testing.T) { + stake := solana.NewWallet().PublicKey() + block := &b.Block{Slot: 12} + require.NoError(t, replaceInflationRewardRecords( + block, + []*accounts.Account{ + {Key: stake, Lamports: 105}, + {Key: sealevel.SysvarEpochRewardsAddr, Lamports: 1}, + }, + []*accounts.Account{ + {Key: stake, Lamports: 100}, + {Key: sealevel.SysvarEpochRewardsAddr, Lamports: 1}, + }, + rpc.RewardTypeStaking, + nil, + )) + require.Len(t, block.Rewards, 1) + require.Equal(t, stake, block.Rewards[0].Pubkey) + require.Equal(t, int64(5), block.Rewards[0].Lamports) +} + +func TestInflationRewardRecordsUseCalculationTimeCommission(t *testing.T) { + for _, tc := range []struct { + name string + bps uint16 + delayed bool + basisPoints bool + customCollector bool + collectorIsVote bool + disabledInflation bool + wantCommission *uint8 + wantVotingRewards uint64 + }{ + {name: "legacy", bps: 700, wantCommission: uint8Ptr(7), wantVotingRewards: 70}, + {name: "zero", wantCommission: uint8Ptr(0)}, + {name: "full commission", bps: 10_000, wantCommission: uint8Ptr(100), wantVotingRewards: 1_000}, + {name: "delayed", bps: 4_000, delayed: true, wantCommission: uint8Ptr(7), wantVotingRewards: 70}, + {name: "credits only", bps: 700, disabledInflation: true, wantCommission: uint8Ptr(7)}, + {name: "basis points", bps: 1_234, basisPoints: true, wantVotingRewards: 124}, + {name: "custom collector", bps: 700, customCollector: true, wantVotingRewards: 70}, + {name: "custom collector is vote", bps: 700, customCollector: true, collectorIsVote: true, wantVotingRewards: 70}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(dir, "accounts"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "largest_file_id"), make([]byte, 8), 0o644)) + db, err := accountsdb.OpenDb(dir) + require.NoError(t, err) + db.InitCaches() + t.Cleanup(db.CloseDb) + global.ClearPendingStakePubkeys() + t.Cleanup(global.ClearPendingStakePubkeys) + for _, epoch := range []uint64{70, 71} { + global.ClearEpochStakes(epoch) + t.Cleanup(func() { global.ClearEpochStakes(epoch) }) + } + + const slot = uint64(7_100) + vote, stake, collector := solana.PublicKey{81}, solana.PublicKey{82}, solana.PublicKey{83} + if !tc.customCollector || tc.collectorIsVote { + collector = vote + } + stakeState := sealevel.StakeStateV2{Status: sealevel.StakeStateV2StatusStake, Stake: sealevel.StakeStateV2Stake{ + Stake: sealevel.Stake{Delegation: sealevel.Delegation{ + VoterPubkey: vote, StakeLamports: 100, ActivationEpoch: math.MaxUint64, DeactivationEpoch: math.MaxUint64, + }}, + }} + var encoded bytes.Buffer + require.NoError(t, stakeState.MarshalWithEncoder(bin.NewBinEncoder(&encoded))) + stakeData := make([]byte, 200) + copy(stakeData, encoded.Bytes()) + stakeAccount := &accounts.Account{Key: stake, Lamports: 1_000, Owner: a.StakeProgramAddr, Data: stakeData} + stored := make(chan struct{}) + require.NoError(t, db.StoreAccounts([]*accounts.Account{ + stakeAccount, {Key: collector, Lamports: 1_000, Owner: a.VoteProgramAddr}, + }, slot, func() { close(stored) })) + <-stored + global.EnqueuePendingStakePubkey(slot, stake) + _, err = global.FlushPendingStakePubkeysThrough(dir, slot) + require.NoError(t, err) + db.RootedDurable = true + + f := &features.Features{} + if tc.basisPoints { + f.EnableFeature(features.CommissionRateInBasisPoints, 0) + } + if tc.customCollector { + f.EnableFeature(features.CustomCommissionCollector, 0) + } + voteState := &sealevel.VoteStateVersions{Type: sealevel.VoteStateVersionV4, V4: sealevel.VoteState4{ + InflationRewardsCommissionBps: tc.bps, InflationRewardsCollector: collector, + EpochCredits: []sealevel.EpochCredits{{Epoch: 70, Credits: 1_000}}, + }} + if tc.delayed { + f.EnableFeature(features.DelayCommissionUpdates, 0) + global.PutEpochStakes(70, map[solana.PublicKey]uint64{vote: 100}, + map[solana.PublicKey]*epochstakes.VoteAccount{vote: {InflationRewardsCommissionBps: new(uint16(700))}}, 100) + } + newRateEpoch := uint64(0) + pointValueRewards := uint64(1_000) + if tc.disabledInflation { + pointValueRewards = 0 + } + calculated, err := rewards.CalculateRewardsStreaming(db, slot, &sealevel.SysvarStakeHistory{}, &newRateEpoch, + map[solana.PublicKey]*sealevel.VoteStateVersions{vote: voteState}, rewards.PointValue{Rewards: pointValueRewards}, 70, [32]byte{}, + &sealevel.SlotCtx{Features: f}, f, &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 100}, rewards.RewardCalculationMode{ + FullAlpenglow: true, RewardEpochDelegatedStakes: map[solana.PublicKey]uint64{vote: 100}, + }) + require.NoError(t, err) + t.Cleanup(func() { rewards.CleanupPartitionedSpoolFiles(calculated.SpoolDir, slot, calculated.NumPartitions) }) + require.Equal(t, uint64(1), calculated.NumPartitions) + voting, voteParents, votingAmount := rewards.DistributeVotingRewards(db, calculated.ValidatorRewards, slot, nil) + require.Equal(t, tc.wantVotingRewards, votingAmount) + + // Distribution must use calculation-time metadata, not the current vote + // state or a stake account whose delegation changed after calculation. + voteState.V4.InflationRewardsCommissionBps = 9_900 + stakeState.Stake.Stake.Delegation.VoterPubkey = solana.PublicKey{84} + currentStake := stakeAccount.Clone() + require.NoError(t, sealevel.MarshalStakeStakeInto(&stakeState, currentStake.Data)) + staking, stakeParents, stakingAmount, burned, stakeCommissions := rewards.DistributeStakingRewardsFromSpool( + db, calculated.SpoolDir, slot, 0, slot+1, func(key solana.PublicKey) (*accounts.Account, error) { + require.Equal(t, stake, key) + return currentStake, nil + }, calculated.VoteCommissions) + require.Zero(t, burned) + wantStakingAmount, wantRecords := uint64(1_000)-tc.wantVotingRewards, 2 + if tc.disabledInflation { + wantStakingAmount, wantRecords = 0, 0 + } else if wantStakingAmount == 0 { + wantRecords = 1 + } + require.Equal(t, wantStakingAmount, stakingAmount) + require.Len(t, staking, 1) + require.Equal(t, uint64(1_000)+wantStakingAmount, staking[0].Lamports) + updatedStake, err := sealevel.UnmarshalStakeState(staking[0].Data) + require.NoError(t, err) + require.Equal(t, uint64(1_000), updatedStake.Stake.Stake.CreditsObserved) + untrustedCommission := uint8(99) + block := &b.Block{Slot: slot + 1, Epoch: 71, Rewards: []rpc.BlockReward{ + {Pubkey: collector, RewardType: rpc.RewardTypeVoting, Commission: &untrustedCommission}, + {Pubkey: stake, RewardType: rpc.RewardTypeStaking, Commission: &untrustedCommission}, + }} + require.NoError(t, replaceInflationRewardRecords(block, voting, voteParents, rpc.RewardTypeVoting, calculated.VoteCommissions)) + require.NoError(t, replaceInflationRewardRecords(block, staking, stakeParents, rpc.RewardTypeStaking, stakeCommissions)) + require.Len(t, block.Rewards, wantRecords) + for _, reward := range block.Rewards { + require.Equal(t, tc.wantCommission, reward.Commission, "reward type %s", reward.RewardType) + } + historyDir := t.TempDir() + history, err := epochrewards.Open(historyDir, 100) + require.NoError(t, err) + require.NoError(t, history.RecordBlock(block)) + history.RecordCoverage(70, block.Slot, true, true) + require.NoError(t, history.Prepare(block.Slot)) + require.NoError(t, history.SetRooted(block.Slot)) + reopened, err := epochrewards.Open(historyDir, 100) + require.NoError(t, err) + require.NoError(t, reopened.SetRooted(block.Slot)) + require.True(t, reopened.EpochAvailable(70)) + _, hasStakingReward := reopened.Get(70, stake.String(), false) + require.Equal(t, wantStakingAmount > 0, hasStakingReward) + for _, reward := range block.Rewards { + recorded, ok := reopened.Get(70, reward.Pubkey.String(), false) + require.True(t, ok, "reward type %s", reward.RewardType) + require.Equal(t, uint64(reward.Lamports), recorded.Amount) + require.Equal(t, reward.PostBalance, recorded.PostBalance) + require.Equal(t, block.Slot, recorded.EffectiveSlot) + require.Equal(t, tc.wantCommission, recorded.Commission) + } + }) + } +} + +func uint8Ptr(value uint8) *uint8 { return &value } diff --git a/pkg/replay/reward_read_view.go b/pkg/replay/reward_read_view.go new file mode 100644 index 00000000..1d5ca5a5 --- /dev/null +++ b/pkg/replay/reward_read_view.go @@ -0,0 +1,23 @@ +package replay + +import "github.com/Overclock-Validator/mithril/pkg/rewards" + +// rewardPublicationTargets separates already verified reads from recoverable +// disk state. The caller must apply execution verification and exact finality +// before passing verifiedThrough; only disk waits for every reward partition. +func (t *unrootedTail) rewardPublicationTargets(completion *partitionedRewardsCompletion, info *rewards.PartitionedRewardDistributionInfo, boundary, verifiedThrough uint64) (readThrough, durableThrough uint64) { + return t.executedSlotAtOrBefore(verifiedThrough), completion.limitPromotion(info, boundary, verifiedThrough) +} + +// executedSlotAtOrBefore selects a complete retained bank at the verified +// prefix. Skipped slots and unwound banks have no context and cannot advance an +// RPC epoch. The scan is bounded by the existing unrooted-tail halt cap. +func (t *unrootedTail) executedSlotAtOrBefore(through uint64) uint64 { + var slot uint64 + for held, ctx := range t.contexts { + if ctx != nil && ctx.Slot == held && held <= through && held > slot { + slot = held + } + } + return slot +} diff --git a/pkg/replay/reward_read_view_test.go b/pkg/replay/reward_read_view_test.go new file mode 100644 index 00000000..1b903930 --- /dev/null +++ b/pkg/replay/reward_read_view_test.go @@ -0,0 +1,37 @@ +package replay + +import ( + "testing" + + "github.com/Overclock-Validator/mithril/pkg/rewards" + "github.com/stretchr/testify/require" +) + +func TestRewardPublicationDoesNotWaitForDistribution(t *testing.T) { + tail := asyncTestTail(nil, 256, 260) + info := &rewards.PartitionedRewardDistributionInfo{NumRewardPartitionsRemaining: 1} + var completion partitionedRewardsCompletion + for _, verified := range []uint64{255, 256, 257, 260} { + read, durable := tail.rewardPublicationTargets(&completion, info, 256, verified) + require.Equal(t, tail.executedSlotAtOrBefore(verified), read) + require.Equal(t, uint64(255), durable, "disk must retain the replayable boundary while rewards are incomplete") + } + info.NumRewardPartitionsRemaining = 0 + completion.observeBank(info, testUnwindBankSysvars(t, 260, 100)) + read, durable := tail.rewardPublicationTargets(&completion, info, 256, 260) + require.Equal(t, uint64(260), read) + require.Equal(t, uint64(260), durable, "verified completion permits disk promotion") +} + +func TestRewardReadSlotUsesSurvivingExecutedBank(t *testing.T) { + tail := asyncTestTail(nil, 255, 257, 260) + require.Equal(t, uint64(255), tail.executedSlotAtOrBefore(256), "a finalized skipped epoch boundary is not a new bank") + require.Equal(t, uint64(257), tail.executedSlotAtOrBefore(259)) + require.Equal(t, uint64(260), tail.executedSlotAtOrBefore(1000)) + parent, _ := tail.unwind(257) + require.NotNil(t, parent) + require.Equal(t, uint64(255), parent.Slot) + require.Equal(t, uint64(255), tail.executedSlotAtOrBefore(1000), "finality ahead of a rewound high-water mark cannot restore discarded context") + delete(tail.contexts, 255) + require.Zero(t, tail.executedSlotAtOrBefore(1000), "a retired durable context needs no new in-memory publication") +} diff --git a/pkg/replay/rewards.go b/pkg/replay/rewards.go index d850c312..70a98beb 100644 --- a/pkg/replay/rewards.go +++ b/pkg/replay/rewards.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "fmt" + "math" "os" "path/filepath" "sort" @@ -486,6 +487,7 @@ func beginPartitionedEpochRewardsDistribution(acctsDb *accountsdb.AccountsDb, sl partitionedRewardsInfo.SpoolDir = streamResult.SpoolDir partitionedRewardsInfo.SpoolSlot = streamResult.SpoolSlot + partitionedRewardsInfo.VoteCommissions = streamResult.VoteCommissions partitionedRewardsInfo.NumRewardPartitionsRemaining = streamResult.NumPartitions maybeDumpEpochCalculatedRewards(dbgOpts, epoch, slot, streamResult) @@ -493,6 +495,9 @@ func beginPartitionedEpochRewardsDistribution(acctsDb *accountsdb.AccountsDb, sl rewardLoader := epochRewardAccountLoader(acctsDb, slot, slotCtx, stagedEpochAccts) updatedAccts, parentUpdatedAccts, voteRewardsDistributed := rewards.DistributeVotingRewards(acctsDb, streamResult.ValidatorRewards, slot, rewardLoader) + if err := replaceInflationRewardRecords(block, updatedAccts, parentUpdatedAccts, rpc.RewardTypeVoting, streamResult.VoteCommissions); err != nil { + panic(err) + } newEpochRewards := sealevel.SysvarEpochRewards{DistributionStartingBlockHeight: block.BlockHeight + 1, NumPartitions: streamResult.NumPartitions, ParentBlockhash: block.LastBlockhash, @@ -524,7 +529,7 @@ func beginPartitionedEpochRewardsDistribution(acctsDb *accountsdb.AccountsDb, sl capitalizingEpochRewards(voteRewardsDistributed, streamResult.TotalStakerRewards) } -func distributePartitionedEpochRewardsForSlot(acctsDb *accountsdb.AccountsDb, parentCtx *sealevel.SlotCtx, stagedEpochAccts []*accounts.Account, epochCtx *ReplayCtx, partitionedEpochRewardsInfo *rewards.PartitionedRewardDistributionInfo, currentSlot uint64, currentBlockHeight uint64) ([]*accounts.Account, []*accounts.Account) { +func distributePartitionedEpochRewardsForSlot(acctsDb *accountsdb.AccountsDb, parentCtx *sealevel.SlotCtx, stagedEpochAccts []*accounts.Account, epochCtx *ReplayCtx, partitionedEpochRewardsInfo *rewards.PartitionedRewardDistributionInfo, currentSlot uint64, currentBlockHeight uint64) ([]*accounts.Account, []*accounts.Account, map[solana.PublicKey]uint8) { rewardLoader := epochRewardAccountLoader(acctsDb, currentSlot, parentCtx, stagedEpochAccts) epochRewardsAcct, err := rewardLoader(sealevel.SysvarEpochRewardsAddr) if err != nil { @@ -540,12 +545,12 @@ func distributePartitionedEpochRewardsForSlot(acctsDb *accountsdb.AccountsDb, pa // FirstStakingRewardSlot while its block height is still one before the // distribution start recorded in the freshly staged EpochRewards sysvar. if currentBlockHeight < epochRewards.DistributionStartingBlockHeight { - return nil, nil + return nil, nil, nil } partitionIdx := currentBlockHeight - epochRewards.DistributionStartingBlockHeight - distributedAccts, parentDistributedAccts, distributedLamports, burnedLamports := rewards.DistributeStakingRewardsFromSpool(acctsDb, partitionedEpochRewardsInfo.SpoolDir, partitionedEpochRewardsInfo.SpoolSlot, partitionIdx, currentSlot, rewardLoader) + distributedAccts, parentDistributedAccts, distributedLamports, burnedLamports, stakeCommissions := rewards.DistributeStakingRewardsFromSpool(acctsDb, partitionedEpochRewardsInfo.SpoolDir, partitionedEpochRewardsInfo.SpoolSlot, partitionIdx, currentSlot, rewardLoader, partitionedEpochRewardsInfo.VoteCommissions) parentDistributedAccts = append(parentDistributedAccts, epochRewardsAcct.Clone()) // EpochRewards sysvar advances by distributed + burned (matching Agave/FD), @@ -583,5 +588,49 @@ func distributePartitionedEpochRewardsForSlot(acctsDb *accountsdb.AccountsDb, pa mlog.Log.Warnf("partition %d: distributed=%d burned=%d", partitionIdx, distributedLamports, burnedLamports) } - return distributedAccts, parentDistributedAccts + return distributedAccts, parentDistributedAccts, stakeCommissions +} + +func replaceInflationRewardRecords(block *block.Block, updated, parents []*accounts.Account, rewardType rpc.RewardType, commissions map[solana.PublicKey]uint8) error { + if block == nil { + return fmt.Errorf("record %s rewards: nil block", rewardType) + } + if len(updated) != len(parents) { + return fmt.Errorf("record %s rewards at slot %d: %d updated accounts, %d parents", rewardType, block.Slot, len(updated), len(parents)) + } + records := make([]rpc.BlockReward, 0, len(updated)) + for index, account := range updated { + if account == nil || account.Key == sealevel.SysvarEpochRewardsAddr { + continue + } + parent := parents[index] + if parent == nil { + return fmt.Errorf("record %s rewards at slot %d: account %s has no parent", rewardType, block.Slot, account.Key) + } + if account.Lamports < parent.Lamports { + return fmt.Errorf("record %s rewards at slot %d: account %s balance decreased from %d to %d", rewardType, block.Slot, account.Key, parent.Lamports, account.Lamports) + } + amount := account.Lamports - parent.Lamports + if amount > math.MaxInt64 { + return fmt.Errorf("record %s rewards at slot %d: account %s reward %d overflows int64", rewardType, block.Slot, account.Key, amount) + } + if rewardType == rpc.RewardTypeStaking && amount == 0 { + continue + } + var commission *uint8 + if value, ok := commissions[account.Key]; ok { + commission = &value + } + records = append(records, rpc.BlockReward{ + Pubkey: account.Key, Lamports: int64(amount), PostBalance: account.Lamports, RewardType: rewardType, Commission: commission, + }) + } + kept := block.Rewards[:0] + for _, reward := range block.Rewards { + if reward.RewardType != rewardType { + kept = append(kept, reward) + } + } + block.Rewards = append(kept, records...) + return nil } diff --git a/pkg/replay/rewards_empty_partition_test.go b/pkg/replay/rewards_empty_partition_test.go index 89a7c74f..e85b549f 100644 --- a/pkg/replay/rewards_empty_partition_test.go +++ b/pkg/replay/rewards_empty_partition_test.go @@ -79,14 +79,14 @@ func TestSameBankRewardPartitionWaitsForStartingBlockHeight(t *testing.T) { NumRewardPartitionsRemaining: 1, } replayCtx := &ReplayCtx{Capitalization: 12345} - updated, parents := distributePartitionedEpochRewardsForSlot(db, nil, []*accounts.Account{activeAccount}, replayCtx, distribution, 11, 100) + updated, parents, _ := distributePartitionedEpochRewardsForSlot(db, nil, []*accounts.Account{activeAccount}, replayCtx, distribution, 11, 100) require.Equal(t, uint64(1), distribution.NumRewardPartitionsRemaining) require.Equal(t, uint64(12345), replayCtx.Capitalization) require.Empty(t, updated) require.Empty(t, parents) - updated, parents = distributePartitionedEpochRewardsForSlot(db, nil, []*accounts.Account{activeAccount}, replayCtx, distribution, 11, 101) + updated, parents, _ = distributePartitionedEpochRewardsForSlot(db, nil, []*accounts.Account{activeAccount}, replayCtx, distribution, 11, 101) require.Zero(t, distribution.NumRewardPartitionsRemaining) require.Equal(t, uint64(12345), replayCtx.Capitalization) diff --git a/pkg/rewards/alpenglow_rewards_test.go b/pkg/rewards/alpenglow_rewards_test.go index ae9d451b..61ba328d 100644 --- a/pkg/rewards/alpenglow_rewards_test.go +++ b/pkg/rewards/alpenglow_rewards_test.go @@ -2,6 +2,7 @@ package rewards import ( "math" + "sync/atomic" "testing" "github.com/Overclock-Validator/mithril/pkg/features" @@ -88,6 +89,21 @@ func TestAlpenglowCommissionSplitPreservesFractionalLamport(t *testing.T) { }, tower) } +func TestZeroCommissionRetainsVotingRewardEntry(t *testing.T) { + votePubkey := solana.NewWallet().PublicKey() + voteState := &sealevel.VoteStateVersions{ + Type: sealevel.VoteStateVersionV4, + V4: sealevel.VoteState4{InflationRewardsCommissionBps: 0}, + } + split := voteCommissionSplit(voteState, 1_000, true, true) + require.Zero(t, split.VoterPortion) + + rewards := make(map[solana.PublicKey]*atomic.Uint64) + accumulateVotingReward(rewards, votePubkey, voteState, false, split.VoterPortion) + require.Contains(t, rewards, votePubkey) + require.Zero(t, rewards[votePubkey].Load()) +} + func TestV4CommissionKeepsBasisPointPrecision(t *testing.T) { voteState := &sealevel.VoteStateVersions{ Type: sealevel.VoteStateVersionV4, diff --git a/pkg/rewards/inactive_stakes_test.go b/pkg/rewards/inactive_stakes_test.go index 278e8c3d..25fef8ae 100644 --- a/pkg/rewards/inactive_stakes_test.go +++ b/pkg/rewards/inactive_stakes_test.go @@ -107,8 +107,8 @@ func TestEpoch116InactiveStakeBankHash(t *testing.T) { votes, PointValue{Rewards: 12922370184029}, 115, [32]byte{}, &sealevel.SlotCtx{Features: f}, f, &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 54000}, RewardCalculationMode{FullAlpenglow: true}) require.NoError(t, err) - updated, _, distributed, burned := DistributeStakingRewardsFromSpool( - db, result.SpoolDir, result.SpoolSlot, 0, 6264001, nil) + updated, _, distributed, burned, _ := DistributeStakingRewardsFromSpool( + db, result.SpoolDir, result.SpoolSlot, 0, 6264001, nil, result.VoteCommissions) require.Zero(t, distributed) require.Zero(t, burned) require.Equal(t, fixture.ExpectedBankhash, calculateHash(updated)) diff --git a/pkg/rewards/rewards.go b/pkg/rewards/rewards.go index a8df079b..20e8f74b 100644 --- a/pkg/rewards/rewards.go +++ b/pkg/rewards/rewards.go @@ -38,6 +38,7 @@ type PartitionedRewardDistributionInfo struct { NumRewardPartitionsRemaining uint64 SpoolDir string SpoolSlot uint64 + VoteCommissions map[solana.PublicKey]uint8 } type CalculatedStakePoints struct { @@ -319,13 +320,13 @@ func DistributeVotingRewards(acctsDb *accountsdb.AccountsDb, validatorRewards ma return updatedAccts, parentUpdatedAccts, totalVotingRewards.Load() } -func DistributeStakingRewardsFromSpool(acctsDb *accountsdb.AccountsDb, spoolDir string, spoolSlot uint64, partitionIdx uint64, currentSlot uint64, loader RewardAccountLoader) ([]*accounts.Account, []*accounts.Account, uint64, uint64) { +func DistributeStakingRewardsFromSpool(acctsDb *accountsdb.AccountsDb, spoolDir string, spoolSlot uint64, partitionIdx uint64, currentSlot uint64, loader RewardAccountLoader, voteCommissions map[solana.PublicKey]uint8) ([]*accounts.Account, []*accounts.Account, uint64, uint64, map[solana.PublicKey]uint8) { reader, err := NewPartitionReader(spoolDir, spoolSlot, uint32(partitionIdx)) if err != nil { panic(fmt.Sprintf("unable to open partition %d spool for distribution: %s", partitionIdx, err)) } if reader == nil { - return nil, nil, 0, 0 + return nil, nil, 0, 0, nil } defer reader.Close() @@ -342,7 +343,17 @@ func DistributeStakingRewardsFromSpool(acctsDb *accountsdb.AccountsDb, spoolDir } if len(records) == 0 { - return nil, nil, 0, 0 + return nil, nil, 0, 0, nil + } + + var stakeCommissions map[solana.PublicKey]uint8 + if voteCommissions != nil { + stakeCommissions = make(map[solana.PublicKey]uint8, len(records)) + for _, record := range records { + if commission, ok := voteCommissions[record.VotePubkey]; ok { + stakeCommissions[record.StakePubkey] = commission + } + } } accts := make([]*accounts.Account, len(records)) @@ -436,7 +447,7 @@ func DistributeStakingRewardsFromSpool(acctsDb *accountsdb.AccountsDb, spoolDir } } - return accts, parentAccts, distributedLamports.Load(), burnedLamports.Load() + return accts, parentAccts, distributedLamports.Load(), burnedLamports.Load(), stakeCommissions } func minimumStakeDelegation(slotCtx *sealevel.SlotCtx) uint64 { @@ -637,6 +648,17 @@ func voteInflationRewardsCollector(votePubkey solana.PublicKey, voteState *seale return votePubkey } +func accumulateVotingReward(validatorRewards map[solana.PublicKey]*atomic.Uint64, votePubkey solana.PublicKey, voteState *sealevel.VoteStateVersions, customCollector bool, amount uint64) { + // Zero commission still produces a vote reward record. + collector := voteInflationRewardsCollector(votePubkey, voteState, customCollector) + accumulator := validatorRewards[collector] + if accumulator == nil { + accumulator = &atomic.Uint64{} + validatorRewards[collector] = accumulator + } + accumulator.Add(amount) +} + func calculateStakePointsAndCredits( pubkey solana.PublicKey, stakeHistory *sealevel.SysvarStakeHistory, @@ -798,8 +820,11 @@ type StreamingRewardsResult struct { TotalPoints wide.Uint128 TotalStakerRewards uint64 ValidatorRewards map[solana.PublicKey]*atomic.Uint64 - NumStakeRewards uint64 - NumPartitions uint64 + // VoteCommissions records selected legacy percentages. Custom collectors + // and basis-point commissions do not populate the legacy commission field. + VoteCommissions map[solana.PublicKey]uint8 + NumStakeRewards uint64 + NumPartitions uint64 } // spoolWriteRequest is sent to the single-writer goroutine for spool writes. @@ -969,6 +994,18 @@ func CalculateRewardsStreaming( return nil, err } } + var voteCommissions map[solana.PublicKey]uint8 + if !f.IsActive(features.CustomCommissionCollector) && !f.IsActive(features.CommissionRateInBasisPoints) { + voteCommissions = make(map[solana.PublicKey]uint8, len(voteCache)) + for votePubkey, selected := range voteCache { + if historical := commissionVoteStates[votePubkey]; historical != nil { + selected = historical + } + if selected != nil { + voteCommissions[votePubkey] = uint8(voteInflationCommissionBPS(selected, false) / 100) + } + } + } tempWriter, err := NewTempSpoolWriter(spoolDir, slot) if err != nil { @@ -1081,17 +1118,8 @@ func CalculateRewardsStreaming( return nil, fmt.Errorf("temp spool write failed: %w", err) } - if splitResult.VoterPortion > 0 { - collector := voteInflationRewardsCollector( - rec.VotePubkey, voteState, f.IsActive(features.CustomCommissionCollector), - ) - accumulator := validatorRewards[collector] - if accumulator == nil { - accumulator = &atomic.Uint64{} - validatorRewards[collector] = accumulator - } - accumulator.Add(splitResult.VoterPortion) - } + accumulateVotingReward(validatorRewards, rec.VotePubkey, voteState, + f.IsActive(features.CustomCommissionCollector), splitResult.VoterPortion) } pointsReader.Close() @@ -1168,6 +1196,7 @@ func CalculateRewardsStreaming( TotalPoints: reportedPoints, TotalStakerRewards: phase2TotalStakerRewards, ValidatorRewards: validatorRewards, + VoteCommissions: voteCommissions, NumStakeRewards: actualRewardCount, NumPartitions: numPartitions, }, nil diff --git a/pkg/rpcserver/errors.go b/pkg/rpcserver/errors.go index ee2a1654..7e8d3893 100644 --- a/pkg/rpcserver/errors.go +++ b/pkg/rpcserver/errors.go @@ -15,8 +15,88 @@ const ( rpcCodeSendTransactionPreflightFailure jsonrpc.ErrorCode = -32002 // -32016 is Agave's reserved code for MinContextSlotNotReached. rpcCodeMinContextSlotNotReached jsonrpc.ErrorCode = -32016 + rpcCodeBlockNotAvailable jsonrpc.ErrorCode = -32004 + rpcCodeBlockCleanedUp jsonrpc.ErrorCode = -32001 + rpcCodeEpochRewardsPeriodActive jsonrpc.ErrorCode = -32017 ) +// EpochRewardHistoryPrunedError identifies requests outside retained history. +type EpochRewardHistoryPrunedError struct { + Message string +} + +func (e *EpochRewardHistoryPrunedError) Error() string { + return e.Message +} + +func (e *EpochRewardHistoryPrunedError) ToJSONRPCError() (jsonrpc.JSONRPCError, error) { + return jsonrpc.JSONRPCError{Code: rpcCodeBlockCleanedUp, Message: e.Error()}, nil +} + +func (e *EpochRewardHistoryPrunedError) FromJSONRPCError(rpcErr jsonrpc.JSONRPCError) error { + if rpcErr.Code != rpcCodeBlockCleanedUp { + return fmt.Errorf("unexpected code %d for EpochRewardHistoryPrunedError", rpcErr.Code) + } + e.Message = rpcErr.Message + return nil +} + +// EpochRewardsPeriodActiveError reports the verified reward-distribution progress. +type EpochRewardsPeriodActiveError struct { + Slot uint64 `json:"slot"` + CurrentBlockHeight uint64 `json:"currentBlockHeight"` + RewardsCompleteBlockHeight uint64 `json:"rewardsCompleteBlockHeight"` +} + +func (e *EpochRewardsPeriodActiveError) Error() string { + return fmt.Sprintf("Epoch rewards period still active at slot %d", e.Slot) +} + +func (e *EpochRewardsPeriodActiveError) ToJSONRPCError() (jsonrpc.JSONRPCError, error) { + return jsonrpc.JSONRPCError{Code: rpcCodeEpochRewardsPeriodActive, Message: e.Error(), Data: e}, nil +} + +func (e *EpochRewardsPeriodActiveError) FromJSONRPCError(rpcErr jsonrpc.JSONRPCError) error { + if rpcErr.Code != rpcCodeEpochRewardsPeriodActive { + return fmt.Errorf("unexpected code %d for EpochRewardsPeriodActiveError", rpcErr.Code) + } + raw, err := json.Marshal(rpcErr.Data) + if err != nil { + return err + } + return json.Unmarshal(raw, e) +} + +var ( + _ jsonrpc.RPCErrorCodec = (*EpochRewardHistoryPrunedError)(nil) + _ jsonrpc.RPCErrorCodec = (*EpochRewardsPeriodActiveError)(nil) +) + +// EpochRewardHistoryUnavailableError distinguishes unknown history from no reward. +type EpochRewardHistoryUnavailableError struct { + Epoch uint64 `json:"epoch"` + ContextSlot uint64 `json:"contextSlot"` +} + +func (e *EpochRewardHistoryUnavailableError) Error() string { + return fmt.Sprintf("Epoch reward history is unavailable for epoch %d", e.Epoch) +} + +func (e *EpochRewardHistoryUnavailableError) ToJSONRPCError() (jsonrpc.JSONRPCError, error) { + return jsonrpc.JSONRPCError{Code: rpcCodeBlockNotAvailable, Message: e.Error(), Data: e}, nil +} + +func (e *EpochRewardHistoryUnavailableError) FromJSONRPCError(rpcErr jsonrpc.JSONRPCError) error { + if rpcErr.Code != rpcCodeBlockNotAvailable { + return fmt.Errorf("unexpected code %d for EpochRewardHistoryUnavailableError", rpcErr.Code) + } + raw, err := json.Marshal(rpcErr.Data) + if err != nil { + return err + } + return json.Unmarshal(raw, e) +} + type MinContextSlotNotReachedError struct { ContextSlot uint64 } diff --git a/pkg/rpcserver/errors_test.go b/pkg/rpcserver/errors_test.go index 6967eb7d..0fa33f45 100644 --- a/pkg/rpcserver/errors_test.go +++ b/pkg/rpcserver/errors_test.go @@ -81,3 +81,20 @@ func TestInvalidParamsError_FromJSONRPCError(t *testing.T) { require.NoError(t, e.FromJSONRPCError(jsonrpc.JSONRPCError{Code: -32602, Message: "x"})) assert.Equal(t, "x", e.Message) } + +func TestEpochRewardHistoryUnavailableError(t *testing.T) { + want := &EpochRewardHistoryUnavailableError{Epoch: 7, ContextSlot: 1_024} + rpcErr, err := want.ToJSONRPCError() + require.NoError(t, err) + raw, err := json.Marshal(rpcErr) + require.NoError(t, err) + require.Contains(t, string(raw), `"code":-32004`) + require.Contains(t, string(raw), `"data":{"epoch":7,"contextSlot":1024}`) + var wire jsonrpc.JSONRPCError + require.NoError(t, json.Unmarshal(raw, &wire)) + var got EpochRewardHistoryUnavailableError + require.NoError(t, got.FromJSONRPCError(wire)) + require.Equal(t, *want, got) + require.Error(t, got.FromJSONRPCError(jsonrpc.JSONRPCError{Code: -32016})) + require.Error(t, got.FromJSONRPCError(jsonrpc.JSONRPCError{Code: -32004, Data: map[string]any{"epoch": "invalid"}})) +} diff --git a/pkg/rpcserver/get_inflation_reward.go b/pkg/rpcserver/get_inflation_reward.go new file mode 100644 index 00000000..77db1aef --- /dev/null +++ b/pkg/rpcserver/get_inflation_reward.go @@ -0,0 +1,125 @@ +package rpcserver + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/Overclock-Validator/mithril/pkg/rewards" + "github.com/Overclock-Validator/mithril/pkg/safemath" + "github.com/filecoin-project/go-jsonrpc" + "github.com/gagliardetto/solana-go" +) + +const maxInflationRewardAddresses = 32 + +type inflationRewardConfig struct { + Commitment string `json:"commitment"` + Epoch *uint64 `json:"epoch"` + MinContextSlot *uint64 `json:"minContextSlot"` +} + +// InflationRewardResp is one getInflationReward result entry. +type InflationRewardResp struct { + Epoch uint64 `json:"epoch"` + EffectiveSlot uint64 `json:"effectiveSlot"` + Amount uint64 `json:"amount"` + PostBalance uint64 `json:"postBalance"` + Commission *uint8 `json:"commission"` +} + +// GetInflationReward reads verified canonical rewards. On Alpenglow, confirmed +// and finalized share this view, which may precede the durable checkpoint. +// The recovery lock pins its slot, epoch, coverage and records for this request. +func (rpcServer *RpcServer) GetInflationReward(ctx context.Context, p jsonrpc.RawParams) ([]*InflationRewardResp, error) { + rpcServer.historyRecoveryMu.RLock() + defer rpcServer.historyRecoveryMu.RUnlock() + if rpcServer.historyRecoveryPending { + return nil, fmt.Errorf("RPC history is unavailable during account recovery") + } + if err := ctx.Err(); err != nil { + return nil, err + } + var params []json.RawMessage + if err := json.Unmarshal(p, ¶ms); err != nil || len(params) < 1 || len(params) > 2 { + return nil, &InvalidParamsError{Message: "getInflationReward requires addresses and optional config"} + } + var addresses []string + if err := json.Unmarshal(params[0], &addresses); err != nil || addresses == nil { + return nil, &InvalidParamsError{Message: "invalid getInflationReward addresses"} + } + if len(addresses) > maxInflationRewardAddresses { + return nil, &InvalidParamsError{Message: fmt.Sprintf("Too many inputs provided; max %d", maxInflationRewardAddresses)} + } + for _, address := range addresses { + if _, err := solana.PublicKeyFromBase58(address); err != nil { + return nil, &InvalidParamsError{Message: fmt.Sprintf("invalid address %q", address)} + } + } + + config := inflationRewardConfig{Commitment: "finalized"} + if len(params) == 2 && string(params[1]) != "null" { + if err := json.Unmarshal(params[1], &config); err != nil { + return nil, &InvalidParamsError{Message: "invalid getInflationReward config"} + } + if config.Commitment == "" { + config.Commitment = "finalized" + } + } + if config.Commitment != "confirmed" && config.Commitment != "finalized" { + return nil, &InvalidParamsError{Message: "getInflationReward requires confirmed or finalized commitment"} + } + if rpcServer.epochRewards == nil { + return nil, fmt.Errorf("node has no retained epoch reward history") + } + if rpcServer.epochSchedule == nil { + return nil, fmt.Errorf("node has no epoch schedule available") + } + + contextSlot := rpcServer.epochRewards.PublishedSlot() + if config.MinContextSlot != nil && contextSlot < *config.MinContextSlot { + return nil, &MinContextSlotNotReachedError{ContextSlot: contextSlot} + } + epoch := rpcServer.epochSchedule.GetEpoch(contextSlot) + if config.Epoch != nil { + epoch = *config.Epoch + } else if epoch > 0 { + epoch-- + } + + result := make([]*InflationRewardResp, len(addresses)) + available := rpcServer.epochRewards.EpochPublished(epoch) + distribution, distributing := rpcServer.epochRewards.PublishedDistribution(epoch) + var unavailable error = &EpochRewardHistoryUnavailableError{Epoch: epoch, ContextSlot: contextSlot} + boundary := rpcServer.epochSchedule.FirstSlotInEpoch(safemath.SaturatingAddU64(epoch, 1)) + if boundary < rpcServer.epochRewards.FirstRetainedSlot() { + unavailable = &EpochRewardHistoryPrunedError{Message: fmt.Sprintf("Reward history for slot %d has been cleaned up", boundary)} + } + if len(addresses) == 0 && !available && !distributing { + return nil, unavailable + } + for index, address := range addresses { + record, ok := rpcServer.epochRewards.GetPublished(epoch, address) + if !ok { + if !available { + if !distributing { + return nil, unavailable + } + partition := rewards.CalculateRewardPartitionForPubkey(solana.MustPublicKeyFromBase58(address), distribution.ParentBlockhash, distribution.NumPartitions) + partitionHeight := safemath.SaturatingAddU64(distribution.StartingBlockHeight, partition) + if distribution.BlockHeight < partitionHeight { + return nil, &EpochRewardsPeriodActiveError{ + Slot: contextSlot, CurrentBlockHeight: distribution.BlockHeight, + RewardsCompleteBlockHeight: safemath.SaturatingAddU64(distribution.StartingBlockHeight, distribution.NumPartitions), + } + } + } + continue + } + result[index] = &InflationRewardResp{ + Epoch: record.Epoch, EffectiveSlot: record.EffectiveSlot, + Amount: record.Amount, PostBalance: record.PostBalance, Commission: record.Commission, + } + } + return result, nil +} diff --git a/pkg/rpcserver/get_inflation_reward_test.go b/pkg/rpcserver/get_inflation_reward_test.go new file mode 100644 index 00000000..eaaadcf6 --- /dev/null +++ b/pkg/rpcserver/get_inflation_reward_test.go @@ -0,0 +1,308 @@ +package rpcserver + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/Overclock-Validator/mithril/pkg/addresses" + b "github.com/Overclock-Validator/mithril/pkg/block" + "github.com/Overclock-Validator/mithril/pkg/epochrewards" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/filecoin-project/go-jsonrpc" + bin "github.com/gagliardetto/binary" + "github.com/gagliardetto/solana-go" + "github.com/gagliardetto/solana-go/rpc" + "github.com/stretchr/testify/require" +) + +func TestGetInflationRewardCommitmentAndDefaults(t *testing.T) { + store, err := epochrewards.Open(t.TempDir(), 512) + require.NoError(t, err) + server := &RpcServer{ + epochSchedule: &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 128}, + slotCtx: &sealevel.SlotCtx{Slot: 260}, + epochRewards: store, + } + address := solana.NewWallet().PublicKey() + missing := solana.NewWallet().PublicKey() + require.NoError(t, server.RecordEpochRewards(&b.Block{ + Slot: 256, Epoch: 2, + Rewards: []rpc.BlockReward{{ + Pubkey: address, Lamports: 42, PostBalance: 1_042, RewardType: rpc.RewardTypeVoting, + }}, + })) + + confirmed, err := server.GetInflationReward(context.Background(), rawParams(t, []any{ + []string{address.String(), missing.String()}, map[string]any{"commitment": "confirmed"}, + })) + require.Error(t, err, "executed rewards do not establish complete rooted history") + require.Nil(t, confirmed) + + finalized, err := server.GetInflationReward(context.Background(), rawParams(t, []any{ + []string{address.String()}, map[string]any{"commitment": "finalized", "epoch": 1}, + })) + require.Error(t, err) + require.Nil(t, finalized) + _, err = server.GetInflationReward(context.Background(), rawParams(t, []any{ + []string{address.String()}, map[string]any{"commitment": "finalized", "epoch": 1, "minContextSlot": 1}, + })) + var contextErr *MinContextSlotNotReachedError + require.ErrorAs(t, err, &contextErr) + require.Zero(t, contextErr.ContextSlot) + + require.NoError(t, server.PrepareEpochRewards(256)) + require.NoError(t, server.SetRootedEpochRewardsSlot(256)) + finalized, err = server.GetInflationReward(context.Background(), rawParams(t, []any{ + []string{address.String()}, map[string]any{"epoch": 1}, + })) + require.NoError(t, err) + require.Equal(t, &InflationRewardResp{Epoch: 1, EffectiveSlot: 256, Amount: 42, PostBalance: 1_042}, finalized[0]) + confirmed, err = server.GetInflationReward(context.Background(), rawParams(t, []any{ + []string{address.String()}, map[string]any{"commitment": "confirmed"}, + })) + require.NoError(t, err) + require.Equal(t, finalized, confirmed) + _, err = server.GetInflationReward(context.Background(), rawParams(t, []any{ + []string{address.String()}, map[string]any{"commitment": "confirmed", "minContextSlot": 260}, + })) + require.ErrorAs(t, err, &contextErr) + require.Equal(t, uint64(256), contextErr.ContextSlot) +} + +func TestGetInflationRewardRejectsInvalidRequests(t *testing.T) { + store, err := epochrewards.Open(t.TempDir(), 512) + require.NoError(t, err) + server := &RpcServer{ + epochSchedule: &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 128}, + slotCtx: &sealevel.SlotCtx{Slot: 260}, + epochRewards: store, + } + address := solana.NewWallet().PublicKey().String() + tooMany := make([]string, 33) + for index := range tooMany { + tooMany[index] = address + } + + tests := []struct { + name string + params []any + typeOf any + }{ + {name: "null addresses", params: []any{nil}, typeOf: &InvalidParamsError{}}, + {name: "invalid address", params: []any{[]string{"not-a-key"}}, typeOf: &InvalidParamsError{}}, + {name: "processed commitment", params: []any{[]string{address}, map[string]any{"commitment": "processed"}}, typeOf: &InvalidParamsError{}}, + {name: "minimum context", params: []any{[]string{address}, map[string]any{"minContextSlot": 261}}, typeOf: &MinContextSlotNotReachedError{}}, + {name: "too many addresses", params: []any{tooMany}, typeOf: &InvalidParamsError{}}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := server.GetInflationReward(context.Background(), rawParams(t, test.params)) + require.Error(t, err) + switch test.typeOf.(type) { + case *InvalidParamsError: + require.IsType(t, &InvalidParamsError{}, err) + case *MinContextSlotNotReachedError: + require.IsType(t, &MinContextSlotNotReachedError{}, err) + } + }) + } +} + +func TestGetInflationRewardAfterSnapshotBeforeReplay(t *testing.T) { + store, err := epochrewards.Open(t.TempDir(), 512) + require.NoError(t, err) + server := &RpcServer{ + epochSchedule: &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 128}, + epochRewards: store, + } + address := solana.NewWallet().PublicKey() + require.NoError(t, server.RecordEpochRewards(&b.Block{ + Slot: 256, Epoch: 2, + Rewards: []rpc.BlockReward{{ + Pubkey: address, Lamports: 42, PostBalance: 1_042, RewardType: rpc.RewardTypeVoting, + }}, + })) + require.NoError(t, server.PrepareEpochRewards(256)) + require.NoError(t, server.SetRootedEpochRewardsSlot(256)) + + result, err := server.GetInflationReward(context.Background(), rawParams(t, []any{ + []string{address.String()}, map[string]any{"epoch": 1, "minContextSlot": 256}, + })) + require.NoError(t, err) + require.Equal(t, &InflationRewardResp{Epoch: 1, EffectiveSlot: 256, Amount: 42, PostBalance: 1_042}, result[0]) +} + +func TestInflationRewardNullCommissionJSON(t *testing.T) { + encoded, err := json.Marshal(InflationRewardResp{Epoch: 1, EffectiveSlot: 256, Amount: 0, PostBalance: 1_000}) + require.NoError(t, err) + require.JSONEq(t, `{"epoch":1,"effectiveSlot":256,"amount":0,"postBalance":1000,"commission":null}`, string(encoded)) +} + +func TestGetInflationRewardAddressLimitCompatibility(t *testing.T) { + store, err := epochrewards.Open(t.TempDir(), 512) + require.NoError(t, err) + server := &RpcServer{epochSchedule: &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 128}, epochRewards: store} + address := solana.PublicKey{1}.String() + require.NoError(t, server.RecordEpochRewards(&b.Block{Slot: 256, ParentSlot: 255, Epoch: 2})) + bank, err := sealevel.NewBankSysvars(256) + require.NoError(t, err) + server.RecordEpochRewardStatus(&b.Block{Slot: 256, ParentSlot: 255, Epoch: 2}, bank, false) + require.NoError(t, server.PrepareEpochRewards(256)) + require.NoError(t, server.SetRootedEpochRewardsSlot(256)) + for _, count := range []int{6, 32, 33} { + t.Run(fmt.Sprint(count), func(t *testing.T) { + addresses := make([]string, count) + for index := range addresses { + addresses[index] = address + } + result, err := server.GetInflationReward(context.Background(), rawParams(t, []any{addresses})) + if count > 32 { + require.IsType(t, &InvalidParamsError{}, err) + return + } + require.NoError(t, err) + require.Len(t, result, count) + }) + } +} + +func TestGetInflationRewardUnavailableEpochs(t *testing.T) { + store, err := epochrewards.Open(t.TempDir(), 128) + require.NoError(t, err) + server := &RpcServer{epochSchedule: &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 128}, epochRewards: store} + address := solana.PublicKey{1} + require.NoError(t, server.RecordEpochRewards(&b.Block{Slot: 256, ParentSlot: 255, Epoch: 2, + Rewards: []rpc.BlockReward{{Pubkey: address, Lamports: 7, PostBalance: 107, RewardType: rpc.RewardTypeVoting}}, + })) + require.NoError(t, server.PrepareEpochRewards(256)) + require.NoError(t, server.SetRootedEpochRewardsSlot(256)) + t.Run("future", func(t *testing.T) { + _, err := server.GetInflationReward(context.Background(), rawParams(t, []any{[]string{address.String()}, map[string]any{"epoch": 3}})) + require.Error(t, err, "an unavailable reward epoch must not look like an account with no reward") + }) + t.Run("pruned", func(t *testing.T) { + require.NoError(t, server.SetRootedEpochRewardsSlot(512)) + _, err := server.GetInflationReward(context.Background(), rawParams(t, []any{[]string{address.String()}, map[string]any{"epoch": 1}})) + require.Error(t, err, "pruned reward history must not look like an account with no reward") + }) +} + +func TestEpochRewardAvailabilityHTTP(t *testing.T) { + server := NewRpcServer(nil, 0, &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 128}, solana.Hash{1}) + t.Cleanup(func() { require.NoError(t, server.listener.Close()) }) + dir := t.TempDir() + require.NoError(t, server.EnableEpochRewards(dir, 512, 0)) + httpServer := httptest.NewServer(server) + t.Cleanup(httpServer.Close) + client := &http.Client{Timeout: 5 * time.Second} + address, missing := solana.PublicKey{1}, solana.PublicKey{2} + call := func(method string, params []any) (json.RawMessage, *jsonrpc.JSONRPCError) { + t.Helper() + body, err := json.Marshal(map[string]any{"jsonrpc": "2.0", "id": 1, "method": method, "params": params}) + require.NoError(t, err) + response, err := client.Post(httpServer.URL, "application/json", bytes.NewReader(body)) + require.NoError(t, err) + defer response.Body.Close() + var reply struct { + Result json.RawMessage `json:"result"` + Error *jsonrpc.JSONRPCError `json:"error"` + } + require.NoError(t, json.NewDecoder(response.Body).Decode(&reply)) + return reply.Result, reply.Error + } + block := &b.Block{Slot: 256, ParentSlot: 255, Epoch: 2, BlockHeight: 100, Rewards: []rpc.BlockReward{ + {Pubkey: address, Lamports: 7, PostBalance: 107, RewardType: rpc.RewardTypeVoting}, + }} + require.NoError(t, server.RecordEpochRewards(block)) + server.RecordEpochRewardStatus(block, rewardDistributionBank(t, 256, sealevel.SysvarEpochRewards{ + Active: true, DistributionStartingBlockHeight: 101, NumPartitions: 1, + }), true) + require.NoError(t, server.PrepareEpochRewards(256)) + require.NoError(t, server.SetRootedEpochRewardsSlot(256)) + _, rpcErr := call("getInflationReward", []any{[]string{missing.String()}, map[string]any{"epoch": 1}}) + require.NotNil(t, rpcErr) + require.Equal(t, jsonrpc.ErrorCode(-32017), rpcErr.Code) + require.Equal(t, "Epoch rewards period still active at slot 256", rpcErr.Message) + require.Equal(t, map[string]any{"slot": float64(256), "currentBlockHeight": float64(100), "rewardsCompleteBlockHeight": float64(102)}, rpcErr.Data) + _, rpcErr = call("getInflationReward", []any{[]string{address.String(), missing.String()}, map[string]any{"epoch": 1}}) + require.NotNil(t, rpcErr, "a retained legacy record does not certify a missing address") + result, rpcErr := call("getInflationReward", []any{[]string{address.String()}, map[string]any{"epoch": 1}}) + require.Nil(t, rpcErr, "known committed entries remain readable before complete coverage") + require.Contains(t, string(result), `"amount":7`) + + completed := &b.Block{Slot: 257, ParentSlot: 256, Epoch: 2} + require.NoError(t, server.RecordEpochRewards(completed)) + server.RecordEpochRewardStatus(completed, rewardCoverageBank(t, 257, false), true) + require.NoError(t, server.PrepareEpochRewards(257)) + require.False(t, server.epochRewards.EpochAvailable(1)) + require.NoError(t, server.SetRootedEpochRewardsSlot(257)) + for _, commitment := range []string{"confirmed", "finalized"} { + result, rpcErr = call("getInflationReward", []any{[]string{address.String(), missing.String()}, map[string]any{"commitment": commitment}}) + require.Nil(t, rpcErr) + var rewards []*InflationRewardResp + require.NoError(t, json.Unmarshal(result, &rewards)) + require.Len(t, rewards, 2) + require.Equal(t, uint64(7), rewards[0].Amount) + require.Nil(t, rewards[1]) + } + require.NoError(t, server.EnableEpochRewards(dir, 512, 257)) + require.True(t, server.epochRewards.EpochAvailable(1)) + _, rpcErr = call("getInflationReward", []any{[]string{missing.String()}, map[string]any{"minContextSlot": 258}}) + require.NotNil(t, rpcErr) + require.Equal(t, jsonrpc.ErrorCode(-32016), rpcErr.Code) + _, rpcErr = call("recordEpochRewardStatus", []any{}) + require.NotNil(t, rpcErr) + require.Equal(t, jsonrpc.ErrorCode(-32601), rpcErr.Code) + require.NoError(t, server.SetRootedEpochRewardsSlot(1024)) + _, rpcErr = call("getInflationReward", []any{[]string{missing.String()}, map[string]any{"epoch": 1}}) + require.NotNil(t, rpcErr) + require.Equal(t, jsonrpc.ErrorCode(-32001), rpcErr.Code) +} + +func TestEpochRewardStatusRequiresObservedBoundaryAndSysvar(t *testing.T) { + store, err := epochrewards.Open(t.TempDir(), 512) + require.NoError(t, err) + server := &RpcServer{epochSchedule: &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 128}, epochRewards: store} + server.RecordEpochRewardStatus(&b.Block{Slot: 300, ParentSlot: 299, Epoch: 2}, rewardCoverageBank(t, 300, false), true) + require.NoError(t, store.Prepare(300)) + require.NoError(t, store.SetRooted(300)) + require.False(t, store.EpochAvailable(1), "a mid-epoch snapshot cannot establish missing reward history") + boundary := &b.Block{Slot: 384, ParentSlot: 383, Epoch: 3} + server.RecordEpochRewardStatus(boundary, nil, true) + missingSysvar, err := sealevel.NewBankSysvars(385) + require.NoError(t, err) + server.RecordEpochRewardStatus(&b.Block{Slot: 385, ParentSlot: 384, Epoch: 3}, missingSysvar, true) + server.RecordEpochRewardStatus(&b.Block{Slot: 386, ParentSlot: 385, Epoch: 3}, rewardCoverageBank(t, 387, false), true) + require.NoError(t, store.Prepare(386)) + require.NoError(t, store.SetRooted(386)) + require.False(t, store.EpochAvailable(2), "missing or wrong-bank sysvars cannot complete an epoch") +} + +func rewardCoverageBank(t *testing.T, slot uint64, active bool) *sealevel.BankSysvars { + return rewardDistributionBank(t, slot, sealevel.SysvarEpochRewards{Active: active}) +} + +func rewardDistributionBank(t *testing.T, slot uint64, value sealevel.SysvarEpochRewards) *sealevel.BankSysvars { + t.Helper() + var encoded bytes.Buffer + require.NoError(t, value.MarshalWithEncoder(bin.NewBinEncoder(&encoded))) + bank, err := sealevel.NewBankSysvars(slot, &accounts.Account{ + Key: sealevel.SysvarEpochRewardsAddr, Lamports: 1, Owner: addresses.SysvarOwnerAddr, Data: encoded.Bytes(), + }) + require.NoError(t, err) + return bank +} + +func rawParams(t *testing.T, value any) jsonrpc.RawParams { + t.Helper() + encoded, err := json.Marshal(value) + require.NoError(t, err) + return jsonrpc.RawParams(encoded) +} diff --git a/pkg/rpcserver/history_recovery_test.go b/pkg/rpcserver/history_recovery_test.go new file mode 100644 index 00000000..ad4dac1a --- /dev/null +++ b/pkg/rpcserver/history_recovery_test.go @@ -0,0 +1,114 @@ +package rpcserver + +import ( + "reflect" + "runtime" + "testing" + "time" + + b "github.com/Overclock-Validator/mithril/pkg/block" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/gagliardetto/solana-go" + "github.com/gagliardetto/solana-go/rpc" + "github.com/stretchr/testify/require" +) + +func TestHistoryMutationsWaitForReaders(t *testing.T) { + var disabled *RpcServer + disabled.SetHistoryRecoveryPending(true) + disabled.SetHistoryRecoveryPending(false) + + _, registered := reflect.TypeOf(rpcHandler{}).MethodByName("SetHistoryRecoveryPending") + require.False(t, registered, "the internal recovery gate must not be an RPC method") + + tests := []struct { + name string + mutate func(*RpcServer) error + }{ + {name: "recovery", mutate: func(server *RpcServer) error { + server.SetHistoryRecoveryPending(true) + return nil + }}, + {name: "prepare", mutate: func(server *RpcServer) error { return server.PrepareEpochRewards(3) }}, + {name: "publish", mutate: func(server *RpcServer) error { return server.SetRootedEpochRewardsSlot(2) }}, + {name: "rewind", mutate: func(server *RpcServer) error { return server.RewindEpochRewards(2) }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + server := &RpcServer{} + require.NoError(t, server.EnableEpochRewards(t.TempDir(), 100, 0)) + require.NoError(t, server.RecordEpochRewards(&b.Block{Slot: 2, Epoch: 1, Rewards: []rpc.BlockReward{{ + Pubkey: solana.PublicKey{1}, Lamports: 1, PostBalance: 2, RewardType: rpc.RewardTypeVoting, + }}})) + require.NoError(t, server.PrepareEpochRewards(2)) + server.historyRecoveryMu.RLock() + readerHeld := true + defer func() { + if readerHeld { + server.historyRecoveryMu.RUnlock() + } + }() + done := make(chan error, 1) + go func() { done <- test.mutate(server) }() + + // Observe the waiting writer without releasing the in-flight reader. + timeout := time.After(5 * time.Second) + for server.historyRecoveryMu.TryRLock() { + server.historyRecoveryMu.RUnlock() + select { + case err := <-done: + require.NoError(t, err) + t.Fatal("history mutation finished before the in-flight reader") + case <-timeout: + t.Fatal("history mutation did not wait for the in-flight reader") + default: + runtime.Gosched() + } + } + require.Zero(t, server.epochRewards.RootedSlot()) + select { + case <-done: + t.Fatal("history mutation finished before the in-flight reader") + default: + } + server.historyRecoveryMu.RUnlock() + readerHeld = false + select { + case err := <-done: + require.NoError(t, err) + case <-timeout: + t.Fatal("history mutation remained blocked after the reader finished") + } + require.Equal(t, test.name == "recovery", server.historyRecoveryPending) + server.SetHistoryRecoveryPending(false) + require.False(t, server.historyRecoveryPending) + }) + } +} + +func TestEpochRewardHistoryRecoveryGate(t *testing.T) { + server := &RpcServer{epochSchedule: &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 2}} + require.NoError(t, server.EnableEpochRewards(t.TempDir(), 100, 0)) + address := solana.PublicKey{1} + require.NoError(t, server.RecordEpochRewards(&b.Block{ + Slot: 2, Epoch: 1, + Rewards: []rpc.BlockReward{{ + Pubkey: address, Lamports: 1, PostBalance: 2, RewardType: rpc.RewardTypeVoting, + }}, + })) + require.NoError(t, server.PrepareEpochRewards(2)) + require.NoError(t, server.SetRootedEpochRewardsSlot(2)) + params := rawParams(t, []any{[]string{address.String()}, map[string]any{"epoch": 0}}) + + for _, pending := range []bool{false, true, false} { + server.SetHistoryRecoveryPending(pending) + rewards, err := server.GetInflationReward(t.Context(), params) + if pending { + require.EqualError(t, err, "RPC history is unavailable during account recovery") + require.Nil(t, rewards) + continue + } + require.NoError(t, err) + require.Equal(t, []*InflationRewardResp{{Epoch: 0, EffectiveSlot: 2, Amount: 1, PostBalance: 2}}, rewards) + } +} diff --git a/pkg/rpcserver/inflation_availability_reason_test.go b/pkg/rpcserver/inflation_availability_reason_test.go new file mode 100644 index 00000000..c6092a73 --- /dev/null +++ b/pkg/rpcserver/inflation_availability_reason_test.go @@ -0,0 +1,164 @@ +package rpcserver + +import ( + "context" + "encoding/json" + "math" + "testing" + + b "github.com/Overclock-Validator/mithril/pkg/block" + "github.com/Overclock-Validator/mithril/pkg/rewards" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/filecoin-project/go-jsonrpc" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func TestInflationRewardUnavailableReasons(t *testing.T) { + for _, tc := range []struct { + name string + root uint64 + epoch uint64 + code jsonrpc.ErrorCode + }{ + {"distribution", 256, 1, -32017}, + {"pruned", 1024, 1, -32001}, + {"future", 256, 3, -32004}, + {"maximum epoch", 256, math.MaxUint64, -32004}, + } { + t.Run(tc.name, func(t *testing.T) { + server := &RpcServer{epochSchedule: &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 128}} + require.NoError(t, server.EnableEpochRewards(t.TempDir(), 512, 255)) + block := &b.Block{Slot: 256, ParentSlot: 255, Epoch: 2, BlockHeight: 100} + server.RecordEpochRewardStatus(block, rewardDistributionBank(t, 256, sealevel.SysvarEpochRewards{ + Active: true, DistributionStartingBlockHeight: 101, NumPartitions: 3, + }), true) + require.NoError(t, server.PrepareEpochRewards(256)) + require.NoError(t, server.SetRootedEpochRewardsSlot(tc.root)) + _, err := server.GetInflationReward(context.Background(), rawParams(t, []any{[]string{solana.PublicKey{2}.String()}, map[string]any{"epoch": tc.epoch}})) + require.Error(t, err) + converter, ok := err.(interface { + ToJSONRPCError() (jsonrpc.JSONRPCError, error) + }) + require.True(t, ok) + wire, err := converter.ToJSONRPCError() + require.NoError(t, err) + require.Equal(t, tc.code, wire.Code) + if tc.code == -32017 { + data, err := json.Marshal(wire.Data) + require.NoError(t, err) + require.JSONEq(t, `{"slot":256,"currentBlockHeight":100,"rewardsCompleteBlockHeight":104}`, string(data)) + } + }) + } +} + +func TestInflationRewardErrorCodec(t *testing.T) { + for _, original := range []jsonrpc.RPCErrorCodec{ + &EpochRewardHistoryPrunedError{Message: "Reward history for slot 256 has been cleaned up"}, + &EpochRewardsPeriodActiveError{Slot: 256, CurrentBlockHeight: 100, RewardsCompleteBlockHeight: 104}, + } { + wire, err := original.ToJSONRPCError() + require.NoError(t, err) + encoded, err := json.Marshal(wire) + require.NoError(t, err) + var decoded jsonrpc.JSONRPCError + require.NoError(t, json.Unmarshal(encoded, &decoded)) + var restored jsonrpc.RPCErrorCodec + if wire.Code == rpcCodeBlockCleanedUp { + restored = &EpochRewardHistoryPrunedError{} + require.Nil(t, decoded.Data) + } else { + restored = &EpochRewardsPeriodActiveError{} + } + require.NoError(t, restored.FromJSONRPCError(decoded)) + require.Equal(t, original, restored) + decoded.Code = 1 + require.Error(t, restored.FromJSONRPCError(decoded)) + } +} + +func TestInflationRewardPartitionProgress(t *testing.T) { + server := &RpcServer{epochSchedule: &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 128}} + dir := t.TempDir() + require.NoError(t, server.EnableEpochRewards(dir, 512, 255)) + var first, last solana.PublicKey + for i := 1; first.IsZero() || last.IsZero(); i++ { + address := solana.PublicKey{byte(i)} + switch rewards.CalculateRewardPartitionForPubkey(address, [32]byte{}, 3) { + case 0: + first = address + case 2: + last = address + } + } + capture := func(slot, parent, height uint64, active bool) { + t.Helper() + block := &b.Block{Slot: slot, ParentSlot: parent, Epoch: 2, BlockHeight: height} + server.RecordEpochRewardStatus(block, rewardDistributionBank(t, slot, sealevel.SysvarEpochRewards{ + Active: active, DistributionStartingBlockHeight: 101, NumPartitions: 3, + }), true) + } + query := func(addresses ...string) ([]*InflationRewardResp, error) { + return server.GetInflationReward(context.Background(), rawParams(t, []any{addresses, map[string]any{"epoch": 1}})) + } + capture(256, 255, 100, true) + capture(260, 256, 101, true) // Skipped slots do not advance reward partitions. + capture(263, 260, 102, true) + server.PublishEpochRewards(256) + _, err := query(first.String()) + require.IsType(t, &EpochRewardsPeriodActiveError{}, err, "later executed banks are not yet verified") + server.PublishEpochRewards(260) + result, err := query(first.String()) + require.NoError(t, err) + require.Equal(t, []*InflationRewardResp{nil}, result, "absence in an already processed partition is known") + _, err = query(first.String(), last.String()) + require.IsType(t, &EpochRewardsPeriodActiveError{}, err, "one incomplete partition fails the whole request") + require.Equal(t, uint64(101), err.(*EpochRewardsPeriodActiveError).CurrentBlockHeight) + require.NoError(t, server.PrepareEpochRewards(260)) + require.NoError(t, server.SetRootedEpochRewardsSlot(260)) + require.NoError(t, server.EnableEpochRewards(dir, 512, 260)) + result, err = query(first.String()) + require.NoError(t, err) + require.Nil(t, result[0], "durable progress survives reopening") + _, err = query(last.String()) + require.IsType(t, &EpochRewardsPeriodActiveError{}, err) + capture(265, 260, 103, false) + server.PublishEpochRewards(265) + result, err = query(last.String()) + require.NoError(t, err) + require.Nil(t, result[0]) + require.NoError(t, server.RewindEpochRewards(265)) + _, err = query(last.String()) + require.IsType(t, &EpochRewardsPeriodActiveError{}, err, "rewind restores committed progress") +} + +func TestInflationRewardEmptyRequestRequiresHistory(t *testing.T) { + server := &RpcServer{epochSchedule: &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 128}} + require.NoError(t, server.EnableEpochRewards(t.TempDir(), 512, 255)) + _, err := server.GetInflationReward(context.Background(), rawParams(t, []any{[]string{}, map[string]any{"epoch": uint64(math.MaxUint64)}})) + require.IsType(t, &EpochRewardHistoryUnavailableError{}, err) +} + +func TestInflationRewardProgressRequiresCurrentBank(t *testing.T) { + server := &RpcServer{epochSchedule: &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 128}} + require.NoError(t, server.EnableEpochRewards(t.TempDir(), 512, 255)) + server.RecordEpochRewardStatus(&b.Block{Slot: 256, ParentSlot: 255, Epoch: 2, BlockHeight: 100}, + rewardDistributionBank(t, 256, sealevel.SysvarEpochRewards{Active: true, DistributionStartingBlockHeight: 101, NumPartitions: 3}), true) + server.PublishEpochRewards(256) + server.RecordEpochRewardStatus(&b.Block{Slot: 257, ParentSlot: 256, Epoch: 2, BlockHeight: 101}, rewardCoverageBank(t, 258, true), true) + server.PublishEpochRewards(257) + _, err := server.GetInflationReward(context.Background(), rawParams(t, []any{[]string{solana.PublicKey{1}.String()}})) + require.IsType(t, &EpochRewardHistoryUnavailableError{}, err, "older progress cannot be paired with a newer published context") +} + +func TestInflationRewardSkippedBoundaryRemainsReadable(t *testing.T) { + server := &RpcServer{epochSchedule: &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 128}} + require.NoError(t, server.EnableEpochRewards(t.TempDir(), 100, 255)) + server.RecordEpochRewardStatus(&b.Block{Slot: 260, ParentSlot: 255, Epoch: 2}, rewardCoverageBank(t, 260, false), false) + require.NoError(t, server.PrepareEpochRewards(260)) + require.NoError(t, server.SetRootedEpochRewardsSlot(357)) + result, err := server.GetInflationReward(context.Background(), rawParams(t, []any{[]string{solana.PublicKey{1}.String()}, map[string]any{"epoch": 1}})) + require.NoError(t, err, "an observed retained boundary is stronger evidence than the nominal skipped start slot") + require.Nil(t, result[0]) +} diff --git a/pkg/rpcserver/inflation_commitment_test.go b/pkg/rpcserver/inflation_commitment_test.go new file mode 100644 index 00000000..54d600db --- /dev/null +++ b/pkg/rpcserver/inflation_commitment_test.go @@ -0,0 +1,98 @@ +package rpcserver + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + b "github.com/Overclock-Validator/mithril/pkg/block" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/filecoin-project/go-jsonrpc" + "github.com/gagliardetto/solana-go" + "github.com/gagliardetto/solana-go/rpc" + "github.com/stretchr/testify/require" +) + +func TestInflationRewardHTTPUsesDurableEpoch(t *testing.T) { + server := NewRpcServer(nil, 0, &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 128}, solana.Hash{1}) + t.Cleanup(func() { require.NoError(t, server.listener.Close()) }) + require.NoError(t, server.EnableEpochRewards(t.TempDir(), 512, 0)) + address := solana.PublicKey{1} + reward := func(slot, epoch, amount uint64) *b.Block { + return &b.Block{Slot: slot, Epoch: epoch, Rewards: []rpc.BlockReward{ + {Pubkey: address, Lamports: int64(amount), PostBalance: 100 + amount, RewardType: rpc.RewardTypeVoting}, + }} + } + require.NoError(t, server.RecordEpochRewards(reward(128, 1, 5))) + require.NoError(t, server.PrepareEpochRewards(255)) + require.NoError(t, server.SetRootedEpochRewardsSlot(255)) + // An executed slot alone does not establish a certified or committed bank. + server.SetSlotCtx(&sealevel.SlotCtx{Slot: 260}) + require.NoError(t, server.RecordEpochRewards(reward(256, 2, 7))) + endpoint := httptest.NewServer(server) + t.Cleanup(endpoint.Close) + client := &http.Client{Timeout: 3 * time.Second} + call := func(config map[string]any) ([]*InflationRewardResp, *jsonrpc.JSONRPCError) { + t.Helper() + body, err := json.Marshal(map[string]any{"jsonrpc": "2.0", "id": 1, "method": "getInflationReward", + "params": []any{[]string{address.String()}, config}}) + require.NoError(t, err) + response, err := client.Post(endpoint.URL, "application/json", bytes.NewReader(body)) + require.NoError(t, err) + defer response.Body.Close() + var reply struct { + Result []*InflationRewardResp `json:"result"` + Error *jsonrpc.JSONRPCError `json:"error"` + } + require.NoError(t, json.NewDecoder(response.Body).Decode(&reply)) + return reply.Result, reply.Error + } + for _, phase := range []string{"pending", "prepared"} { + if phase == "prepared" { + require.NoError(t, server.PrepareEpochRewards(256)) + } + for _, commitment := range []string{"confirmed", "finalized"} { + result, err := call(map[string]any{"commitment": commitment}) + require.Nil(t, err, phase) + require.Len(t, result, 1) + require.Equal(t, uint64(0), result[0].Epoch) + require.Equal(t, uint64(5), result[0].Amount) + _, err = call(map[string]any{"commitment": commitment, "epoch": 1}) + require.NotNil(t, err) + require.Equal(t, jsonrpc.ErrorCode(-32004), err.Code) + _, err = call(map[string]any{"commitment": commitment, "minContextSlot": 256}) + require.NotNil(t, err) + require.Equal(t, jsonrpc.ErrorCode(-32016), err.Code) + require.Equal(t, map[string]any{"contextSlot": float64(255)}, err.Data) + } + } + // Replay's verified publication can make both commitments readable before + // an account checkpoint. Pending capture and prepared files above did not. + server.PublishEpochRewards(256) + for _, commitment := range []string{"confirmed", "finalized"} { + result, err := call(map[string]any{"commitment": commitment, "minContextSlot": 256}) + require.Nil(t, err) + require.Equal(t, uint64(1), result[0].Epoch) + require.Equal(t, uint64(7), result[0].Amount) + } + require.Equal(t, uint64(255), server.epochRewards.RootedSlot()) + server.SetHistoryRecoveryPending(true) + _, rpcErr := call(map[string]any{"minContextSlot": 256}) + require.NotNil(t, rpcErr) + server.PublishEpochRewards(256) + server.SetHistoryRecoveryPending(false) + _, rpcErr = call(map[string]any{"minContextSlot": 256}) + require.NotNil(t, rpcErr, "publication during recovery must not survive reopening the gate") + require.Equal(t, jsonrpc.ErrorCode(-32016), rpcErr.Code) + require.NoError(t, server.SetRootedEpochRewardsSlot(256)) + for _, commitment := range []string{"confirmed", "finalized"} { + result, err := call(map[string]any{"commitment": commitment, "minContextSlot": 256}) + require.Nil(t, err) + require.Len(t, result, 1) + require.Equal(t, uint64(1), result[0].Epoch) + require.Equal(t, uint64(7), result[0].Amount) + } +} diff --git a/pkg/rpcserver/inflation_publication_test.go b/pkg/rpcserver/inflation_publication_test.go new file mode 100644 index 00000000..982c687e --- /dev/null +++ b/pkg/rpcserver/inflation_publication_test.go @@ -0,0 +1,94 @@ +package rpcserver + +import ( + "context" + "fmt" + "sync" + "testing" + + b "github.com/Overclock-Validator/mithril/pkg/block" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/gagliardetto/solana-go" + "github.com/gagliardetto/solana-go/rpc" + "github.com/stretchr/testify/require" +) + +func TestInflationRewardVerifiedPublicationBeforeDurableFold(t *testing.T) { + server := &RpcServer{epochSchedule: &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 128}} + require.NoError(t, server.EnableEpochRewards(t.TempDir(), 512, 255)) + address := solana.PublicKey{1} + block := &b.Block{Slot: 256, ParentSlot: 255, Epoch: 2, Rewards: []rpc.BlockReward{ + {Pubkey: address, Lamports: 7, PostBalance: 107, RewardType: rpc.RewardTypeVoting}, + }} + require.NoError(t, server.RecordEpochRewards(block)) + server.RecordEpochRewardStatus(block, rewardCoverageBank(t, 256, false), true) + params := rawParams(t, []any{[]string{address.String()}, map[string]any{"minContextSlot": 256}}) + _, err := server.GetInflationReward(context.Background(), params) + require.Error(t, err, "execution alone is not verified publication") + server.PublishEpochRewards(256) + result, err := server.GetInflationReward(context.Background(), params) + require.NoError(t, err, "a verified canonical reward must not wait for a durable fold") + require.Len(t, result, 1) + require.Equal(t, uint64(1), result[0].Epoch) + require.Equal(t, uint64(7), result[0].Amount) + require.Equal(t, uint64(255), server.epochRewards.RootedSlot(), "read publication must not advance the recovery checkpoint") +} + +func TestInflationRewardPublicationPinsWholeRequest(t *testing.T) { + server := &RpcServer{epochSchedule: &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 128}} + require.NoError(t, server.EnableEpochRewards(t.TempDir(), 100000, 0)) + addresses := make([]string, maxInflationRewardAddresses) + for i := range addresses { + addresses[i] = solana.PublicKey{byte(i + 1)}.String() + } + params := rawParams(t, []any{addresses}) + publish := func(epoch uint64) error { + block := &b.Block{Slot: epoch * 128, ParentSlot: epoch*128 - 1, Epoch: epoch} + for _, address := range addresses { + block.Rewards = append(block.Rewards, rpc.BlockReward{Pubkey: solana.MustPublicKeyFromBase58(address), + Lamports: int64(epoch), PostBalance: 100 + epoch, RewardType: rpc.RewardTypeVoting}) + } + if err := server.RecordEpochRewards(block); err != nil { + return err + } + server.PublishEpochRewards(block.Slot) + return nil + } + require.NoError(t, publish(1)) + var wg sync.WaitGroup + errors := make(chan error, 5) + wg.Add(1) + go func() { + defer wg.Done() + for epoch := uint64(2); epoch <= 128; epoch++ { + if err := publish(epoch); err != nil { + errors <- err + return + } + } + }() + for reader := 0; reader < 4; reader++ { + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < 128; i++ { + result, err := server.GetInflationReward(context.Background(), params) + if err != nil { + errors <- err + return + } + for _, record := range result { + if record == nil || record.Epoch != result[0].Epoch || record.Amount != record.Epoch+1 || record.EffectiveSlot != (record.Epoch+1)*128 { + errors <- fmt.Errorf("mixed reward view: %+v", result) + return + } + } + } + }() + } + wg.Wait() + close(errors) + for err := range errors { + require.NoError(t, err) + } +} diff --git a/pkg/rpcserver/rpc_registration_test.go b/pkg/rpcserver/rpc_registration_test.go new file mode 100644 index 00000000..2d781021 --- /dev/null +++ b/pkg/rpcserver/rpc_registration_test.go @@ -0,0 +1,91 @@ +package rpcserver + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/gagliardetto/solana-go" +) + +func TestRPCHandlerMethodSet(t *testing.T) { + typ := reflect.TypeOf(rpcHandler{}) + if typ.NumMethod() != len(supportedRPCMethods) { + t.Fatalf("registered %d methods, public method list has %d", typ.NumMethod(), len(supportedRPCMethods)) + } + for i := 0; i < typ.NumMethod(); i++ { + name := typ.Method(i).Name + name = strings.ToLower(name[:1]) + name[1:] + if _, ok := supportedRPCMethods[name]; !ok { + t.Fatalf("non-public method registered: %s", name) + } + } +} + +func TestInternalMethodsAreNotRPC(t *testing.T) { + for _, mode := range []string{"single", "mixed-batch", "oversized-single", "direct-dispatch"} { + t.Run(mode, func(t *testing.T) { + rpc := NewRpcServer(nil, 0, &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 32}, solana.Hash{1}) + t.Cleanup(func() { _ = rpc.listener.Close() }) + directory := filepath.Join(t.TempDir(), "history") + call := map[string]any{"jsonrpc": "2.0", "id": 1, "method": "enableEpochRewards", "params": []any{directory, 100, 0}} + if mode == "oversized-single" { + call["padding"] = strings.Repeat("x", maxQuietMethodProbeBody+1) + } + var request any = call + if mode == "mixed-batch" { + request = []any{map[string]any{"jsonrpc": "2.0", "id": 2, "method": "getGenesisHash", "params": []any{}}, call} + } + body, err := json.Marshal(request) + if err != nil { + t.Fatal(err) + } + response := httptest.NewRecorder() + if mode == "direct-dispatch" { + rpc.rpcService.HandleRequest(context.Background(), bytes.NewReader(body), response) + } else { + rpc.ServeHTTP(response, httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(body))) + } + var replies []struct { + ID int `json:"id"` + Result string `json:"result"` + Error *struct { + Code int `json:"code"` + } `json:"error"` + } + result := response.Body.Bytes() + if mode != "mixed-batch" { + result = append(append([]byte{'['}, result...), ']') + } + if err := json.Unmarshal(result, &replies); err != nil { + t.Fatalf("decode response %q: %v", result, err) + } + found := false + for _, reply := range replies { + if reply.ID == 1 { + found = true + if reply.Error == nil || reply.Error.Code != -32601 { + t.Fatalf("internal method was not rejected: %s", result) + } + } + if reply.ID == 2 && (reply.Error != nil || reply.Result != (solana.Hash{1}).String()) { + t.Fatalf("public batch method failed: %s", result) + } + } + if !found || (mode == "mixed-batch" && len(replies) != 2) { + t.Fatalf("missing response: %s", result) + } + if _, err := os.Stat(directory); !os.IsNotExist(err) { + t.Fatalf("internal request created history directory: %v", err) + } + }) + } +} diff --git a/pkg/rpcserver/rpcserver.go b/pkg/rpcserver/rpcserver.go index 8a3dd7af..b8459d61 100644 --- a/pkg/rpcserver/rpcserver.go +++ b/pkg/rpcserver/rpcserver.go @@ -14,6 +14,8 @@ import ( "time" "github.com/Overclock-Validator/mithril/pkg/accountsdb" + b "github.com/Overclock-Validator/mithril/pkg/block" + "github.com/Overclock-Validator/mithril/pkg/epochrewards" "github.com/Overclock-Validator/mithril/pkg/mlog" "github.com/Overclock-Validator/mithril/pkg/sealevel" "github.com/filecoin-project/go-jsonrpc" @@ -21,7 +23,28 @@ import ( "github.com/gagliardetto/solana-go" ) +// rpcHandler exposes only public RPC methods to go-jsonrpc's reflection-based +// registration. Internal setters and lifecycle hooks remain Go-only. +type rpcHandler struct { + rpcMethods +} + +type rpcMethods interface { + GetAccountInfo(context.Context, jsonrpc.RawParams) (GetAccountInfoResp, error) + GetBankHash(context.Context, jsonrpc.RawParams) (string, error) + GetBlockHeight(context.Context, jsonrpc.RawParams) (uint64, error) + GetEpochInfo(context.Context, jsonrpc.RawParams) (GetEpochInfoResp, error) + GetGenesisHash(context.Context, jsonrpc.RawParams) (string, error) + GetInflationReward(context.Context, jsonrpc.RawParams) ([]*InflationRewardResp, error) + GetLatestBlockhash(context.Context, jsonrpc.RawParams) (GetLatestBlockhashResp, error) + SendTransaction(context.Context, jsonrpc.RawParams) (string, error) + SimulateTransaction(context.Context, jsonrpc.RawParams) (SimulateTransactionResp, error) +} + type RpcServer struct { + historyRecoveryMu sync.RWMutex + historyRecoveryPending bool + isReady bool rpcService *jsonrpc.RPCServer serv *httptest.Server @@ -31,6 +54,7 @@ type RpcServer struct { slotCtx *sealevel.SlotCtx slotCtxMu sync.RWMutex genesisHash string + epochRewards *epochrewards.Store leaderTPUCacheMu sync.RWMutex leaderTPUByIdentity map[solana.PublicKey]tpuEndpoint @@ -53,11 +77,129 @@ var supportedRPCMethods = map[string]struct{}{ "getBlockHeight": {}, "getEpochInfo": {}, "getGenesisHash": {}, + "getInflationReward": {}, "getLatestBlockhash": {}, "sendTransaction": {}, "simulateTransaction": {}, } +// EnableEpochRewards attaches durable Alpenglow reward history to the RPC server. +func (rpcServer *RpcServer) EnableEpochRewards(dir string, retentionSlots, rootedSlot uint64) error { + store, err := epochrewards.Open(dir, retentionSlots) + if err != nil { + return err + } + if err := store.SetRooted(rootedSlot); err != nil { + return err + } + rpcServer.epochRewards = store + return nil +} + +// RecordEpochRewards records rewards from a successfully replayed block. +func (rpcServer *RpcServer) RecordEpochRewards(block *b.Block) error { + if rpcServer == nil || rpcServer.epochRewards == nil { + return nil + } + return rpcServer.epochRewards.RecordBlock(block) +} + +// RecordEpochRewardStatus records coverage from the successfully replayed bank. +func (rpcServer *RpcServer) RecordEpochRewardStatus(block *b.Block, bank *sealevel.BankSysvars, partitioned bool) { + if rpcServer == nil || rpcServer.epochRewards == nil || rpcServer.epochSchedule == nil || block == nil || block.Epoch == 0 { + return + } + if block.ParentSlot >= block.Slot || rpcServer.epochSchedule.GetEpoch(block.Slot) != block.Epoch { + return + } + boundary := rpcServer.epochSchedule.GetEpoch(block.ParentSlot) < block.Epoch + complete := false + if bank != nil && bank.Slot() == block.Slot { + if partitioned { + rewards, ok := bank.EpochRewards() + complete = ok && !rewards.Active + } else { + complete = boundary + } + } + rpcServer.epochRewards.RecordCoverage(block.Epoch-1, block.Slot, boundary, complete) + if partitioned && bank != nil && bank.Slot() == block.Slot && block.BlockHeight != 0 { + if rewards, ok := bank.EpochRewards(); ok && rewards.Active { + rpcServer.epochRewards.RecordDistribution(block.Epoch-1, block.Slot, epochrewards.Distribution{ + BlockHeight: block.BlockHeight, StartingBlockHeight: rewards.DistributionStartingBlockHeight, + NumPartitions: rewards.NumPartitions, ParentBlockhash: rewards.ParentBlockhash, + }) + } + } +} + +// PrepareEpochRewards stages rewards with the matching durable fold. +func (rpcServer *RpcServer) PrepareEpochRewards(through uint64) error { + if rpcServer == nil || rpcServer.epochRewards == nil { + return nil + } + rpcServer.historyRecoveryMu.Lock() + defer rpcServer.historyRecoveryMu.Unlock() + return rpcServer.epochRewards.Prepare(through) +} + +// PublishEpochRewards exposes a prefix that replay has verified as canonical. +// It is independent of the durable account checkpoint and is not an RPC method. +func (rpcServer *RpcServer) PublishEpochRewards(slot uint64) { + if rpcServer == nil || rpcServer.epochRewards == nil { + return + } + rpcServer.historyRecoveryMu.Lock() + defer rpcServer.historyRecoveryMu.Unlock() + if !rpcServer.historyRecoveryPending { + rpcServer.epochRewards.Publish(slot) + } +} + +// ResetEpochRewardView withdraws process-local publication during replay recovery. +func (rpcServer *RpcServer) ResetEpochRewardView() { + if rpcServer == nil || rpcServer.epochRewards == nil { + return + } + rpcServer.historyRecoveryMu.Lock() + defer rpcServer.historyRecoveryMu.Unlock() + rpcServer.epochRewards.ResetPublished() +} + +// SetRootedEpochRewardsSlot sets the durable reward watermark. +func (rpcServer *RpcServer) SetRootedEpochRewardsSlot(slot uint64) error { + if rpcServer == nil || rpcServer.epochRewards == nil { + return nil + } + rpcServer.historyRecoveryMu.Lock() + defer rpcServer.historyRecoveryMu.Unlock() + return rpcServer.epochRewards.SetRooted(slot) +} + +// RewindEpochRewards drops speculative rewards from a discarded fork suffix. +func (rpcServer *RpcServer) RewindEpochRewards(fromSlot uint64) error { + if rpcServer == nil || rpcServer.epochRewards == nil { + return nil + } + rpcServer.historyRecoveryMu.Lock() + defer rpcServer.historyRecoveryMu.Unlock() + return rpcServer.epochRewards.Rewind(fromSlot) +} + +// SetHistoryRecoveryPending blocks history reads during a durable account rewind. +// Failed recovery leaves the gate closed without deleting retained records. +func (rpcServer *RpcServer) SetHistoryRecoveryPending(pending bool) { + if rpcServer == nil { + return + } + rpcServer.historyRecoveryMu.Lock() + defer rpcServer.historyRecoveryMu.Unlock() + rpcServer.historyRecoveryPending = pending + if pending && rpcServer.epochRewards != nil { + rpcServer.epochRewards.ResetPublished() + } +} + func NewRpcServer(acctsDb *accountsdb.AccountsDb, port uint16, epochSchedule *sealevel.SysvarEpochSchedule, genesisHash solana.Hash) *RpcServer { var err error rpcServer := &RpcServer{genesisHash: genesisHash.String()} @@ -77,7 +219,7 @@ func NewRpcServer(acctsDb *accountsdb.AccountsDb, port uint16, epochSchedule *se jsonrpc.WithServerErrors(rpcErrors), ) - rpcServer.rpcService.Register("MithrilRpc", rpcServer) + rpcServer.rpcService.Register("MithrilRpc", rpcHandler{rpcServer}) rpcServer.acctsDb = acctsDb if epochSchedule != nil { rpcServer.epochSchedule = epochSchedule diff --git a/pkg/snapshot/build_db.go b/pkg/snapshot/build_db.go index b0004cfe..dbc9745c 100644 --- a/pkg/snapshot/build_db.go +++ b/pkg/snapshot/build_db.go @@ -51,6 +51,7 @@ func CleanAccountsDbDir(accountsDbDir string) { // so retaining them would waste space and make stale diagnostics look // actionable. "transaction-status-checkpoints", + "rpc-epoch-rewards", "mithril_db", "mithril_db_log_shards", "bankhash_db", diff --git a/pkg/snapshot/cleanup_test.go b/pkg/snapshot/cleanup_test.go index d7799049..0df9377c 100644 --- a/pkg/snapshot/cleanup_test.go +++ b/pkg/snapshot/cleanup_test.go @@ -9,12 +9,16 @@ import ( "github.com/stretchr/testify/require" ) -func TestCleanAccountsDbDirRemovesTransactionStatusCheckpoints(t *testing.T) { +func TestCleanAccountsDbDirRemovesReplaySidecars(t *testing.T) { root := t.TempDir() checkpointDir := filepath.Join(root, "transaction-status-checkpoints") + rewardDir := filepath.Join(root, "rpc-epoch-rewards") require.NoError(t, os.MkdirAll(checkpointDir, 0o755)) + require.NoError(t, os.MkdirAll(rewardDir, 0o755)) require.NoError(t, os.WriteFile(filepath.Join(checkpointDir, "stale.bin"), []byte("stale"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(rewardDir, "stale.json"), []byte("stale"), 0o644)) CleanAccountsDbDir(root) assert.NoDirExists(t, checkpointDir) + assert.NoDirExists(t, rewardDir) } diff --git a/pkg/turbine/repair_pacing_test.go b/pkg/turbine/repair_pacing_test.go index ad747cb1..c4e00b67 100644 --- a/pkg/turbine/repair_pacing_test.go +++ b/pkg/turbine/repair_pacing_test.go @@ -5,6 +5,7 @@ import ( "crypto/rand" "net" "testing" + "testing/synctest" "time" "github.com/Overclock-Validator/mithril/pkg/gossip" @@ -537,6 +538,11 @@ func TestAdmissionCapBoundsOutstanding(t *testing.T) { // (the scan covers the slot on its own cadence), a full one sends the // followup cap plus the reserved chained probe and keeps the remainder. func TestFollowupsAreMeteredByTokenBucket(t *testing.T) { + // Response admission must not refill the bucket just because the test is slow. + synctest.Test(t, testFollowupsAreMeteredByTokenBucket) +} + +func testFollowupsAreMeteredByTokenBucket(t *testing.T) { conn, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1)}) if err != nil { t.Fatalf("listen: %v", err) @@ -564,9 +570,7 @@ func TestFollowupsAreMeteredByTokenBucket(t *testing.T) { } c, from, packet, shred := prime() - // Drain the bucket. Two calls: the admission cap trims the first grant - // by the primed outstanding entry, leaving a stray token. - c.takeRateTokens(repairMaxRequestsPerSecond) + // Drain the initial allowance without advancing the clock. c.takeRateTokens(repairMaxRequestsPerSecond) if !observeRepairForTest(c, conn, packet, from, shred) { t.Fatal("response itself must match") @@ -574,6 +578,10 @@ func TestFollowupsAreMeteredByTokenBucket(t *testing.T) { if got := c.requests.Load(); got != 0 { t.Fatalf("drained bucket sent %d followups, want 0", got) } + time.Sleep(time.Millisecond) + if got, want := c.takeRateTokens(repairMaxRequestsPerSecond), repairMaxRequestsPerSecond/1000; got != want { + t.Fatalf("refilled bucket granted %d tokens, want %d after 1 ms", got, want) + } c, from, packet, shred = prime() c.mu.Lock()